1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24 ProviderRole,
25 },
26 session::{
27 HealthReport, ModuleControlPush, ModuleControlRequest, ModuleControlRequestFromModule,
28 ModuleControlResponse, ModuleControlResponseToModule, MODULE_CONTROL_OP_HEALTH_CHECK,
29 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
30 },
31 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
32 Priority, RouteTarget, PROTOCOL_VERSION,
33};
34use tokio::time::{timeout_at, Instant};
35use tracing::{debug, info, warn};
36
37use crate::{
38 capability_requirements::{
39 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
40 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
41 RuntimeModule,
42 },
43 daemon_config::RestartRequiredSection,
44 forwarding::{
45 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
46 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
47 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
48 },
49 observability::{
50 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
51 },
52 provenance::{
53 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
54 },
55 registry::{ChannelState, ConnectionId, Registry, RegistryError},
56 router::{RouteCtx, RouterError},
57 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
58 stderr_tail::{CaptureState, TailEntry},
59 supervise::{
60 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
61 SupervisorHandle, SwapHelloAdmission,
62 },
63 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
64};
65
66pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
72
73const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
74const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
75const CAP_PING_PONG: &str = "ping_pong_v1";
76const CAP_SESSION_ATTACH: &str = "session_attach_v1";
77const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
78
79const SUBC_CONTROL_OPS: &[&str] = &[
80 ops::SERVER_DESCRIBE,
81 ops::CATALOG_LIST,
82 ops::ROUTE_OPEN,
83 ops::ROUTE_POLL,
84 ops::ROUTE_CLOSING,
85 ops::ROUTE_CLOSED,
86 ops::SUPERVISOR_LIST,
87 ops::SUPERVISOR_RESTART,
88 ops::SUPERVISOR_SWAP,
89 ops::SUPERVISOR_RELOAD,
90 ops::SUPERVISOR_RESCAN,
91 ops::SUPERVISOR_RELEASE_RESERVED,
92 ops::SUPERVISOR_SET_ENABLED,
93 ops::SUPERVISOR_HEALTH_PROBE,
94 ops::SUPERVISOR_HEALTH,
95 ops::SUPERVISOR_STDERR_TAIL,
96 ops::SUPERVISOR_TERMINALS,
97 ops::SUPERVISOR_ROUTES,
98 ops::SUPERVISOR_PROVENANCE,
99 ops::SUPERVISOR_SPAWN_SNAPSHOT,
100 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
101];
102
103const MODULE_TO_SUBC_CONTROL_OPS: &[&str] =
104 &[MODULE_TO_SUBC_OP_CATALOG_UPDATE, "supervisor.live_roots"];
105
106const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
107
108pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
116
117pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
132
133pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
151
152const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
153const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
154
155fn reload_verdict(
156 configured: &Path,
157 spawned_from: Option<&Path>,
158 image: subc_control::RunningImageAgreement,
159) -> PendingReloadVerdict {
160 let path = match spawned_from {
161 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
162 Some(spawned_from) => ReloadPathAgreement::Mismatch {
163 configured: configured.to_path_buf(),
164 spawned_from: spawned_from.to_path_buf(),
165 },
166 None => ReloadPathAgreement::Unavailable {
167 reason: if matches!(
168 image,
169 subc_control::RunningImageAgreement::Unavailable {
170 reason: subc_control::RunningImageUnavailableReason::NotRunning
171 }
172 ) {
173 ReloadPathUnavailableReason::NotRunning
174 } else {
175 ReloadPathUnavailableReason::SpawnedPathUnavailable
176 },
177 },
178 };
179 PendingReloadVerdict { path, image }
180}
181
182#[derive(Clone)]
183struct DaemonProvenanceFacts {
184 build: DaemonBuildProvenance,
185 pid: Option<u32>,
186 started_at_ms: Option<u64>,
187 start_clock: Option<crate::clock::StartClock>,
188 executable_path: Option<PathBuf>,
189 executable_identity: Option<SpawnedFileIdentity>,
190 process_start_time: Option<u64>,
191 probe: ExecutableIdentityProbe,
192}
193
194impl Default for DaemonProvenanceFacts {
195 fn default() -> Self {
196 Self {
197 build: DaemonBuildProvenance {
198 build_git_sha: None,
199 build_lock_digest: None,
200 },
201 pid: None,
202 started_at_ms: None,
203 start_clock: None,
204 executable_path: None,
205 executable_identity: None,
206 process_start_time: None,
207 probe: ExecutableIdentityProbe::default(),
208 }
209 }
210}
211
212#[derive(Debug, Clone)]
213struct SupervisorRescanContext {
214 supervisor: Supervisor,
215 config_path: PathBuf,
216 configured_port: Option<u16>,
217 storage_config: Option<crate::daemon_config::StorageConfig>,
218 admission_facts_carrier_module_id: Option<String>,
219 admission_facts_targets: Option<Vec<String>>,
220}
221
222#[derive(Clone)]
224pub struct ControlHandler {
225 registry: Arc<Registry>,
226 forwarding: Arc<ForwardingTable>,
227 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
228 supervisor: SupervisorHandle,
229 subc_capabilities: Arc<[String]>,
230 route_bind_relay_timeout: Duration,
234 route_bind_relay_timeouts: BTreeMap<String, Duration>,
238 route_bind_breakers: RouteBindBreakers,
241 route_bind_concurrency: RouteBindConcurrency,
244 route_bind_breaker_threshold: u32,
246 route_bind_breaker_cooldown: Duration,
248 health_probe_timeout: Duration,
249 storage_config: Option<crate::daemon_config::StorageConfig>,
252 machine_id: Option<crate::machine_id::MachineId>,
256 admission_facts_carrier_module_id: Option<String>,
257 admission_facts_targets: Option<Vec<String>>,
258 rescan: Option<SupervisorRescanContext>,
259 connected_clients: ConnectedClients,
260 counters: DaemonCounters,
261 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
262 daemon_provenance: DaemonProvenanceFacts,
263 #[cfg(test)]
264 control_dispatch_delay: Option<Duration>,
265 #[cfg(test)]
266 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
267}
268
269impl fmt::Debug for ControlHandler {
270 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
271 f.debug_struct("ControlHandler")
272 .field("registry", &self.registry)
273 .field("forwarding", &self.forwarding)
274 .field("process_liveness", &self.process_liveness.is_some())
275 .field("supervisor", &self.supervisor)
276 .field("subc_capabilities", &self.subc_capabilities)
277 .finish()
278 }
279}
280
281struct RouteOpenRequest {
282 target: RouteTarget,
283 identity: BindIdentity,
284 consumer_identity: Option<ConsumerIdentity>,
285 consumer_capabilities: Option<Vec<String>>,
286 admission_facts: Option<serde_json::Value>,
287}
288
289struct RouteBindReservationGuard {
290 forwarding: Arc<ForwardingTable>,
291 endpoint: ModuleEndpointId,
292 relay_corr: u64,
293 armed: bool,
294}
295
296struct ModuleControlRpcGuard {
297 forwarding: Arc<ForwardingTable>,
298 endpoint: ModuleEndpointId,
299 corr: u64,
300 armed: bool,
301}
302
303impl ModuleControlRpcGuard {
304 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
305 Self {
306 forwarding,
307 endpoint,
308 corr,
309 armed: true,
310 }
311 }
312
313 fn disarm(&mut self) {
314 self.armed = false;
315 }
316}
317
318impl Drop for ModuleControlRpcGuard {
319 fn drop(&mut self) {
320 if self.armed {
321 let _ = self
322 .forwarding
323 .cancel_module_control_rpc(self.endpoint, self.corr);
324 }
325 }
326}
327
328impl RouteBindReservationGuard {
329 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
330 Self {
331 forwarding,
332 endpoint,
333 relay_corr,
334 armed: true,
335 }
336 }
337
338 fn release_and_disarm(&mut self) {
339 if !self.armed {
340 return;
341 }
342 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
343 self.endpoint,
344 self.relay_corr,
345 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
346 ) {
347 send_goodbye_target_best_effort(
348 &self.forwarding.counters(),
349 &target,
350 "canceled route.bind",
351 );
352 }
353 self.armed = false;
354 }
355
356 fn disarm(&mut self) {
357 self.armed = false;
358 }
359}
360
361impl Drop for RouteBindReservationGuard {
362 fn drop(&mut self) {
363 self.release_and_disarm();
364 }
365}
366
367#[derive(Debug, Clone, Default)]
397pub(crate) struct RouteBindBreakers {
398 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
399}
400
401#[derive(Debug, Clone, Default)]
402pub(crate) struct RouteBindConcurrency {
403 modules: Arc<Mutex<HashMap<String, usize>>>,
404}
405
406struct RouteBindConcurrencyGuard {
407 concurrency: RouteBindConcurrency,
408 module_id: String,
409}
410
411impl RouteBindConcurrency {
412 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
415 let mut modules = self
416 .modules
417 .lock()
418 .expect("route.bind concurrency mutex poisoned");
419 let in_flight = modules.entry(module_id.to_string()).or_default();
420 if *in_flight >= limit {
421 return Err(*in_flight);
422 }
423 *in_flight += 1;
424 Ok(RouteBindConcurrencyGuard {
425 concurrency: self.clone(),
426 module_id: module_id.to_string(),
427 })
428 }
429}
430
431impl Drop for RouteBindConcurrencyGuard {
432 fn drop(&mut self) {
433 let mut modules = self
434 .concurrency
435 .modules
436 .lock()
437 .expect("route.bind concurrency mutex poisoned");
438 let remove = {
439 let in_flight = modules
440 .get_mut(&self.module_id)
441 .expect("admitted route.bind has a concurrency entry");
442 *in_flight -= 1;
443 *in_flight == 0
444 };
445 if remove {
446 modules.remove(&self.module_id);
447 }
448 }
449}
450
451#[derive(Debug, Default)]
452struct ModuleBreakerState {
453 consecutive_timeouts: u32,
455 cooldown_until: Option<Instant>,
458 probe_in_flight: bool,
463}
464
465enum RouteBindAdmission<'a> {
467 Admitted {
468 guard: RouteBindBreakerGuard<'a>,
469 probe: bool,
472 },
473 Refused {
474 consecutive_timeouts: u32,
475 retry_in: Duration,
479 probe_in_flight: bool,
480 },
481}
482
483struct RouteBindBreakerGuard<'a> {
490 breakers: RouteBindBreakers,
491 module_id: &'a str,
492 settled: bool,
493}
494
495impl RouteBindBreakerGuard<'_> {
496 fn record_accepted(&mut self) -> bool {
500 self.settled = true;
501 self.breakers.record_accepted(self.module_id)
502 }
503
504 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
507 self.settled = true;
508 self.breakers
509 .record_timeout(self.module_id, threshold, cooldown)
510 }
511
512 fn record_inconclusive(&mut self) {
521 self.settled = true;
522 self.breakers.record_inconclusive(self.module_id);
523 }
524}
525
526impl Drop for RouteBindBreakerGuard<'_> {
527 fn drop(&mut self) {
528 if !self.settled {
529 self.breakers.record_inconclusive(self.module_id);
530 }
531 }
532}
533
534struct BreakerOpened {
538 consecutive_timeouts: u32,
539 reopened_after_probe: bool,
542}
543
544impl RouteBindBreakers {
545 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
546 self.modules
547 .lock()
548 .expect("route.bind breaker mutex poisoned")
549 }
550
551 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
554 let admitted = |probe| RouteBindAdmission::Admitted {
555 guard: RouteBindBreakerGuard {
556 breakers: self.clone(),
557 module_id,
558 settled: false,
559 },
560 probe,
561 };
562
563 let mut modules = self.lock();
564 let Some(state) = modules.get_mut(module_id) else {
565 return admitted(false);
566 };
567 let Some(cooldown_until) = state.cooldown_until else {
568 return admitted(false);
569 };
570 if state.probe_in_flight {
571 return RouteBindAdmission::Refused {
572 consecutive_timeouts: state.consecutive_timeouts,
573 retry_in: Duration::ZERO,
574 probe_in_flight: true,
575 };
576 }
577 let now = Instant::now();
578 if now < cooldown_until {
579 return RouteBindAdmission::Refused {
580 consecutive_timeouts: state.consecutive_timeouts,
581 retry_in: cooldown_until - now,
582 probe_in_flight: false,
583 };
584 }
585 state.probe_in_flight = true;
586 admitted(true)
587 }
588
589 fn record_accepted(&self, module_id: &str) -> bool {
590 self.lock()
591 .remove(module_id)
592 .is_some_and(|state| state.cooldown_until.is_some())
593 }
594
595 fn record_timeout(
596 &self,
597 module_id: &str,
598 threshold: u32,
599 cooldown: Duration,
600 ) -> Option<BreakerOpened> {
601 let mut modules = self.lock();
602 let state = modules.entry(module_id.to_string()).or_default();
603 let was_open = state.cooldown_until.is_some();
604 let was_probe = state.probe_in_flight;
605 state.probe_in_flight = false;
606 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
607 if state.consecutive_timeouts < threshold {
608 return None;
609 }
610 state.cooldown_until = Some(Instant::now() + cooldown);
611 Some(BreakerOpened {
612 consecutive_timeouts: state.consecutive_timeouts,
613 reopened_after_probe: was_open && was_probe,
614 })
615 }
616
617 fn record_inconclusive(&self, module_id: &str) {
618 if let Some(state) = self.lock().get_mut(module_id) {
619 state.probe_in_flight = false;
620 }
621 }
622
623 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
638 self.lock()
639 .remove(module_id)
640 .map(|state| state.consecutive_timeouts)
641 .filter(|discarded| *discarded > 0)
642 }
643
644 fn open_snapshot(&self) -> Option<serde_json::Value> {
651 let now = Instant::now();
652 let modules = self.lock();
653 let open = modules
654 .iter()
655 .filter_map(|(module_id, state)| {
656 let cooldown_until = state.cooldown_until?;
657 Some((
658 module_id.clone(),
659 serde_json::json!({
660 "consecutive_timeouts": state.consecutive_timeouts,
661 "cooldown_remaining_ms":
662 cooldown_until.saturating_duration_since(now).as_millis() as u64,
663 "probe_in_flight": state.probe_in_flight,
664 }),
665 ))
666 })
667 .collect::<serde_json::Map<String, serde_json::Value>>();
668 (!open.is_empty()).then_some(serde_json::Value::Object(open))
669 }
670}
671
672impl ControlHandler {
673 pub fn new(registry: Arc<Registry>) -> Self {
674 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
675 }
676
677 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
678 let counters = forwarding.counters();
679 let route_bind_breakers = forwarding.route_bind_breakers();
683 let route_bind_concurrency = forwarding.route_bind_concurrency();
684 Self {
685 registry,
686 forwarding,
687 process_liveness: None,
688 supervisor: SupervisorHandle::new(),
689 subc_capabilities: Arc::from([
690 CAP_MANIFEST_REGISTRATION.to_string(),
691 CAP_CHANNEL_LIFECYCLE.to_string(),
692 CAP_PING_PONG.to_string(),
693 CAP_SESSION_ATTACH.to_string(),
694 CAP_ADMISSION_FACTS_RELAY.to_string(),
695 ]),
696 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
697 route_bind_relay_timeouts: BTreeMap::new(),
698 route_bind_breakers,
699 route_bind_concurrency,
700 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
701 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
702 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
703 storage_config: None,
704 machine_id: None,
705 admission_facts_carrier_module_id: None,
706 admission_facts_targets: None,
707 rescan: None,
708 connected_clients: ConnectedClients::new(),
709 counters,
710 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
711 daemon_provenance: DaemonProvenanceFacts::default(),
712 #[cfg(test)]
713 control_dispatch_delay: None,
714 #[cfg(test)]
715 provenance_probe_override: None,
716 }
717 }
718
719 pub fn with_storage_config(
722 mut self,
723 storage_config: Option<crate::daemon_config::StorageConfig>,
724 ) -> Self {
725 self.storage_config = storage_config;
726 self
727 }
728
729 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
732 self.machine_id = machine_id;
733 self
734 }
735
736 pub fn with_admission_facts_config(
740 mut self,
741 carrier_module_id: Option<String>,
742 targets: Option<Vec<String>>,
743 ) -> Self {
744 self.admission_facts_carrier_module_id = carrier_module_id;
745 self.admission_facts_targets = targets;
746 self
747 }
748
749 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
752 self.route_bind_relay_timeout = timeout;
753 self
754 }
755
756 pub fn with_route_bind_relay_timeouts(
762 mut self,
763 timeouts: impl IntoIterator<Item = (String, Duration)>,
764 ) -> Self {
765 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
766 self
767 }
768
769 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
775 self.route_bind_relay_timeouts
776 .get(module_id)
777 .copied()
778 .unwrap_or(self.route_bind_relay_timeout)
779 }
780
781 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
789 self.route_bind_breaker_threshold = threshold.max(1);
790 self.route_bind_breaker_cooldown = cooldown;
791 self
792 }
793
794 #[cfg(test)]
795 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
796 self.health_probe_timeout = timeout;
797 self
798 }
799
800 #[cfg(test)]
801 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
802 self.control_dispatch_delay = Some(delay);
803 self
804 }
805
806 pub fn with_process_liveness(
807 mut self,
808 process_liveness: Arc<dyn ModuleProcessLiveness>,
809 ) -> Self {
810 self.process_liveness = Some(process_liveness);
811 self
812 }
813
814 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
815 self.supervisor = supervisor;
816 self
817 }
818
819 pub fn with_daemon_provenance(
820 mut self,
821 pid: u32,
822 started_at_ms: u64,
823 executable_path: Option<PathBuf>,
824 build_git_sha: Option<String>,
825 build_lock_digest: Option<String>,
826 ) -> Self {
827 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
828 let process_start_time = process_start_time(pid);
829 self.daemon_provenance = DaemonProvenanceFacts {
830 build: DaemonBuildProvenance {
831 build_git_sha,
832 build_lock_digest,
833 },
834 pid: Some(pid),
835 started_at_ms: Some(started_at_ms),
836 start_clock: None,
837 executable_path,
838 executable_identity,
839 process_start_time,
840 probe: ExecutableIdentityProbe::default(),
841 };
842 self
843 }
844
845 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
846 self.daemon_provenance.start_clock = Some(clock);
847 self
848 }
849
850 #[cfg(test)]
851 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
852 self.provenance_probe_override = Some(result);
853 self
854 }
855
856 pub fn with_capability_config(
860 self,
861 modules: impl IntoIterator<Item = (String, bool)>,
862 reserved_capabilities: BTreeMap<String, String>,
863 ) -> Self {
864 self.capability_evaluator
865 .configure(modules, reserved_capabilities);
866 self
867 }
868
869 pub fn with_supervisor_rescan(
870 mut self,
871 supervisor: Supervisor,
872 config_path: impl Into<PathBuf>,
873 configured_port: Option<u16>,
874 ) -> Self {
875 self.rescan = Some(SupervisorRescanContext {
876 supervisor,
877 config_path: config_path.into(),
878 configured_port,
879 storage_config: self.storage_config.clone(),
880 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
881 admission_facts_targets: self.admission_facts_targets.clone(),
882 });
883 self
884 }
885
886 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
887 self.connected_clients = connected_clients;
888 self
889 }
890
891 pub fn forwarding(&self) -> Arc<ForwardingTable> {
892 Arc::clone(&self.forwarding)
893 }
894
895 pub(crate) fn counters(&self) -> DaemonCounters {
896 self.counters.clone()
897 }
898
899 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
902 tokio::spawn(async move {
903 loop {
904 self.capability_evaluator
905 .wait_for_change_or_deadline()
906 .await;
907 self.refresh_capability_requirements();
908 }
909 });
910 }
911
912 fn runtime_capability_snapshot(
913 &self,
914 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
915 let runtime = self
916 .supervisor
917 .list()
918 .into_iter()
919 .map(|module| {
920 let status = module.status().map_err(|err| {
921 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
922 })?;
923 Ok(RuntimeModule {
924 module_id: status.module_id,
925 state: status.state,
926 enabled: status.enabled,
927 })
928 })
929 .collect::<Result<Vec<_>, RouterError>>()?;
930 let (_, registrations) = self.registry.list_modules().map_err(|err| {
931 RouterError::backend(
932 0,
933 0,
934 format!("failed to list capability registrations: {err}"),
935 )
936 })?;
937 let registrations = registrations
938 .into_iter()
939 .map(|registration| RegisteredModule {
940 module_id: registration.manifest.module_id,
941 module_version: registration.manifest.module_version,
942 capabilities: registration.manifest.capabilities,
943 })
944 .collect();
945 Ok((runtime, registrations))
946 }
947
948 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
955 let cached_registration = RegisteredModule {
956 module_id: registration.manifest.module_id.clone(),
957 module_version: registration.manifest.module_version.clone(),
958 capabilities: registration.manifest.capabilities.clone(),
959 };
960 if self.capability_evaluator.record_hello(&cached_registration) {
961 warn!(
962 module_id = %cached_registration.module_id,
963 "capability claims drifted from the cached manifest"
964 );
965 }
966 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
967 self.enforce_capability_denies();
968 }
969 self.refresh_capability_requirements();
970 }
971
972 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
976 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
977 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
978 self.supervisor.set_swap_promotion_observer(observer);
979 }
980
981 pub fn refresh_capability_requirements(&self) {
982 match self.runtime_capability_snapshot() {
983 Ok((runtime, registrations)) => {
984 log_requirement_events(
985 self.capability_evaluator
986 .evaluate_now(&runtime, ®istrations),
987 );
988 }
989 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
990 }
991 }
992
993 fn enforce_capability_denies(&self) {
997 let (_, registrations) = match self.registry.list_modules() {
998 Ok(snapshot) => snapshot,
999 Err(err) => {
1000 warn!(error = %err, "failed to read registrations for capability deny census");
1001 return;
1002 }
1003 };
1004 let manifests = registrations
1005 .into_iter()
1006 .map(|registration| {
1007 (
1008 registration.manifest.module_id.clone(),
1009 registration.manifest,
1010 )
1011 })
1012 .collect::<BTreeMap<_, _>>();
1013 let census = match self.forwarding.route_census(None) {
1014 Ok(census) => census,
1015 Err(err) => {
1016 warn!(error = %err, "failed to read route census for capability deny enforcement");
1017 return;
1018 }
1019 };
1020
1021 for (target_module_id, routes) in census {
1022 let Some(target_manifest) = manifests.get(&target_module_id) else {
1023 continue;
1024 };
1025 let mut closed_routes = Vec::new();
1026 let mut module_goodbyes = Vec::new();
1027 for route in routes {
1028 let Principal::Reserved {
1029 module_id: opening_module_id,
1030 } = &route.principal
1031 else {
1032 continue;
1033 };
1034 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1035 continue;
1036 };
1037 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1038 continue;
1039 };
1040
1041 match self.forwarding.release_client_route(
1042 route.goodbye_target.connection_id,
1043 route.goodbye_target.channel,
1044 route.goodbye_target.epoch,
1045 ) {
1046 Ok(RouteRelease::Removed(module_goodbye)) => {
1047 warn!(
1048 opening_module_id,
1049 target_module_id,
1050 capability,
1051 "force-closing route because an attested capability deny edge now matches"
1052 );
1053 closed_routes.push(route);
1054 module_goodbyes.push(module_goodbye);
1055 }
1056 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1057 Err(err) => warn!(
1058 opening_module_id,
1059 target_module_id,
1060 capability,
1061 error = %err,
1062 "failed to force-close capability-denied route"
1063 ),
1064 }
1065 }
1066
1067 if closed_routes.is_empty() {
1068 continue;
1069 }
1070 send_route_control_pushes(
1071 &self.forwarding,
1072 closed_routes,
1073 ClientControlPush::RouteClosed {
1074 module_id: target_module_id,
1075 reason: RouteCloseReason::CapabilityDenied,
1076 drained: false,
1077 abandoned: 0,
1078 excluded_subscriptions: 0,
1079 terminal: Some(false),
1080 },
1081 );
1082 self.emit_route_goodbyes(module_goodbyes);
1083 }
1084 }
1085
1086 fn not_ready_reason(
1092 &self,
1093 registration: &crate::registry::ModuleRegistration,
1094 ) -> Option<NotReadyReason> {
1095 if !registration.ready {
1096 return Some(NotReadyReason {
1097 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1098 capability: None,
1099 });
1100 }
1101 self.first_unprovided_required_capability(registration)
1102 .map(|capability| NotReadyReason {
1103 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1104 capability: Some(capability),
1105 })
1106 }
1107
1108 fn first_unprovided_required_capability(
1126 &self,
1127 registration: &crate::registry::ModuleRegistration,
1128 ) -> Option<String> {
1129 let required = registration
1130 .manifest
1131 .capabilities
1132 .iter()
1133 .flat_map(|declarations| declarations.requires.iter())
1134 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1135 .map(|requirement| requirement.capability.as_str())
1136 .collect::<BTreeSet<_>>();
1137 if required.is_empty() {
1138 return None;
1139 }
1140 let module_id = registration.manifest.module_id.as_str();
1141 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1142 if required
1143 .iter()
1144 .any(|capability| verdict(capability).is_none())
1145 {
1146 self.refresh_capability_requirements();
1147 }
1148 required
1149 .into_iter()
1150 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1151 .map(str::to_string)
1152 }
1153
1154 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1155 self.capability_evaluator
1156 .statuses()
1157 .into_iter()
1158 .map(capability_requirement_status)
1159 .collect()
1160 }
1161
1162 fn deregister_connection(
1169 &self,
1170 connection_id: ConnectionId,
1171 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1172 self.registry.deregister_connection(connection_id)
1173 }
1174
1175 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1176 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1177 return None;
1178 }
1179 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1180 parse_client_control_request(&frame.body)
1181 else {
1182 return None;
1183 };
1184 Some(target_module_id(&target).to_string())
1185 }
1186
1187 pub(crate) fn route_open_capacity_refusal(
1188 &self,
1189 ctx: &RouteCtx,
1190 frame: &Frame,
1191 target_module_id: &str,
1192 in_flight: usize,
1193 limit: usize,
1194 ) -> Result<Frame, RouterError> {
1195 self.route_open_admission_refusal_frame(
1196 ctx,
1197 frame,
1198 target_module_id,
1199 "open_admission_full",
1200 (in_flight, limit),
1201 format!(
1202 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1203 ),
1204 )
1205 }
1206
1207 fn route_open_target_capacity_refusal(
1208 &self,
1209 ctx: &RouteCtx,
1210 frame: &Frame,
1211 target_module_id: &str,
1212 in_flight: usize,
1213 ) -> Result<Frame, RouterError> {
1214 self.route_open_admission_refusal_frame(
1215 ctx,
1216 frame,
1217 target_module_id,
1218 "target_binds_full",
1219 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1220 format!(
1221 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1222 ),
1223 )
1224 }
1225
1226 fn route_open_admission_refusal_frame(
1234 &self,
1235 ctx: &RouteCtx,
1236 frame: &Frame,
1237 target_module_id: &str,
1238 reason: &'static str,
1239 (in_flight, limit): (usize, usize),
1240 message: impl Into<String>,
1241 ) -> Result<Frame, RouterError> {
1242 let code = error_codes::TARGET_UNAVAILABLE;
1243 self.counters.increment_route_open_refused(code);
1244 info!(
1245 target: "control",
1246 code,
1247 reason,
1248 module_id = ?target_module_id,
1249 connection_id = ctx.connection_id.get(),
1250 in_flight,
1251 limit,
1252 "route.open refused"
1253 );
1254 control_error_frame(frame, code, message.into())
1255 }
1256
1257 #[cfg(test)]
1263 pub fn handle_control(
1264 &self,
1265 connection_id: ConnectionId,
1266 frame: Frame,
1267 ) -> Result<Vec<Frame>, RouterError> {
1268 match frame.header.ty {
1269 FrameType::Ping => Ok(vec![pong(&frame)?]),
1270 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1271 FrameType::Goodbye => self.handle_goodbye(connection_id),
1272 ty => Ok(vec![control_error_frame(
1273 &frame,
1274 "unsupported_control_frame",
1275 format!("unsupported channel-0 frame {ty:?}"),
1276 )?]),
1277 }
1278 }
1279
1280 pub async fn handle_control_frame(
1281 &self,
1282 ctx: &RouteCtx,
1283 frame: Frame,
1284 ) -> Result<Vec<Frame>, RouterError> {
1285 self.handle_control_frame_timed(ctx, frame, None).await
1286 }
1287
1288 pub(crate) async fn handle_control_frame_timed(
1289 &self,
1290 ctx: &RouteCtx,
1291 frame: Frame,
1292 dispatch_started_at: Option<StdInstant>,
1293 ) -> Result<Vec<Frame>, RouterError> {
1294 match frame.header.ty {
1295 FrameType::Ping => Ok(vec![pong(&frame)?]),
1296 FrameType::Hello => {
1297 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1298 }
1299 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1300 FrameType::Cancel => {
1301 if self
1302 .supervisor
1303 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1304 {
1305 Ok(Vec::new())
1306 } else {
1307 Ok(vec![control_error_frame(
1308 &frame,
1309 "unknown_subscription",
1310 "no supervisor spawn subscription has this correlation id",
1311 )?])
1312 }
1313 }
1314 FrameType::Request => {
1315 if self
1316 .forwarding
1317 .module_endpoint_for_connection(ctx.connection_id)
1318 .map_err(RouterError::Forwarding)?
1319 .is_some()
1320 {
1321 if !is_known_module_request_op(&frame.body) {
1322 return Ok(vec![control_error_frame(
1323 &frame,
1324 "unsupported_control_frame",
1325 "module-originated channel-0 REQUEST is not supported",
1326 )?]);
1327 }
1328 let request = match parse_module_control_request_from_module(&frame.body) {
1329 Ok(request) => request,
1330 Err((err, ControlRequestBodyError::UnknownOp)) => {
1331 return Ok(vec![control_error_frame(
1332 &frame,
1333 "unsupported_control_frame",
1334 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1335 )?])
1336 }
1337 Err((err, ControlRequestBodyError::InvalidBody)) => {
1338 return Ok(vec![control_error_frame(
1339 &frame,
1340 "invalid_control_body",
1341 format!("malformed module control body: {err}"),
1342 )?])
1343 }
1344 };
1345 let op = module_control_request_op(&request);
1346 let corr = frame.header.corr;
1347 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1348 let result =
1349 self.handle_module_control_request(ctx.connection_id, frame, request);
1350 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1351 return result;
1352 }
1353
1354 if is_known_module_request_op(&frame.body) {
1355 return Ok(vec![control_error_frame(
1356 &frame,
1357 "not_registered",
1358 "catalog.update requires an active module registration owned by this connection",
1359 )?]);
1360 }
1361
1362 let request = match parse_client_control_request(&frame.body) {
1363 Ok(request) => request,
1364 Err((err, ControlRequestBodyError::UnknownOp)) => {
1365 return Ok(vec![control_error_frame(
1366 &frame,
1367 "unknown_control_op",
1368 format!("unknown client control op: {err}"),
1369 )?])
1370 }
1371 Err((err, ControlRequestBodyError::InvalidBody)) => {
1372 return Ok(vec![control_error_frame(
1373 &frame,
1374 "invalid_control_body",
1375 format!("malformed client control body: {err}"),
1376 )?])
1377 }
1378 };
1379 let op = client_control_request_op(&request);
1380 let corr = frame.header.corr;
1381 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1382 #[cfg(test)]
1383 if let Some(delay) = self.control_dispatch_delay {
1384 tokio::time::sleep(delay).await;
1385 }
1386 let result = self
1387 .handle_client_control_request(ctx, frame, request)
1388 .await;
1389 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1390 result
1391 }
1392 FrameType::Push => {
1393 let Some(endpoint) = self
1394 .forwarding
1395 .module_endpoint_for_connection(ctx.connection_id)
1396 .map_err(RouterError::Forwarding)?
1397 else {
1398 return Ok(vec![control_error_frame(
1399 &frame,
1400 "unsupported_control_frame",
1401 "client-originated channel-0 PUSH is not supported",
1402 )?]);
1403 };
1404 self.handle_status_update(endpoint, frame)
1405 }
1406 FrameType::Response | FrameType::Error
1407 if self
1408 .forwarding
1409 .module_endpoint_for_connection(ctx.connection_id)
1410 .map_err(RouterError::Forwarding)?
1411 .is_some() =>
1412 {
1413 self.handle_module_relay_response(ctx.connection_id, frame)
1414 }
1415 ty => Ok(vec![control_error_frame(
1416 &frame,
1417 "unsupported_control_frame",
1418 format!("unsupported channel-0 frame {ty:?}"),
1419 )?]),
1420 }
1421 }
1422
1423 pub fn cleanup_connection(
1424 &self,
1425 connection_id: ConnectionId,
1426 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1427 let crash_closed = self
1428 .registry
1429 .get_module_by_connection(connection_id)?
1430 .and_then(|registration| {
1431 self.forwarding
1432 .module_endpoint_for_connection(connection_id)
1433 .ok()
1434 .flatten()
1435 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1436 .map(|routes| (registration.manifest.module_id, routes))
1437 });
1438 let crash_closed = crash_closed.map(|(module_id, routes)| {
1439 let terminal = match self.supervisor.get(&module_id) {
1440 None => false,
1441 Some(module) => match module.will_recover_after_connection_loss() {
1442 Ok(will_recover) => !will_recover,
1443 Err(err) => {
1444 warn!(
1445 %module_id,
1446 error = %err,
1447 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1448 );
1449 false
1450 }
1451 },
1452 };
1453 let reason = match self.forwarding.is_daemon_draining() {
1457 Ok(true) => RouteCloseReason::Restart,
1458 Ok(false) => RouteCloseReason::Crash,
1459 Err(err) => {
1460 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1461 RouteCloseReason::Crash
1462 }
1463 };
1464 (module_id, routes, reason, terminal)
1465 });
1466 let registrations = self.deregister_connection(connection_id);
1467 let cleanup = self.forwarding.cleanup_connection_counted(connection_id);
1468 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1473 let abandoned = cleanup
1474 .as_ref()
1475 .map_or(0, |cleanup| cleanup.abandoned_relays);
1476 send_route_control_pushes(
1477 &self.forwarding,
1478 routes,
1479 ClientControlPush::RouteClosed {
1480 module_id,
1481 reason,
1482 drained: false,
1483 abandoned,
1484 excluded_subscriptions: 0,
1485 terminal: Some(terminal),
1486 },
1487 );
1488 }
1489 if let Ok(cleanup) = cleanup {
1490 self.emit_route_goodbyes(cleanup.released);
1491 }
1492 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1496 crate::supervise::notify_registration_release();
1497 self.capability_evaluator.wake_deadline_loop();
1498 self.refresh_capability_requirements();
1499 }
1500 self.supervisor.remove_spawn_subscribers(connection_id);
1501 registrations
1502 }
1503
1504 pub(crate) fn handle_route_goodbye(
1505 &self,
1506 connection_id: ConnectionId,
1507 route_channel: u16,
1508 route_epoch: u32,
1509 ) -> Result<bool, RouterError> {
1510 debug!(
1511 connection_id = connection_id.get(),
1512 route_channel, route_epoch, "handling route GOODBYE"
1513 );
1514 let RouteRelease::Removed(released_route) = self
1515 .forwarding
1516 .release_client_route(connection_id, route_channel, route_epoch)
1517 .map_err(RouterError::Forwarding)?
1518 else {
1519 return Ok(false);
1520 };
1521 self.emit_route_goodbyes(vec![released_route]);
1522 Ok(true)
1523 }
1524
1525 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1526 for released in released_routes {
1527 let frame = match Frame::build_with_version(
1528 released.negotiated_ver,
1529 FrameType::Goodbye,
1530 control_flags(),
1531 released.channel,
1532 released.epoch,
1533 0,
1534 Vec::new(),
1535 ) {
1536 Ok(frame) => frame,
1537 Err(err) => {
1538 warn!(
1539 route_channel = released.channel,
1540 error = %err,
1541 "failed to build route GOODBYE frame"
1542 );
1543 continue;
1544 }
1545 };
1546 if !released.close_on_delivery_failure() {
1547 crate::forwarding::send_module_route_goodbye(
1548 &self.counters,
1549 &released.sink,
1550 frame,
1551 released.module_id.as_deref(),
1552 "client route released",
1553 );
1554 continue;
1555 }
1556 if let Err(err) = released.sink.try_send(frame) {
1557 warn!(
1558 target_connection_id = released.connection_id.get(),
1559 route_channel = released.channel,
1560 error = %err,
1561 "route GOODBYE was not delivered to client; closing target connection"
1562 );
1563 if self
1564 .forwarding
1565 .escalate_client_delivery_failure(
1566 released.connection_id,
1567 released.channel,
1568 released.epoch,
1569 CloseReason::new(
1570 "route_goodbye_delivery_failed",
1571 format!(
1572 "failed to enqueue route GOODBYE for channel {}: {err}",
1573 released.channel
1574 ),
1575 ),
1576 crate::forwarding::UndeliveredFrame {
1577 module_id: released.module_id.as_deref(),
1578 sink: &released.sink,
1579 },
1580 )
1581 .unwrap_or(false)
1582 {
1583 self.counters.increment_goodbye_relay_client_failed();
1584 }
1585 }
1586 }
1587 }
1588
1589 fn send_abandoned_route_bind_goodbye(
1601 &self,
1602 module_sink: &crate::FrameSink,
1603 negotiated_ver: u8,
1604 module_channel: u16,
1605 module_epoch: u32,
1606 ) {
1607 let frame = match Frame::build_with_version(
1608 negotiated_ver,
1609 FrameType::Goodbye,
1610 control_flags(),
1611 module_channel,
1612 module_epoch,
1613 0,
1614 Vec::new(),
1615 ) {
1616 Ok(frame) => frame,
1617 Err(err) => {
1618 warn!(
1619 route_channel = module_channel,
1620 error = %err,
1621 "failed to build GOODBYE for abandoned route.bind"
1622 );
1623 return;
1624 }
1625 };
1626 crate::forwarding::send_module_route_goodbye(
1627 &self.counters,
1628 module_sink,
1629 frame,
1630 None,
1631 "abandoned route.bind",
1632 );
1633 }
1634
1635 fn handle_hello(
1636 &self,
1637 connection_id: ConnectionId,
1638 sink: Option<crate::FrameSink>,
1639 frame: Frame,
1640 ) -> Result<Vec<Frame>, RouterError> {
1641 debug!(
1642 connection_id = connection_id.get(),
1643 corr = frame.header.corr,
1644 "handling HELLO"
1645 );
1646 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1647 Ok(value) => value,
1648 Err(err) => {
1649 return Ok(vec![control_error_frame(
1650 &frame,
1651 "invalid_hello",
1652 format!("malformed HELLO body: {err}"),
1653 )?])
1654 }
1655 };
1656 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1657 return Ok(vec![control_error_frame(
1658 &frame,
1659 "invalid_capability_grammar",
1660 err.to_string(),
1661 )?]);
1662 }
1663 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1664 return Ok(vec![control_error_frame(
1665 &frame,
1666 "invalid_manifest",
1667 err.to_string(),
1668 )?]);
1669 }
1670 if let Some(provenance) = hello_value
1671 .get("manifest")
1672 .and_then(|manifest| manifest.get("provenance"))
1673 {
1674 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1675 return Ok(vec![control_error_frame(
1676 &frame,
1677 "invalid_manifest",
1678 format!("malformed manifest provenance: {err}"),
1679 )?]);
1680 }
1681 }
1682 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1683 Ok(hello) => hello,
1684 Err(err) => {
1685 return Ok(vec![control_error_frame(
1686 &frame,
1687 "invalid_hello",
1688 format!("malformed HELLO body: {err}"),
1689 )?])
1690 }
1691 };
1692
1693 if hello.protocol_ver != hello.manifest.protocol_ver {
1694 return Ok(vec![control_error_frame(
1695 &frame,
1696 "invalid_manifest",
1697 format!(
1698 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1699 hello.protocol_ver, hello.manifest.protocol_ver
1700 ),
1701 )?]);
1702 }
1703
1704 if hello.manifest.module_id.trim().is_empty() {
1705 return Ok(vec![control_error_frame(
1706 &frame,
1707 "invalid_manifest",
1708 "manifest module_id must not be empty",
1709 )?]);
1710 }
1711
1712 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1713 Ok(negotiated_ver) => negotiated_ver,
1714 Err(message) => {
1715 return Ok(vec![control_error_frame(
1716 &frame,
1717 "version_unsupported",
1718 message,
1719 )?])
1720 }
1721 };
1722
1723 let swap_admission = self
1731 .supervisor
1732 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1733 if swap_admission == SwapHelloAdmission::Refused {
1734 warn!(
1735 module_id = %hello.manifest.module_id,
1736 connection_id = connection_id.get(),
1737 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1738 );
1739 return Ok(vec![control_error_frame(
1740 &frame,
1741 "swap_token_invalid",
1742 format!(
1743 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1744 hello.manifest.module_id
1745 ),
1746 )?]);
1747 }
1748 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1749
1750 if let Some(rejection) = (!swap_candidate)
1758 .then(|| {
1759 self.supervisor.reserved_hello_rejection(
1760 &hello.manifest.module_id,
1761 hello.launch_nonce.as_deref(),
1762 )
1763 })
1764 .flatten()
1765 {
1766 let message = match rejection {
1767 ReservedHelloRejection::Exact { module_id } => format!(
1768 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1769 ),
1770 ReservedHelloRejection::Prefix {
1771 prefix,
1772 owner_module_id,
1773 } => format!(
1774 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1775 hello.manifest.module_id
1776 ),
1777 };
1778 return Ok(vec![control_error_frame(
1779 &frame,
1780 "reserved_module",
1781 message,
1782 )?]);
1783 }
1784
1785 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1786 &hello.manifest.module_id,
1787 hello.manifest.capabilities.as_ref(),
1788 );
1789 if let Some(refusal) = reserved_capability_refusals.first() {
1790 let capability = refusal.capability.clone();
1791 let bound_module = refusal.claimants[0].clone();
1792 log_duplicate_claim_events(reserved_capability_refusals);
1793 return Ok(vec![control_error_frame(
1794 &frame,
1795 "reserved_capability",
1796 format!(
1797 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1798 capability, bound_module, hello.manifest.module_id
1799 ),
1800 )?]);
1801 }
1802
1803 if self
1806 .forwarding
1807 .connection_has_client_routes(connection_id)
1808 .map_err(RouterError::Forwarding)?
1809 {
1810 return Ok(vec![control_error_frame(
1811 &frame,
1812 "invalid_hello",
1813 "connection has open client routes and cannot also register as a module",
1814 )?]);
1815 }
1816
1817 let control_ops = effective_module_control_ops(hello.control_ops);
1818 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1821 if swap_candidate {
1822 return self.register_swap_candidate(
1823 connection_id,
1824 sink,
1825 &frame,
1826 hello.manifest,
1827 negotiated_ver,
1828 control_ops,
1829 hello_ack,
1830 );
1831 }
1832 let registration = match self.registry.register_with_control_ops(
1833 hello.manifest,
1834 negotiated_ver,
1835 connection_id,
1836 control_ops,
1837 ) {
1838 Ok(registration) => registration,
1839 Err(RegistryError::DuplicateModuleId { module_id }) => {
1840 return Ok(vec![control_error_frame(
1841 &frame,
1842 "duplicate_module_id",
1843 format!(
1844 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1845 ),
1846 )?])
1847 }
1848 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1849 return Ok(vec![control_error_frame(
1850 &frame,
1851 "invalid_module_id",
1852 err.to_string(),
1853 )?])
1854 }
1855 Err(err) => {
1856 return Ok(vec![control_error_frame(
1857 &frame,
1858 "registry_error",
1859 err.to_string(),
1860 )?])
1861 }
1862 };
1863
1864 let reply = if let Some(sink) = sink {
1865 let concurrency = manifest_concurrency(®istration.manifest);
1881 if let Err(err) = self.forwarding.register_module_connection_acked(
1882 connection_id,
1883 registration.manifest.module_id.clone(),
1884 negotiated_ver,
1885 concurrency,
1886 sink,
1887 hello_ack,
1888 ) {
1889 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1893 crate::supervise::notify_registration_release();
1894 }
1895 return Ok(vec![control_error_frame(
1896 &frame,
1897 forwarding_error_code(&err),
1898 err.to_string(),
1899 )?]);
1900 }
1901 Vec::new()
1902 } else {
1903 vec![hello_ack]
1906 };
1907
1908 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
1917 info!(
1918 module_id = %registration.manifest.module_id,
1919 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
1920 );
1921 }
1922
1923 self.apply_registration_capabilities(®istration);
1924
1925 info!(
1926 module_id = %registration.manifest.module_id,
1927 module_version = %registration.manifest.module_version,
1928 negotiated_ver,
1929 routable_provider = manifest_provides_routable_role(®istration.manifest),
1930 connection_id = connection_id.get(),
1931 "module registered"
1932 );
1933
1934 Ok(reply)
1935 }
1936
1937 #[allow(clippy::too_many_arguments)]
1947 fn register_swap_candidate(
1948 &self,
1949 connection_id: ConnectionId,
1950 sink: Option<crate::FrameSink>,
1951 frame: &Frame,
1952 manifest: ModuleManifest,
1953 negotiated_ver: u8,
1954 control_ops: Vec<String>,
1955 hello_ack: Frame,
1956 ) -> Result<Vec<Frame>, RouterError> {
1957 let module_id = manifest.module_id.clone();
1958 let registration = match self.registry.register_candidate_with_control_ops(
1959 manifest,
1960 negotiated_ver,
1961 connection_id,
1962 control_ops,
1963 ) {
1964 Ok(registration) => registration,
1965 Err(RegistryError::DuplicateModuleId { module_id }) => {
1966 return Ok(vec![control_error_frame(
1967 frame,
1968 "duplicate_module_id",
1969 format!(
1970 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
1971 ),
1972 )?])
1973 }
1974 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1975 return Ok(vec![control_error_frame(
1976 frame,
1977 "invalid_module_id",
1978 err.to_string(),
1979 )?])
1980 }
1981 Err(err) => {
1982 return Ok(vec![control_error_frame(
1983 frame,
1984 "registry_error",
1985 err.to_string(),
1986 )?])
1987 }
1988 };
1989 let reply = if let Some(sink) = sink {
1990 let concurrency = manifest_concurrency(®istration.manifest);
1994 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
1995 connection_id,
1996 module_id.clone(),
1997 negotiated_ver,
1998 concurrency,
1999 sink,
2000 hello_ack,
2001 ) {
2002 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2003 crate::supervise::notify_registration_release();
2004 }
2005 return Ok(vec![control_error_frame(
2006 frame,
2007 forwarding_error_code(&err),
2008 err.to_string(),
2009 )?]);
2010 }
2011 Vec::new()
2012 } else {
2013 vec![hello_ack]
2014 };
2015 self.supervisor.mark_swap_candidate_admitted(&module_id);
2016 info!(
2017 module_id = %module_id,
2018 module_version = %registration.manifest.module_version,
2019 negotiated_ver,
2020 ready = registration.ready,
2021 connection_id = connection_id.get(),
2022 "swap candidate registered; not routable until cutover"
2023 );
2024 Ok(reply)
2025 }
2026
2027 fn build_hello_ack(
2028 &self,
2029 frame: &Frame,
2030 negotiated_ver: u8,
2031 module_id: &str,
2032 ) -> Result<Frame, RouterError> {
2033 let ack = ModuleHelloAckBody {
2034 negotiated_ver,
2035 subc_ops: module_subc_ops(),
2036 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2037 storage: self
2038 .storage_config
2039 .as_ref()
2040 .map(|cfg| cfg.descriptor_for(module_id)),
2041 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2042 };
2043 let body = serde_json::to_vec(&ack).map_err(|err| {
2044 RouterError::backend(
2045 0,
2046 frame.header.corr,
2047 format!("failed to encode HELLO_ACK: {err}"),
2048 )
2049 })?;
2050
2051 Frame::build_with_version(
2052 negotiated_ver,
2053 FrameType::HelloAck,
2054 control_flags(),
2055 0,
2056 0,
2057 frame.header.corr,
2058 body,
2059 )
2060 .map_err(RouterError::FrameBuild)
2061 }
2062
2063 async fn handle_client_control_request(
2064 &self,
2065 ctx: &RouteCtx,
2066 frame: Frame,
2067 request: ClientControlRequest,
2068 ) -> Result<Vec<Frame>, RouterError> {
2069 match request {
2070 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2071 ClientControlRequest::CatalogList { module_id } => {
2072 self.handle_catalog_list(frame, module_id)
2073 }
2074 ClientControlRequest::RouteOpen {
2075 target,
2076 identity,
2077 consumer_identity,
2078 consumer_capabilities,
2079 admission_facts,
2080 } => {
2081 self.handle_route_open(
2082 ctx,
2083 frame,
2084 RouteOpenRequest {
2085 target,
2086 identity,
2087 consumer_identity,
2088 consumer_capabilities,
2089 admission_facts,
2090 },
2091 )
2092 .await
2093 }
2094 ClientControlRequest::RoutePoll {
2095 route_channel,
2096 route_epoch,
2097 kind,
2098 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2099 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2100 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2101 self.handle_supervisor_spawn_snapshot(frame)
2102 }
2103 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2104 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2105 }
2106 ClientControlRequest::SupervisorRestart {
2107 module_id,
2108 drain_timeout_ms,
2109 } => {
2110 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2111 .await
2112 }
2113 ClientControlRequest::SupervisorSwap {
2114 module_id,
2115 ready_timeout_ms,
2116 } => {
2117 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2118 .await
2119 }
2120 ClientControlRequest::SupervisorReload { module_id } => {
2121 self.handle_supervisor_reload(frame, module_id).await
2122 }
2123 ClientControlRequest::SupervisorRescan { preview } => {
2124 self.handle_supervisor_rescan(frame, preview).await
2125 }
2126 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2127 self.handle_supervisor_release_reserved(frame, module_id)
2128 .await
2129 }
2130 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2131 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2132 .await
2133 }
2134 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2135 self.handle_supervisor_health_probe(frame, module_id).await
2136 }
2137 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2138 ClientControlRequest::SupervisorRoutes { module_id } => {
2139 self.handle_supervisor_routes(frame, module_id)
2140 }
2141 ClientControlRequest::SupervisorProvenance { module_id } => {
2142 self.handle_supervisor_provenance(frame, module_id).await
2143 }
2144 ClientControlRequest::SupervisorStderrTail {
2145 module_id,
2146 max_lines,
2147 max_bytes,
2148 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2149 ClientControlRequest::SupervisorTerminals { module_id } => {
2150 self.handle_supervisor_terminals(frame, module_id).await
2151 }
2152 }
2153 }
2154
2155 fn handle_module_control_request(
2156 &self,
2157 connection_id: ConnectionId,
2158 frame: Frame,
2159 request: ModuleControlRequestFromModule,
2160 ) -> Result<Vec<Frame>, RouterError> {
2161 match request {
2162 ModuleControlRequestFromModule::CatalogUpdate {
2163 provides,
2164 capabilities,
2165 ready,
2166 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2167 ModuleControlRequestFromModule::LiveRoots {} => {
2168 let registered = self
2169 .registry
2170 .get_module_by_connection(connection_id)
2171 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2172 let Some(registration) = registered else {
2173 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2174 };
2175 let response = self
2176 .forwarding
2177 .live_roots(®istration.manifest.module_id)
2178 .map_err(RouterError::Forwarding)?;
2179 Ok(vec![control_response_body_frame(
2180 &frame,
2181 &response,
2182 "ModuleControlResponseToModule::LiveRoots",
2183 )?])
2184 }
2185 }
2186 }
2187
2188 fn handle_catalog_update(
2189 &self,
2190 connection_id: ConnectionId,
2191 frame: Frame,
2192 provides: Vec<ProviderRole>,
2193 capabilities: Option<CapabilityDeclarations>,
2194 ready: Option<bool>,
2195 ) -> Result<Vec<Frame>, RouterError> {
2196 self.refresh_capability_requirements();
2197 let Some(registration) = self
2198 .registry
2199 .get_module_by_connection(connection_id)
2200 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2201 else {
2202 return Ok(vec![control_error_frame(
2203 &frame,
2204 "not_registered",
2205 "catalog.update requires an active module registration owned by this connection",
2206 )?]);
2207 };
2208
2209 if let Some(message) =
2210 catalog_update_frozen_field_message(®istration.manifest, &provides)
2211 {
2212 return Ok(vec![control_error_frame(
2213 &frame,
2214 "catalog_update_frozen_field",
2215 message,
2216 )?]);
2217 }
2218
2219 let mut candidate = registration.manifest.clone();
2220 candidate.provides = provides.clone();
2221 candidate.capabilities = capabilities
2222 .clone()
2223 .or_else(|| registration.manifest.capabilities.clone());
2224 if let Err(err) = candidate.validate_capability_grammar() {
2225 return Ok(vec![control_error_frame(
2226 &frame,
2227 "invalid_capability_grammar",
2228 err.to_string(),
2229 )?]);
2230 }
2231
2232 let updated = self
2233 .registry
2234 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2235 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2236 if updated.is_none() {
2237 return Ok(vec![control_error_frame(
2238 &frame,
2239 "not_registered",
2240 "catalog.update requires an active module registration owned by this connection",
2241 )?]);
2242 }
2243 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2244 log_duplicate_claim_events(
2245 self.capability_evaluator
2246 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2247 );
2248 }
2249 if capability_census_trigger(
2250 registration.manifest.capabilities.as_ref(),
2251 updated
2252 .as_ref()
2253 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2254 ) {
2255 self.enforce_capability_denies();
2256 }
2257 self.refresh_capability_requirements();
2258
2259 let response = ModuleControlResponseToModule::CatalogUpdate {};
2260 control_response_body_frame(
2261 &frame,
2262 &response,
2263 "ModuleControlResponseToModule::CatalogUpdate",
2264 )
2265 .map(|frame| vec![frame])
2266 }
2267
2268 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2269 self.refresh_capability_requirements();
2270 let mut counters = self.counters.snapshot();
2276 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2277 self.forwarding.client_route_concentration(),
2278 counters.as_object_mut(),
2279 ) {
2280 obj.insert(
2281 "client_connections_with_routes".into(),
2282 connections_with_routes.into(),
2283 );
2284 obj.insert("max_routes_on_one_connection".into(), max.into());
2285 }
2286 if let (Some(open_breakers), Some(obj)) = (
2292 self.route_bind_breakers.open_snapshot(),
2293 counters.as_object_mut(),
2294 ) {
2295 obj.insert("route_bind_breakers_open".into(), open_breakers);
2296 }
2297 let response = ClientControlResponse::ServerDescribe {
2298 protocol_ver: PROTOCOL_VERSION,
2299 subc_ops: subc_ops(),
2300 capabilities: self.subc_capabilities.as_ref().to_vec(),
2301 connected_clients: self.connected_clients.count(),
2302 counters: Some(counters),
2303 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2304 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2305 capability_requirements: self.capability_requirement_statuses(),
2306 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2307 };
2308 Ok(vec![control_response_body_frame(
2309 &frame,
2310 &response,
2311 "ClientControlResponse::ServerDescribe",
2312 )?])
2313 }
2314
2315 fn handle_catalog_list(
2316 &self,
2317 frame: Frame,
2318 module_id: Option<String>,
2319 ) -> Result<Vec<Frame>, RouterError> {
2320 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2321 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2322 })?;
2323 let entries = modules
2324 .into_iter()
2325 .filter(|registration| {
2326 module_id
2327 .as_deref()
2328 .map(|wanted| registration.manifest.module_id == wanted)
2329 .unwrap_or(true)
2330 })
2331 .map(|registration| {
2332 let not_ready = self.not_ready_reason(®istration);
2333 let roles = registration.manifest.provides;
2334 CatalogEntry {
2335 module_id: registration.manifest.module_id,
2336 ready: not_ready.is_none(),
2337 not_ready,
2338 module_version: Some(registration.manifest.module_version),
2339 roles,
2340 control_ops: registration.control_ops,
2341 capabilities: registration.manifest.capabilities,
2342 self_signals: registration.manifest.self_signals,
2343 }
2344 })
2345 .collect();
2346 let response = ClientControlResponse::CatalogList {
2347 generation,
2348 modules: entries,
2349 subc_ops: subc_ops(),
2350 };
2351 Ok(vec![control_response_body_frame(
2352 &frame,
2353 &response,
2354 "ClientControlResponse::CatalogList",
2355 )?])
2356 }
2357
2358 fn route_open_principal(
2359 &self,
2360 frame: &Frame,
2361 consumer_identity: Option<ConsumerIdentity>,
2362 ) -> Result<Result<Principal, Frame>, RouterError> {
2363 let Some(consumer_identity) = consumer_identity else {
2364 return Ok(Ok(Principal::Direct));
2365 };
2366
2367 if self.supervisor.spawned_consumer_authorized(
2368 &consumer_identity.module_id,
2369 &consumer_identity.launch_nonce,
2370 ) {
2371 return Ok(Ok(Principal::Reserved {
2372 module_id: consumer_identity.module_id,
2373 }));
2374 }
2375
2376 Ok(Err(control_error_frame(
2377 frame,
2378 "bad_consumer_identity",
2379 format!(
2380 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2381 consumer_identity.module_id
2382 ),
2383 )?))
2384 }
2385
2386 fn route_open_refusal_frame(
2393 &self,
2394 ctx: &RouteCtx,
2395 frame: &Frame,
2396 module_id: &str,
2397 reason: &'static str,
2398 code: &'static str,
2399 message: impl Into<String>,
2400 ) -> Result<Frame, RouterError> {
2401 self.observe_route_open_refusal(ctx, module_id, reason, code);
2402 control_error_frame(frame, code, message.into())
2403 }
2404
2405 fn route_open_breaker_refusal_frame(
2423 &self,
2424 ctx: &RouteCtx,
2425 frame: &Frame,
2426 module_id: &str,
2427 consecutive_timeouts: u32,
2428 retry_in: Duration,
2429 probe_in_flight: bool,
2430 ) -> Result<Frame, RouterError> {
2431 self.counters
2432 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2433 debug!(
2434 target: "control",
2435 code = "module_timeout",
2436 module_id = ?module_id,
2437 connection_id = ctx.connection_id.get(),
2438 consecutive_timeouts,
2439 retry_in_ms = retry_in.as_millis() as u64,
2440 probe_in_flight,
2441 "route.open refused by open bind-relay breaker"
2442 );
2443 let detail = if probe_in_flight {
2444 "one probe bind is already in flight; retry once it settles".to_string()
2445 } else {
2446 format!("not relaying for another {retry_in:?}")
2447 };
2448 control_error_frame(
2449 frame,
2450 "module_timeout",
2451 format!(
2452 "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2453 relays; {detail}"
2454 ),
2455 )
2456 }
2457
2458 fn observe_route_open_refusal(
2471 &self,
2472 ctx: &RouteCtx,
2473 module_id: &str,
2474 reason: &'static str,
2475 code: &'static str,
2476 ) {
2477 self.counters.increment_route_open_refused(code);
2478 info!(
2479 target: "control",
2480 code,
2481 reason,
2482 module_id = ?module_id,
2483 connection_id = ctx.connection_id.get(),
2484 "route.open refused"
2485 );
2486 }
2487
2488 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2533 self.counters.increment_route_open_accepted(principal);
2534 info!(
2535 target: "control",
2536 principal,
2537 module_id,
2538 connection_id = ctx.connection_id.get(),
2539 "route.open accepted"
2540 );
2541 }
2542
2543 fn supervised_absent_route_open_refusal_frame(
2544 &self,
2545 ctx: &RouteCtx,
2546 frame: &Frame,
2547 module_id: &str,
2548 code: &'static str,
2549 status: &crate::supervise::ModuleStatus,
2550 ) -> Result<Frame, RouterError> {
2551 self.counters.increment_route_open_refused(code);
2552 info!(
2553 target: "control",
2554 code,
2555 reason = "supervised_not_registered",
2556 module_id = ?module_id,
2557 connection_id = ctx.connection_id.get(),
2558 state = %status.state,
2559 enabled = status.enabled,
2560 live = status.live,
2561 "route.open refused"
2562 );
2563 control_error_frame(
2564 frame,
2565 code,
2566 format!(
2567 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2568 status.state, status.enabled, status.live
2569 ),
2570 )
2571 }
2572
2573 async fn handle_route_open(
2574 &self,
2575 ctx: &RouteCtx,
2576 frame: Frame,
2577 request: RouteOpenRequest,
2578 ) -> Result<Vec<Frame>, RouterError> {
2579 let RouteOpenRequest {
2580 target,
2581 mut identity,
2582 consumer_identity,
2583 consumer_capabilities,
2584 admission_facts,
2585 } = request;
2586 let target_module_id = target_module_id(&target).to_string();
2587 debug!(
2588 connection_id = ctx.connection_id.get(),
2589 corr = frame.header.corr,
2590 module_id = %target_module_id,
2591 "handling route.open"
2592 );
2593
2594 let Some(registration) = self
2613 .registry
2614 .get_module(&target_module_id)
2615 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2616 else {
2617 if let Some((status, warming)) =
2618 self.supervisor_status(&target_module_id, frame.header.corr)?
2619 {
2620 if status.protocol == ModuleProtocol::None {
2627 return Ok(vec![self.route_open_refusal_frame(
2628 ctx,
2629 &frame,
2630 &target_module_id,
2631 "protocol_none",
2632 error_codes::MODULE_NO_PROTOCOL,
2633 format!(
2634 "module_id '{target_module_id}' is declared protocol: none; \
2635 it speaks no subc wire and serves no routes"
2636 ),
2637 )?]);
2638 }
2639 let code = if warming {
2640 "module_warming"
2641 } else {
2642 "target_unavailable"
2643 };
2644 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2645 ctx,
2646 &frame,
2647 &target_module_id,
2648 code,
2649 &status,
2650 )?]);
2651 }
2652 if let Some(removed_ago_ms) =
2653 self.supervisor.removal_tombstone_age_ms(&target_module_id)
2654 {
2655 return Ok(vec![self.route_open_refusal_frame(
2656 ctx,
2657 &frame,
2658 &target_module_id,
2659 "removed",
2660 error_codes::MODULE_REMOVED,
2661 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
2662 )?]);
2663 }
2664 return Ok(vec![self.route_open_refusal_frame(
2665 ctx,
2666 &frame,
2667 &target_module_id,
2668 "not_registered",
2669 error_codes::UNKNOWN_MODULE,
2670 format!("module_id '{target_module_id}' is not registered"),
2671 )?]);
2672 };
2673
2674 if !registration.ready {
2678 self.counters
2679 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
2680 info!(
2681 target: "control",
2682 code = error_codes::MODULE_WARMING,
2683 module_id = ?target_module_id,
2684 connection_id = ctx.connection_id.get(),
2685 reason = "declared_not_ready",
2686 "route.open refused"
2687 );
2688 return Ok(vec![control_error_body_frame(
2689 &frame,
2690 ErrorBody {
2691 code: error_codes::MODULE_WARMING.to_string(),
2692 message: format!(
2693 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
2694 ),
2695 detail: Some(serde_json::json!({
2696 "reason": "declared_not_ready"
2697 })),
2698 },
2699 )?]);
2700 }
2701
2702 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
2721 self.counters
2722 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
2723 info!(
2724 target: "control",
2725 code = error_codes::MODULE_WARMING,
2726 module_id = ?target_module_id,
2727 connection_id = ctx.connection_id.get(),
2728 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2729 capability = %capability,
2730 "route.open refused"
2731 );
2732 return Ok(vec![control_error_body_frame(
2733 &frame,
2734 ErrorBody {
2735 code: error_codes::MODULE_WARMING.to_string(),
2736 message: format!(
2737 "module_id '{target_module_id}' requires capability '{capability}', \
2738 which no registered module provides; retry"
2739 ),
2740 detail: Some(serde_json::json!({
2741 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2742 "capability": capability,
2743 })),
2744 },
2745 )?]);
2746 }
2747
2748 if !target_has_required_role(&target, ®istration.manifest.provides) {
2749 return Ok(vec![self.route_open_refusal_frame(
2750 ctx,
2751 &frame,
2752 &target_module_id,
2753 "role_not_provided",
2754 "target_unavailable",
2755 format!("module_id '{target_module_id}' does not provide the requested target"),
2756 )?]);
2757 }
2758
2759 if registration.state != ChannelState::Active {
2760 return Ok(vec![self.route_open_refusal_frame(
2761 ctx,
2762 &frame,
2763 &target_module_id,
2764 "registration_not_active",
2765 "target_unavailable",
2766 format!("module_id '{target_module_id}' is not active"),
2767 )?]);
2768 }
2769
2770 if self
2771 .forwarding
2772 .module_is_draining(&target_module_id)
2773 .map_err(RouterError::Forwarding)?
2774 {
2775 return Ok(vec![self.route_open_refusal_frame(
2776 ctx,
2777 &frame,
2778 &target_module_id,
2779 "reloading",
2780 "module_reloading",
2781 format!("module_id '{target_module_id}' is reloading"),
2782 )?]);
2783 }
2784
2785 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
2786 process_liveness.process_live(&target_module_id) == Some(false)
2787 }) {
2788 if process_liveness.process_replacing(&target_module_id) {
2795 return Ok(vec![self.route_open_refusal_frame(
2796 ctx,
2797 &frame,
2798 &target_module_id,
2799 "reloading",
2800 "module_reloading",
2801 format!("module_id '{target_module_id}' is reloading"),
2802 )?]);
2803 }
2804 return Ok(vec![self.route_open_refusal_frame(
2805 ctx,
2806 &frame,
2807 &target_module_id,
2808 "supervisor_not_live",
2809 "target_unavailable",
2810 format!("module_id '{target_module_id}' is not live"),
2811 )?]);
2812 }
2813
2814 if !self
2815 .forwarding
2816 .has_live_module_connection(&target_module_id)
2817 .map_err(RouterError::Forwarding)?
2818 {
2819 return Ok(vec![self.route_open_refusal_frame(
2820 ctx,
2821 &frame,
2822 &target_module_id,
2823 "no_forwarding_connection",
2824 "target_unavailable",
2825 format!("module_id '{target_module_id}' has no live forwarding connection"),
2826 )?]);
2827 }
2828
2829 if let Some(error) =
2830 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
2831 {
2832 self.observe_route_open_refusal(
2833 ctx,
2834 &target_module_id,
2835 "op_not_allowed",
2836 "op_not_allowed",
2837 );
2838 return Ok(vec![error]);
2839 }
2840
2841 let principal = match self.route_open_principal(&frame, consumer_identity)? {
2842 Ok(principal) => principal,
2843 Err(error) => {
2844 self.observe_route_open_refusal(
2845 ctx,
2846 &target_module_id,
2847 "bad_consumer_identity",
2848 "bad_consumer_identity",
2849 );
2850 return Ok(vec![error]);
2851 }
2852 };
2853
2854 if let Principal::Reserved {
2858 module_id: opening_module_id,
2859 } = &principal
2860 {
2861 if let Some(opening_registration) = self
2862 .registry
2863 .get_module(opening_module_id)
2864 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2865 {
2866 if let Some(capability) =
2867 denied_capability(&opening_registration.manifest, ®istration.manifest)
2868 {
2869 warn!(
2870 opening_module_id,
2871 target_module_id,
2872 capability,
2873 "refusing route.open because an attested capability deny edge matches"
2874 );
2875 return Ok(vec![self.route_open_refusal_frame(
2876 ctx,
2877 &frame,
2878 &target_module_id,
2879 "capability_deny_edge",
2880 "capability_forbidden",
2881 format!(
2882 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
2883 ),
2884 )?]);
2885 }
2886 }
2887 }
2888
2889 if admission_facts.is_some() {
2890 let carrier_matches = matches!(
2891 &principal,
2892 Principal::Reserved { module_id }
2893 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
2894 );
2895 if !carrier_matches {
2896 return Ok(vec![self.route_open_refusal_frame(
2897 ctx,
2898 &frame,
2899 &target_module_id,
2900 "admission_facts_carrier_not_permitted",
2901 "admission_facts_not_permitted",
2902 "admission facts may only be carried by the configured reserved module",
2903 )?]);
2904 }
2905
2906 let target_allowed = self
2907 .admission_facts_targets
2908 .as_ref()
2909 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
2910 if !target_allowed {
2911 return Ok(vec![self.route_open_refusal_frame(
2912 ctx,
2913 &frame,
2914 &target_module_id,
2915 "admission_facts_target_not_listed",
2916 "admission_facts_target_not_allowed",
2917 format!(
2918 "admission facts are not permitted for target module_id '{target_module_id}'"
2919 ),
2920 )?]);
2921 }
2922
2923 }
2927
2928 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
2950 Ok(project_root) => project_root,
2951 Err(err) => {
2952 return Ok(vec![control_error_frame(
2953 &frame,
2954 "invalid_project_root",
2955 err.to_string(),
2956 )?])
2957 }
2958 };
2959 identity.project_root = project_root.as_path().to_path_buf();
2960
2961 let _concurrency_guard = match self
2972 .route_bind_concurrency
2973 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
2974 {
2975 Ok(guard) => guard,
2976 Err(in_flight) => {
2977 return Ok(vec![self.route_open_target_capacity_refusal(
2978 ctx,
2979 &frame,
2980 &target_module_id,
2981 in_flight,
2982 )?]);
2983 }
2984 };
2985
2986 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
2989 RouteBindAdmission::Admitted { guard, probe } => {
2990 if probe {
2991 info!(
2992 module_id = %target_module_id,
2993 connection_id = ctx.connection_id.get(),
2994 "route.bind breaker half-open: admitting one probe"
2995 );
2996 }
2997 guard
2998 }
2999 RouteBindAdmission::Refused {
3000 consecutive_timeouts,
3001 retry_in,
3002 probe_in_flight,
3003 } => {
3004 return Ok(vec![self.route_open_breaker_refusal_frame(
3005 ctx,
3006 &frame,
3007 &target_module_id,
3008 consecutive_timeouts,
3009 retry_in,
3010 probe_in_flight,
3011 )?]);
3012 }
3013 };
3014
3015 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3020 let relay_deadline = Instant::now() + route_bind_relay_timeout;
3021 let pending = match self
3022 .forwarding
3023 .begin_route_bind_relay_for(
3024 ctx.connection_id,
3025 ctx.egress.clone(),
3026 response_version(&frame),
3027 frame.header.corr,
3028 &target_module_id,
3029 principal.clone(),
3030 Some(project_root),
3031 relay_deadline,
3032 )
3033 .await
3034 {
3035 Ok(pending) => pending,
3036 Err(err) => {
3037 return Ok(vec![self.route_open_refusal_frame(
3038 ctx,
3039 &frame,
3040 &target_module_id,
3041 "relay_reservation_failed",
3042 forwarding_error_code(&err),
3043 err.to_string(),
3044 )?])
3045 }
3046 };
3047 let crate::forwarding::PendingRouteBindRelay {
3048 endpoint,
3049 module_sink,
3050 negotiated_ver,
3051 client_channel,
3052 client_epoch,
3053 module_channel,
3054 module_epoch,
3055 corr: relay_corr,
3056 receiver,
3057 } = pending;
3058 let mut reservation =
3059 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3060
3061 debug!(
3062 connection_id = ctx.connection_id.get(),
3063 client_channel,
3064 client_epoch,
3065 module_channel,
3066 module_epoch,
3067 "reserved route handle pair"
3068 );
3069 let principal_label = match &principal {
3072 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3073 Principal::Direct => "direct".to_string(),
3074 other => format!("{other:?}"),
3075 };
3076 let relay = ModuleControlRequest::RouteBind {
3077 route_channel: module_channel,
3078 epoch: module_epoch,
3079 target,
3080 identity,
3081 principal: Some(principal),
3082 consumer_capabilities,
3083 admission_facts,
3084 };
3085 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3086 RouterError::backend(
3087 0,
3088 frame.header.corr,
3089 format!("failed to encode route.bind request: {err}"),
3090 )
3091 })?;
3092 let relay_frame = Frame::build_with_version(
3093 negotiated_ver,
3094 FrameType::Request,
3095 control_flags(),
3096 0,
3097 0,
3098 relay_corr,
3099 relay_body,
3100 )
3101 .map_err(RouterError::FrameBuild)?;
3102
3103 if let Err(err) = module_sink.send(relay_frame).await {
3104 reservation.release_and_disarm();
3105 return Ok(vec![self.route_open_refusal_frame(
3106 ctx,
3107 &frame,
3108 &target_module_id,
3109 "relay_send_failed",
3110 "target_unavailable",
3111 err.to_string(),
3112 )?]);
3113 }
3114
3115 if !self
3116 .forwarding
3117 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3118 .map_err(RouterError::Forwarding)?
3119 {
3120 self.send_abandoned_route_bind_goodbye(
3121 &module_sink,
3122 negotiated_ver,
3123 module_channel,
3124 module_epoch,
3125 );
3126 }
3127
3128 match timeout_at(relay_deadline, receiver).await {
3129 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3130 reservation.disarm();
3131 if breaker.record_accepted() {
3132 info!(
3133 module_id = %target_module_id,
3134 "route.bind breaker closed: the probe was accepted"
3135 );
3136 }
3137 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3138 Ok(Vec::new())
3139 }
3140 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3141 reservation.release_and_disarm();
3142 breaker.record_inconclusive();
3146 self.counters
3147 .increment_route_open_refused("module_rejected");
3148 info!(
3149 target: "control",
3150 code = "module_rejected",
3151 module_code = ?body.code,
3152 module_id = ?target_module_id,
3153 connection_id = ctx.connection_id.get(),
3154 "route.open refused"
3155 );
3156 Ok(vec![control_error_body_frame(&frame, body)?])
3157 }
3158 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3159 reservation.release_and_disarm();
3160 breaker.record_inconclusive();
3161 tracing::warn!(
3168 module_id = %target_module_id,
3169 "route.bind relay abandoned: {message}"
3170 );
3171 Ok(vec![self.route_open_refusal_frame(
3172 ctx,
3173 &frame,
3174 &target_module_id,
3175 "relay_abandoned",
3176 "target_unavailable",
3177 message,
3178 )?])
3179 }
3180 Ok(Err(_)) => {
3181 reservation.release_and_disarm();
3182 breaker.record_inconclusive();
3183 Ok(vec![self.route_open_refusal_frame(
3184 ctx,
3185 &frame,
3186 &target_module_id,
3187 "relay_waiter_canceled",
3188 "target_unavailable",
3189 "route.bind relay waiter was canceled before the module responded",
3190 )?])
3191 }
3192 Err(_) => {
3193 reservation.release_and_disarm();
3194 if let Some(opened) = breaker.record_timeout(
3198 self.route_bind_breaker_threshold,
3199 self.route_bind_breaker_cooldown,
3200 ) {
3201 warn!(
3202 module_id = %target_module_id,
3203 consecutive_timeouts = opened.consecutive_timeouts,
3204 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3205 reopened_after_probe = opened.reopened_after_probe,
3206 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3207 );
3208 }
3209 tracing::warn!(
3217 module_id = %target_module_id,
3218 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3219 "route.bind relay timed out: module did not ack within budget"
3220 );
3221 Ok(vec![self.route_open_refusal_frame(
3222 ctx,
3223 &frame,
3224 &target_module_id,
3225 "relay_timed_out",
3226 "module_timeout",
3227 format!(
3228 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3229 route_bind_relay_timeout
3230 ),
3231 )?])
3232 }
3233 }
3234 }
3235
3236 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3237 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3238 snapshot: self.supervisor.spawn_snapshot(),
3239 };
3240 Ok(vec![control_response_body_frame(
3241 &frame,
3242 &response,
3243 "ClientControlResponse::SupervisorSpawnSnapshot",
3244 )?])
3245 }
3246
3247 fn handle_supervisor_spawn_subscribe(
3248 &self,
3249 ctx: &RouteCtx,
3250 frame: Frame,
3251 since: Option<SpawnCursor>,
3252 ) -> Result<Vec<Frame>, RouterError> {
3253 match self.supervisor.subscribe_spawns(
3254 ctx.connection_id,
3255 frame.header.corr,
3256 response_version(&frame),
3257 since,
3258 ctx.egress.clone(),
3259 ) {
3260 Ok(()) => Ok(Vec::new()),
3261 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3262 Ok(vec![control_error_body_frame(
3263 &frame,
3264 ErrorBody {
3265 code: "spawn_cursor_incarnation_mismatch".to_string(),
3266 message: "spawn cursor belongs to a different daemon incarnation"
3267 .to_string(),
3268 detail: Some(serde_json::json!({
3269 "current_daemon_incarnation": current
3270 })),
3271 },
3272 )?])
3273 }
3274 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3275 &frame,
3276 ErrorBody {
3277 code: "spawn_cursor_too_old".to_string(),
3278 message: "spawn cursor predates the retained event ring".to_string(),
3279 detail: Some(serde_json::json!({
3280 "oldest_retained_cursor": oldest
3281 })),
3282 },
3283 )?]),
3284 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3285 0,
3286 frame.header.corr,
3287 format!("failed to open supervisor spawn subscription: {error}"),
3288 )),
3289 }
3290 }
3291
3292 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3293 let generation = self
3294 .registry
3295 .generation()
3296 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3297 let mut modules = Vec::new();
3298 for module in self.supervisor.list() {
3299 let status = module.status_for_control("list").map_err(|err| {
3300 RouterError::backend(
3301 0,
3302 frame.header.corr,
3303 format!("failed to read supervisor status: {err}"),
3304 )
3305 })?;
3306 let (configured, _) = module.configuration().map_err(|err| {
3307 RouterError::backend(
3308 0,
3309 frame.header.corr,
3310 format!("failed to read module configuration: {err}"),
3311 )
3312 })?;
3313 let image = module.running_image_agreement().await;
3315 let pending_reload = Some(reload_verdict(
3316 &configured.program,
3317 status.spawned_from.as_deref(),
3318 image,
3319 ));
3320 modules.push(SupervisorEntry {
3321 module_id: status.module_id,
3322 state: status.state.to_string(),
3323 enabled: status.enabled,
3324 live: status.live,
3325 protocol: status.protocol,
3326 health: status.health.status,
3327 pending_reload,
3328 last_probe_ms: status.health.last_probe_ms,
3329 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3330 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3331 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3332 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3333 restart_count: Some(status.restart_count),
3334 max_restarts: Some(status.max_restarts),
3335 lifetime_restarts: Some(status.lifetime_restarts),
3336 spawn_generation: Some(status.spawn_generation),
3337 restart_window_secs: Some(status.restart_window.as_secs()),
3338 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3339 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3340 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3341 });
3342 }
3343 let response = ClientControlResponse::SupervisorList {
3344 generation,
3345 modules,
3346 };
3347 Ok(vec![control_response_body_frame(
3348 &frame,
3349 &response,
3350 "ClientControlResponse::SupervisorList",
3351 )?])
3352 }
3353
3354 fn handle_supervisor_stderr_tail(
3355 &self,
3356 frame: Frame,
3357 module_id: String,
3358 max_lines: Option<u32>,
3359 max_bytes: Option<u32>,
3360 ) -> Result<Vec<Frame>, RouterError> {
3361 let Some(module) = self.supervisor.get(&module_id) else {
3362 return Ok(vec![control_error_frame(
3363 &frame,
3364 "unknown_module",
3365 format!("module_id '{module_id}' is not supervised"),
3366 )?]);
3367 };
3368
3369 let snapshot = module.stderr_tail(
3370 max_lines.map(|value| value as usize),
3371 max_bytes.map(|value| value as usize),
3372 );
3373
3374 let response = ClientControlResponse::SupervisorStderrTail {
3375 module_id,
3376 tail: StderrTail {
3377 capture: match snapshot.capture {
3378 CaptureState::Captured => StderrCaptureState::Captured,
3379 CaptureState::Incomplete { reason } => {
3380 StderrCaptureState::Incomplete { reason }
3381 }
3382 CaptureState::NotCaptured { reason } => {
3383 StderrCaptureState::NotCaptured { reason }
3384 }
3385 },
3386 entries: snapshot
3387 .entries
3388 .into_iter()
3389 .map(|entry| match entry {
3390 TailEntry::Line { text, truncated } => {
3391 StderrTailEntry::Line { text, truncated }
3392 }
3393 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3394 })
3395 .collect(),
3396 dropped_lines: snapshot.dropped_lines,
3397 },
3398 };
3399 Ok(vec![control_response_body_frame(
3400 &frame,
3401 &response,
3402 "ClientControlResponse::SupervisorStderrTail",
3403 )?])
3404 }
3405
3406 async fn handle_supervisor_terminals(
3407 &self,
3408 frame: Frame,
3409 module_id: String,
3410 ) -> Result<Vec<Frame>, RouterError> {
3411 let Some(module) = self.supervisor.get(&module_id) else {
3412 return Ok(vec![control_error_frame(
3413 &frame,
3414 "unknown_module",
3415 format!("module_id '{module_id}' is not supervised"),
3416 )?]);
3417 };
3418
3419 let terminals = module
3422 .read_durable_terminal_history()
3423 .await
3424 .map_err(|error| {
3425 RouterError::backend(
3426 0,
3427 frame.header.corr,
3428 format!("failed to read terminal history: {error}"),
3429 )
3430 })?;
3431 let response = ClientControlResponse::SupervisorTerminals {
3432 module_id,
3433 terminals,
3434 };
3435 Ok(vec![control_response_body_frame(
3436 &frame,
3437 &response,
3438 "ClientControlResponse::SupervisorTerminals",
3439 )?])
3440 }
3441
3442 fn handle_supervisor_routes(
3443 &self,
3444 frame: Frame,
3445 module_id: Option<String>,
3446 ) -> Result<Vec<Frame>, RouterError> {
3447 let modules = self
3448 .forwarding
3449 .route_census(module_id.as_deref())
3450 .map_err(RouterError::Forwarding)?
3451 .into_iter()
3452 .map(|(module_id, routes)| SupervisorRouteModule {
3453 module_id,
3454 routes: routes
3455 .into_iter()
3456 .map(|route| SupervisorRoute {
3457 consumer: match route.principal {
3458 Principal::Reserved { module_id } => {
3459 SupervisorRouteConsumer::Reserved { module_id }
3460 }
3461 Principal::Direct | Principal::Unverified => {
3462 SupervisorRouteConsumer::Direct {
3463 connection_id: route.goodbye_target.connection_id.get(),
3464 }
3465 }
3466 },
3467 age_ms: Instant::now()
3468 .saturating_duration_since(route.bound_at)
3469 .as_millis()
3470 .try_into()
3471 .unwrap_or(u64::MAX),
3472 draining: route.draining,
3473 drain_reason: route.drain_reason,
3474 })
3475 .collect(),
3476 })
3477 .collect();
3478 let response = ClientControlResponse::SupervisorRoutes { modules };
3479 Ok(vec![control_response_body_frame(
3480 &frame,
3481 &response,
3482 "ClientControlResponse::SupervisorRoutes",
3483 )?])
3484 }
3485
3486 async fn handle_supervisor_provenance(
3487 &self,
3488 frame: Frame,
3489 module_id: Option<String>,
3490 ) -> Result<Vec<Frame>, RouterError> {
3491 let mut selected = if let Some(module_id) = module_id {
3492 let Some(module) = self.supervisor.get(&module_id) else {
3493 return Ok(vec![control_error_frame(
3494 &frame,
3495 "unknown_module",
3496 format!("module_id '{module_id}' is not supervised"),
3497 )?]);
3498 };
3499 vec![module]
3500 } else {
3501 self.supervisor.list()
3502 };
3503
3504 let mut modules = Vec::with_capacity(selected.len());
3505 for module in selected.drain(..) {
3506 let status = module.status().map_err(|err| {
3507 RouterError::backend(
3508 0,
3509 frame.header.corr,
3510 format!("failed to read supervisor status: {err}"),
3511 )
3512 })?;
3513 let module_declared = self
3514 .registry
3515 .get_module(&status.module_id)
3516 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3517 .and_then(|registration| registration.manifest.provenance)
3518 .map(|build| ModuleDeclaredProvenance::Reported { build })
3519 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3520 #[cfg(test)]
3521 let running_image = match &self.provenance_probe_override {
3522 Some(result) => result.clone(),
3523 None => module.running_image_agreement().await,
3524 };
3525 #[cfg(not(test))]
3526 let running_image = module.running_image_agreement().await;
3527 modules.push(SupervisorModuleProvenance {
3528 module_id: status.module_id,
3529 module_declared,
3530 daemon_observed: SupervisorObservedProcess {
3531 pid: status.pid,
3532 spawned_at_ms: status.spawned_at_ms,
3533 spawned_from: status.spawned_from,
3534 running_image,
3535 },
3536 });
3537 }
3538 let daemon = SupervisorDaemonProvenance {
3539 daemon_build: self.daemon_provenance.build.clone(),
3540 daemon_observed: DaemonObservedProcess {
3541 pid: self.daemon_provenance.pid,
3542 started_at_ms: self
3543 .daemon_provenance
3544 .start_clock
3545 .map(|clock| clock.started_at_ms())
3546 .or(self.daemon_provenance.started_at_ms),
3547 running_image: self
3548 .daemon_provenance
3549 .probe
3550 .observe(
3551 self.daemon_provenance.pid,
3552 self.daemon_provenance.executable_path.as_deref(),
3553 self.daemon_provenance.executable_identity,
3554 self.daemon_provenance.process_start_time,
3555 )
3556 .await,
3557 },
3558 };
3559 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3560 Ok(vec![control_response_body_frame(
3561 &frame,
3562 &response,
3563 "ClientControlResponse::SupervisorProvenance",
3564 )?])
3565 }
3566
3567 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3568 self.refresh_capability_requirements();
3569 let generation = self
3570 .registry
3571 .generation()
3572 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3573 let modules = self
3574 .supervisor
3575 .list()
3576 .into_iter()
3577 .map(|module| {
3578 let status = module.status_for_control("health").map_err(|err| {
3579 RouterError::backend(
3580 0,
3581 frame.header.corr,
3582 format!("failed to read supervisor health: {err}"),
3583 )
3584 })?;
3585 let module_id = status.module_id;
3586 let capability_detail = self
3587 .capability_evaluator
3588 .required_problem_detail(&module_id);
3589 Ok(SupervisorHealthEntry {
3590 module_id,
3591 status: status.health.status,
3592 detail: append_capability_problem_detail(
3593 status.health.detail,
3594 capability_detail,
3595 ),
3596 metrics: status.health.metrics,
3597 consecutive_failures: status.health.consecutive_failures,
3598 late_answer_count: status.health.late_answer_count,
3599 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
3600 last_action: status.health.last_action,
3601 last_action_ms: status.health.last_action_ms,
3602 last_probe_ms: status.health.last_probe_ms,
3603 })
3604 })
3605 .collect::<Result<Vec<_>, RouterError>>()?;
3606 let response = ClientControlResponse::SupervisorHealth {
3607 generation,
3608 modules,
3609 };
3610 Ok(vec![control_response_body_frame(
3611 &frame,
3612 &response,
3613 "ClientControlResponse::SupervisorHealth",
3614 )?])
3615 }
3616
3617 async fn handle_supervisor_restart(
3618 &self,
3619 frame: Frame,
3620 module_id: String,
3621 drain_timeout_ms: Option<u64>,
3622 ) -> Result<Vec<Frame>, RouterError> {
3623 let operation_lock = self.supervisor.operation_lock();
3624 let _operation_guard = operation_lock.lock().await;
3625 let Some(module) = self.supervisor.get(&module_id) else {
3626 return Ok(vec![control_error_frame(
3627 &frame,
3628 "unknown_module",
3629 format!("module_id '{module_id}' is not supervised"),
3630 )?]);
3631 };
3632
3633 if let Err(err) = module.restart(drain_timeout_ms).await {
3634 let (code, message) = match err {
3635 crate::supervise::SuperviseError::Disabled { .. } => {
3636 ("module_disabled", err.to_string())
3637 }
3638 crate::supervise::SuperviseError::SwapInProgress { .. } => {
3639 ("swap_in_progress", err.to_string())
3640 }
3641 _ => (
3642 "target_unavailable",
3643 format!("failed to restart module_id '{module_id}': {err}"),
3644 ),
3645 };
3646 return Ok(vec![control_error_frame(&frame, code, message)?]);
3647 }
3648
3649 let response = ClientControlResponse::SupervisorAck {
3650 module_id,
3651 applied: true,
3652 };
3653 Ok(vec![control_response_body_frame(
3654 &frame,
3655 &response,
3656 "ClientControlResponse::SupervisorAck",
3657 )?])
3658 }
3659
3660 async fn handle_supervisor_swap(
3664 &self,
3665 frame: Frame,
3666 module_id: String,
3667 ready_timeout_ms: Option<u64>,
3668 ) -> Result<Vec<Frame>, RouterError> {
3669 let module = {
3676 let operation_lock = self.supervisor.operation_lock();
3677 let _operation_guard = operation_lock.lock().await;
3678 self.supervisor.get(&module_id)
3679 };
3680 let Some(module) = module else {
3681 return Ok(vec![control_error_frame(
3682 &frame,
3683 "unknown_module",
3684 format!("module_id '{module_id}' is not supervised"),
3685 )?]);
3686 };
3687
3688 if let Err(err) = module
3689 .swap(ready_timeout_ms.map(Duration::from_millis))
3690 .await
3691 {
3692 use crate::supervise::SuperviseError;
3693 let message = err.to_string();
3694 let error = match err {
3695 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
3696 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
3697 code: "swap_refused".to_string(),
3698 message,
3699 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
3700 },
3701 SuperviseError::SwapFailed {
3702 arm,
3703 candidate_exit,
3704 ..
3705 } => ErrorBody {
3706 code: "swap_failed".to_string(),
3707 message,
3708 detail: Some(serde_json::json!({
3709 "arm": arm.as_str(),
3710 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
3711 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
3712 })),
3713 },
3714 _ => ErrorBody::new(
3715 "target_unavailable",
3716 format!("failed to swap module_id '{module_id}': {message}"),
3717 ),
3718 };
3719 return Ok(vec![control_error_body_frame(&frame, error)?]);
3720 }
3721
3722 let response = ClientControlResponse::SupervisorAck {
3723 module_id,
3724 applied: true,
3725 };
3726 Ok(vec![control_response_body_frame(
3727 &frame,
3728 &response,
3729 "ClientControlResponse::SupervisorAck",
3730 )?])
3731 }
3732
3733 async fn handle_supervisor_reload(
3734 &self,
3735 frame: Frame,
3736 module_id: String,
3737 ) -> Result<Vec<Frame>, RouterError> {
3738 let operation_lock = self.supervisor.operation_lock();
3739 let _operation_guard = operation_lock.lock().await;
3740 let Some(module) = self.supervisor.get(&module_id) else {
3741 return Ok(vec![control_error_frame(
3742 &frame,
3743 "unknown_module",
3744 format!("module_id '{module_id}' is not supervised"),
3745 )?]);
3746 };
3747
3748 if let Err(err) = module.reload().await {
3749 let (code, message) = match err {
3750 crate::supervise::SuperviseError::Disabled { .. } => {
3751 ("module_disabled", err.to_string())
3752 }
3753 crate::supervise::SuperviseError::SwapInProgress { .. } => {
3754 ("swap_in_progress", err.to_string())
3755 }
3756 _ => (
3757 "reload_failed",
3758 format!("failed to reload module_id '{module_id}': {err}"),
3759 ),
3760 };
3761 return Ok(vec![control_error_frame(&frame, code, message)?]);
3762 }
3763
3764 let response = ClientControlResponse::SupervisorAck {
3765 module_id,
3766 applied: true,
3767 };
3768 Ok(vec![control_response_body_frame(
3769 &frame,
3770 &response,
3771 "ClientControlResponse::SupervisorAck",
3772 )?])
3773 }
3774
3775 async fn handle_supervisor_rescan(
3776 &self,
3777 frame: Frame,
3778 preview: bool,
3779 ) -> Result<Vec<Frame>, RouterError> {
3780 let Some(context) = self.rescan.clone() else {
3781 return Ok(vec![control_error_frame(
3782 &frame,
3783 "rescan_unavailable",
3784 "the daemon was not started with a reloadable config path".to_string(),
3785 )?]);
3786 };
3787
3788 let operation_lock = self.supervisor.operation_lock();
3789 let _operation_guard = operation_lock.lock().await;
3790 let loaded = match crate::daemon_config::load(&context.config_path) {
3791 Ok(config) => config,
3792 Err(err) => {
3793 return Ok(vec![control_error_frame(
3794 &frame,
3795 "invalid_daemon_config",
3796 format!("supervisor rescan rejected daemon config: {err}"),
3797 )?])
3798 }
3799 };
3800 let Some(config) = loaded else {
3808 return Ok(vec![control_error_frame(
3809 &frame,
3810 "invalid_daemon_config",
3811 format!(
3812 "daemon config not found at {}; refusing to rescan (an absent config would \
3813 retire every supervised module)",
3814 context.config_path.display()
3815 ),
3816 )?]);
3817 };
3818 let (
3819 configured_port,
3820 storage_config,
3821 admission_facts_carrier_module_id,
3822 admission_facts_targets,
3823 modules,
3824 reserved_capabilities,
3825 ) = (
3826 config.port,
3827 config.storage,
3828 config.admission_facts_carrier_module_id,
3829 config.admission_facts_targets,
3830 config.modules,
3831 config.reserved_capabilities,
3832 );
3833
3834 let mut restart_required = Vec::new();
3843 for section in RestartRequiredSection::ALL {
3844 let changed = match section {
3845 RestartRequiredSection::Port => configured_port != context.configured_port,
3846 RestartRequiredSection::Storage => storage_config != context.storage_config,
3847 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
3848 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
3849 }
3850 RestartRequiredSection::AdmissionFactsTargets => {
3851 admission_facts_targets != context.admission_facts_targets
3852 }
3853 };
3854 if changed {
3855 restart_required.push(section.label().to_string());
3856 }
3857 }
3858 if !restart_required.is_empty() {
3859 warn!(
3860 config_path = %context.config_path.display(),
3861 sections = %restart_required.join(", "),
3862 "daemon config changed outside the modules section; restart the daemon to apply those changes"
3863 );
3864 }
3865
3866 for configured in &modules {
3867 if let Err(err) = validate_spec(&configured.module_spec()) {
3868 return Ok(vec![control_error_frame(
3869 &frame,
3870 "invalid_daemon_config",
3871 format!("supervisor rescan rejected daemon config: {err}"),
3872 )?]);
3873 }
3874 }
3875
3876 let configured_capabilities = modules
3877 .iter()
3878 .map(|module| (module.module_id.clone(), module.enabled))
3879 .collect::<Vec<_>>();
3880 let preview_capability_warnings = if preview {
3881 let (_, registrations) = self.runtime_capability_snapshot()?;
3882 let current_modules = self
3883 .supervisor
3884 .list()
3885 .into_iter()
3886 .map(|module| module.module_id().to_string())
3887 .collect::<BTreeSet<_>>();
3888 let resulting_modules = configured_capabilities.clone();
3889 let removed = current_modules
3890 .into_iter()
3891 .filter(|module_id| {
3892 !resulting_modules
3893 .iter()
3894 .any(|(configured_id, _)| configured_id == module_id)
3895 })
3896 .collect::<Vec<_>>();
3897 self.capability_evaluator.preview_removal_warnings(
3898 resulting_modules,
3899 &removed,
3900 ®istrations,
3901 )
3902 } else {
3903 Vec::new()
3904 };
3905 let result = match self
3906 .reconcile_supervised_modules(&context.supervisor, modules, preview)
3907 .await
3908 {
3909 Ok(result) => result,
3910 Err(message) => {
3911 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
3912 }
3913 };
3914 if !preview {
3915 self.capability_evaluator
3916 .configure(configured_capabilities, reserved_capabilities);
3917 self.capability_evaluator.wake_deadline_loop();
3918 self.refresh_capability_requirements();
3919 }
3920 let mut result = result;
3921 result.restart_required = restart_required;
3922 result.capability_warnings = preview_capability_warnings;
3923 let response = ClientControlResponse::SupervisorRescan { result };
3924 Ok(vec![control_response_body_frame(
3925 &frame,
3926 &response,
3927 "ClientControlResponse::SupervisorRescan",
3928 )?])
3929 }
3930
3931 async fn handle_supervisor_release_reserved(
3932 &self,
3933 frame: Frame,
3934 module_id: String,
3935 ) -> Result<Vec<Frame>, RouterError> {
3936 let Some(context) = self.rescan.clone() else {
3937 return Ok(vec![control_error_frame(
3938 &frame,
3939 "release_unavailable",
3940 "reserved-id release requires a daemon started with a reloadable config path",
3941 )?]);
3942 };
3943 let operation_lock = self.supervisor.operation_lock();
3944 let _operation_guard = operation_lock.lock().await;
3945 let loaded = match crate::daemon_config::load(&context.config_path) {
3946 Ok(Some(config)) => config,
3947 Ok(None) => {
3948 return Ok(vec![control_error_frame(
3949 &frame,
3950 "invalid_daemon_config",
3951 format!(
3952 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
3953 context.config_path.display()
3954 ),
3955 )?])
3956 }
3957 Err(err) => {
3958 return Ok(vec![control_error_frame(
3959 &frame,
3960 "invalid_daemon_config",
3961 format!("unable to verify reserved-id release against daemon config: {err}"),
3962 )?])
3963 }
3964 };
3965 if loaded
3966 .modules
3967 .iter()
3968 .any(|configured| configured.module_id == module_id)
3969 {
3970 return Ok(vec![control_error_frame(
3971 &frame,
3972 "reserved_module_configured",
3973 format!(
3974 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
3975 ),
3976 )?]);
3977 }
3978 if !self.supervisor.release_retained_reserved_gate(&module_id) {
3979 return Ok(vec![control_error_frame(
3980 &frame,
3981 "reserved_gate_not_retained",
3982 format!(
3983 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
3984 ),
3985 )?]);
3986 }
3987
3988 let response = ClientControlResponse::SupervisorAck {
3989 module_id,
3990 applied: true,
3991 };
3992 Ok(vec![control_response_body_frame(
3993 &frame,
3994 &response,
3995 "ClientControlResponse::SupervisorAck",
3996 )?])
3997 }
3998
3999 async fn reconcile_supervised_modules(
4008 &self,
4009 supervisor: &Supervisor,
4010 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4011 preview: bool,
4012 ) -> Result<SupervisorRescanResult, String> {
4013 let mut current = BTreeMap::new();
4014 for module in self.supervisor.list() {
4015 let (spec, health) = module.configuration().map_err(|err| {
4016 format!(
4017 "failed to read configuration for module_id '{}': {err}",
4018 module.module_id()
4019 )
4020 })?;
4021 let enabled = module
4022 .status()
4023 .map_err(|err| {
4024 format!(
4025 "failed to read status for module_id '{}': {err}",
4026 module.module_id()
4027 )
4028 })?
4029 .enabled;
4030 current.insert(
4031 module.module_id().to_string(),
4032 (module, spec, health, enabled),
4033 );
4034 }
4035 let configured = configured_modules
4036 .into_iter()
4037 .map(|module| (module.module_id.clone(), module))
4038 .collect::<BTreeMap<_, _>>();
4039
4040 let added = configured
4041 .keys()
4042 .filter(|module_id| !current.contains_key(*module_id))
4043 .cloned()
4044 .collect::<Vec<_>>();
4045 let removed = current
4046 .keys()
4047 .filter(|module_id| !configured.contains_key(*module_id))
4048 .cloned()
4049 .collect::<Vec<_>>();
4050 let mut changed_pending_reload = Vec::new();
4051 let mut configuration_changes = BTreeSet::new();
4052 let mut enabled_changes = BTreeSet::new();
4053 let mut unchanged = 0_u32;
4054
4055 for (module_id, configured_module) in &configured {
4056 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4057 else {
4058 continue;
4059 };
4060 let configuration_changed = *current_spec != configured_module.module_spec()
4061 || *current_health != configured_module.health;
4062 let enabled_changed = *current_enabled != configured_module.enabled;
4063 if configuration_changed {
4064 configuration_changes.insert(module_id.clone());
4065 changed_pending_reload.push(module_id.clone());
4066 }
4067 if enabled_changed {
4068 enabled_changes.insert(module_id.clone());
4069 }
4070 if !configuration_changed && !enabled_changed {
4071 unchanged = unchanged.saturating_add(1);
4072 }
4073 }
4074
4075 if preview {
4081 return Ok(SupervisorRescanResult {
4082 added,
4083 removed,
4084 changed_pending_reload,
4085 enabled_changes: enabled_changes.iter().cloned().collect(),
4086 unchanged,
4087 preview: true,
4088 restart_required: Vec::new(),
4092 capability_warnings: Vec::new(),
4093 });
4094 }
4095
4096 for module_id in &removed {
4097 let module = ¤t
4098 .get(module_id)
4099 .expect("removed module came from current supervisor state")
4100 .0;
4101 module.retire().await.map_err(|err| {
4102 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4103 })?;
4104 self.supervisor.record_rescan_removal(module_id);
4138 self.supervisor.retire(module_id);
4139 }
4140
4141 for module_id in configured.keys() {
4142 let Some((module, _, _, _)) = current.get(module_id) else {
4143 continue;
4144 };
4145 let configured_module = configured
4146 .get(module_id)
4147 .expect("configured module id came from configured map");
4148 if configuration_changes.contains(module_id) {
4149 module
4150 .update_configuration(
4151 configured_module.module_spec(),
4152 configured_module.health,
4153 configured_module.drain_timeout_ms,
4154 )
4155 .await
4156 .map_err(|err| {
4157 format!(
4158 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4159 )
4160 })?;
4161 }
4162 if enabled_changes.contains(module_id) {
4163 module
4164 .set_enabled(configured_module.enabled)
4165 .await
4166 .map_err(|err| {
4167 format!(
4168 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4169 configured_module.enabled
4170 )
4171 })?;
4172 }
4173 }
4174
4175 for module_id in &added {
4176 let configured_module = configured
4177 .get(module_id)
4178 .expect("added module id came from configured map");
4179 supervisor
4180 .supervise_configured_with_health(
4181 configured_module.module_spec(),
4182 configured_module.enabled,
4183 configured_module.health,
4184 configured_module.drain_timeout_ms,
4185 configured_module.restart,
4186 )
4187 .map_err(|err| {
4188 format!("failed to add module_id '{module_id}' during rescan: {err}")
4189 })?;
4190 }
4191
4192 Ok(SupervisorRescanResult {
4193 added,
4194 removed,
4195 changed_pending_reload,
4196 enabled_changes: enabled_changes.iter().cloned().collect(),
4197 unchanged,
4198 preview: false,
4199 restart_required: Vec::new(),
4202 capability_warnings: Vec::new(),
4203 })
4204 }
4205
4206 async fn handle_supervisor_set_enabled(
4207 &self,
4208 frame: Frame,
4209 module_id: String,
4210 enabled: bool,
4211 ) -> Result<Vec<Frame>, RouterError> {
4212 let operation_lock = self.supervisor.operation_lock();
4213 let _operation_guard = operation_lock.lock().await;
4214 let Some(module) = self.supervisor.get(&module_id) else {
4215 return Ok(vec![control_error_frame(
4216 &frame,
4217 "unknown_module",
4218 format!("module_id '{module_id}' is not supervised"),
4219 )?]);
4220 };
4221
4222 let applied = match module.set_enabled(enabled).await {
4223 Ok(applied) => applied,
4224 Err(err) => {
4225 return Ok(vec![control_error_frame(
4226 &frame,
4227 "target_unavailable",
4228 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4229 )?])
4230 }
4231 };
4232
4233 self.capability_evaluator.wake_deadline_loop();
4234 self.refresh_capability_requirements();
4235 let response = ClientControlResponse::SupervisorAck { module_id, applied };
4236 Ok(vec![control_response_body_frame(
4237 &frame,
4238 &response,
4239 "ClientControlResponse::SupervisorAck",
4240 )?])
4241 }
4242
4243 async fn handle_supervisor_health_probe(
4244 &self,
4245 frame: Frame,
4246 module_id: String,
4247 ) -> Result<Vec<Frame>, RouterError> {
4248 self.refresh_capability_requirements();
4249 let Some(registration) = self
4250 .registry
4251 .get_module(&module_id)
4252 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4253 else {
4254 return Ok(vec![control_error_frame(
4255 &frame,
4256 "unknown_module",
4257 format!("module_id '{module_id}' is not registered"),
4258 )?]);
4259 };
4260
4261 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4274 {
4275 return Ok(vec![control_error_frame(
4276 &frame,
4277 "health_not_advertised",
4278 format!("module_id '{module_id}' did not advertise health.check"),
4279 )?]);
4280 }
4281
4282 let deadline = Instant::now() + self.health_probe_timeout;
4283 let pending = match self.forwarding.begin_module_control_rpc_for(
4284 &module_id,
4285 MODULE_CONTROL_OP_HEALTH_CHECK,
4286 deadline,
4287 ) {
4288 Ok(pending) => pending,
4289 Err(err) => {
4290 return Ok(vec![control_error_frame(
4291 &frame,
4292 forwarding_error_code(&err),
4293 err.to_string(),
4294 )?])
4295 }
4296 };
4297
4298 let PendingModuleControlRpc {
4299 endpoint,
4300 module_sink,
4301 negotiated_ver,
4302 corr: probe_corr,
4303 receiver,
4304 } = pending;
4305 let mut guard =
4306 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4307 let probe_body =
4308 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4309 RouterError::backend(
4310 0,
4311 frame.header.corr,
4312 format!("failed to encode health.check request: {err}"),
4313 )
4314 })?;
4315 let probe_frame = Frame::build_with_version(
4316 negotiated_ver,
4317 FrameType::Request,
4318 control_flags(),
4319 0,
4320 0,
4321 probe_corr,
4322 probe_body,
4323 )
4324 .map_err(RouterError::FrameBuild)?;
4325
4326 if let Err(err) = module_sink.send(probe_frame).await {
4327 return Ok(vec![control_error_frame(
4328 &frame,
4329 "target_unavailable",
4330 err.to_string(),
4331 )?]);
4332 }
4333
4334 match timeout_at(deadline, receiver).await {
4335 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4336 guard.disarm();
4337 let Some(report) = response.health_report() else {
4338 return Ok(vec![control_error_frame(
4339 &frame,
4340 "invalid_control_body",
4341 "health.check RPC returned a non-health response",
4342 )?]);
4343 };
4344 let HealthReport {
4349 status,
4350 detail,
4351 metrics,
4352 } = report;
4353 let capability_detail = self
4354 .capability_evaluator
4355 .required_problem_detail(&module_id);
4356 let response = ClientControlResponse::SupervisorHealthProbe {
4357 module_id,
4358 status,
4359 detail: append_capability_problem_detail(detail, capability_detail),
4360 metrics,
4361 };
4362 Ok(vec![control_response_body_frame(
4363 &frame,
4364 &response,
4365 "ClientControlResponse::SupervisorHealthProbe",
4366 )?])
4367 }
4368 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4369 guard.disarm();
4370 Ok(vec![control_error_body_frame(&frame, body)?])
4371 }
4372 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4373 guard.disarm();
4374 Ok(vec![control_error_frame(
4375 &frame,
4376 "target_unavailable",
4377 message,
4378 )?])
4379 }
4380 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4381 guard.disarm();
4382 Ok(vec![control_error_frame(
4383 &frame,
4384 "invalid_control_body",
4385 message,
4386 )?])
4387 }
4388 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4389 guard.disarm();
4390 Ok(vec![control_error_frame(
4391 &frame,
4392 "invalid_control_body",
4393 format!("expected module-control op '{expected}', got '{actual}'"),
4394 )?])
4395 }
4396 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4397 guard.disarm();
4398 Ok(vec![control_error_frame(
4399 &frame,
4400 "module_timeout",
4401 format!(
4402 "module_id '{module_id}' answered health.check after {:?}",
4403 self.health_probe_timeout
4404 ),
4405 )?])
4406 }
4407 Ok(Err(_)) => Ok(vec![control_error_frame(
4408 &frame,
4409 "target_unavailable",
4410 "health.check waiter was canceled before the module responded",
4411 )?]),
4412 Err(_) => Ok(vec![control_error_frame(
4413 &frame,
4414 "module_timeout",
4415 format!(
4416 "module_id '{module_id}' did not answer health.check within {:?}",
4417 self.health_probe_timeout
4418 ),
4419 )?]),
4420 }
4421 }
4422
4423 fn supervisor_status(
4424 &self,
4425 module_id: &str,
4426 corr: u64,
4427 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4428 self.supervisor
4429 .get(module_id)
4430 .map(|module| {
4431 let warming = module.is_warming_for_control("status").map_err(|err| {
4432 RouterError::backend(
4433 0,
4434 corr,
4435 format!(
4436 "failed to read supervisor warming state for module_id '{module_id}': {err}"
4437 ),
4438 )
4439 })?;
4440 module.status_for_control("status").map_err(|err| {
4441 RouterError::backend(
4442 0,
4443 corr,
4444 format!(
4445 "failed to read supervisor status for module_id '{module_id}': {err}"
4446 ),
4447 )
4448 }).map(|status| (status, warming))
4449 })
4450 .transpose()
4451 }
4452
4453 fn guard_module_control_op(
4454 &self,
4455 frame: &Frame,
4456 module_id: &str,
4457 op: &str,
4458 ) -> Result<Option<Frame>, RouterError> {
4459 if self.module_grants_op(module_id, op, frame.header.corr)? {
4460 return Ok(None);
4461 }
4462
4463 Ok(Some(control_error_frame(
4464 frame,
4465 "op_not_allowed",
4466 format!("module_id '{module_id}' did not grant control op '{op}'"),
4467 )?))
4468 }
4469
4470 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4471 let Some(registration) = self
4472 .registry
4473 .get_module(module_id)
4474 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4475 else {
4476 return Ok(false);
4477 };
4478 Ok(module_registration_grants_op(®istration.control_ops, op))
4479 }
4480
4481 fn handle_status_update(
4482 &self,
4483 endpoint: ModuleEndpointId,
4484 frame: Frame,
4485 ) -> Result<Vec<Frame>, RouterError> {
4486 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4487 Ok(update) => update,
4488 Err(err) => {
4489 if is_known_module_push_op(&frame.body) {
4494 return Ok(vec![control_error_frame(
4495 &frame,
4496 "invalid_control_body",
4497 format!("malformed module control push body: {err}"),
4498 )?]);
4499 }
4500 return Ok(Vec::new());
4501 }
4502 };
4503
4504 match update {
4505 ModuleControlPush::RouteStatus {
4506 route_channel,
4507 route_epoch,
4508 status,
4509 } => {
4510 self.forwarding
4511 .cache_status(endpoint, route_channel, route_epoch, status)
4512 .map_err(RouterError::Forwarding)?;
4513 }
4514 }
4515 Ok(Vec::new())
4516 }
4517
4518 fn handle_route_poll(
4519 &self,
4520 ctx: &RouteCtx,
4521 frame: Frame,
4522 route_channel: u16,
4523 route_epoch: u32,
4524 kind: PollKind,
4525 ) -> Result<Vec<Frame>, RouterError> {
4526 let snapshot = self
4527 .forwarding
4528 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4529 .map_err(RouterError::Forwarding)?;
4530 let response = match (kind, snapshot) {
4531 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4532 ClientControlResponse::RoutePoll {
4533 route_channel,
4534 route_epoch,
4535 status,
4536 live: None,
4537 }
4538 }
4539 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4540 route_channel,
4541 route_epoch,
4542 status: None,
4543 live: None,
4544 },
4545 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
4546 let live = self
4572 .process_liveness
4573 .as_ref()
4574 .and_then(|source| source.process_live(&module_id))
4575 .unwrap_or(true);
4576 ClientControlResponse::RoutePoll {
4577 route_channel,
4578 route_epoch,
4579 status: None,
4580 live: Some(live),
4581 }
4582 }
4583 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4584 route_channel,
4585 route_epoch,
4586 status: None,
4587 live: Some(false),
4588 },
4589 };
4590
4591 Ok(vec![control_response_body_frame(
4592 &frame,
4593 &response,
4594 "ClientControlResponse::RoutePoll",
4595 )?])
4596 }
4597
4598 pub(crate) fn observe_module_control_completion(
4599 &self,
4600 completion: ModuleControlRpcCompletion,
4601 ) -> bool {
4602 match completion {
4603 ModuleControlRpcCompletion::Unknown => false,
4604 ModuleControlRpcCompletion::Settled => true,
4605 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
4606 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
4607 info!(
4608 module_id = %module_id,
4609 latency_ms,
4610 "late health.check answer proves the module is alive"
4611 );
4612 match self
4613 .supervisor
4614 .record_late_health_answer(&module_id, latency_ms)
4615 {
4616 Ok(true) => {}
4617 Ok(false) => debug!(
4618 module_id = %module_id,
4619 latency_ms,
4620 "late health.check answer has no active supervisor snapshot"
4621 ),
4622 Err(err) => warn!(
4623 module_id = %module_id,
4624 latency_ms,
4625 error = %err,
4626 "failed to record late health.check answer"
4627 ),
4628 }
4629 true
4630 }
4631 }
4632 }
4633
4634 fn refuse_to_end_module_connection_for_a_client(
4655 &self,
4656 module_connection_id: ConnectionId,
4657 corr: u64,
4658 err: ForwardingError,
4659 ) -> Result<(), RouterError> {
4660 if let ForwardingError::ConnectionClosing { connection_id } = err {
4661 if connection_id != module_connection_id {
4662 warn!(
4663 module_connection_id = module_connection_id.get(),
4664 client_connection_id = connection_id.get(),
4665 corr,
4666 "dropping a route.bind response for a closing client; the module connection keeps serving"
4667 );
4668 return Ok(());
4669 }
4670 }
4671 Err(RouterError::Forwarding(err))
4672 }
4673
4674 fn handle_module_relay_response(
4675 &self,
4676 connection_id: ConnectionId,
4677 frame: Frame,
4678 ) -> Result<Vec<Frame>, RouterError> {
4679 let mut secondary_error = None;
4680 let outcome = match frame.header.ty {
4681 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
4682 Ok(probe) if probe.op == "route.bind" => {
4683 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
4684 Ok(ModuleControlResponse::RouteBindAck {}) => {
4685 RouteBindRelayOutcome::Accepted
4686 }
4687 Ok(other) => {
4688 let message =
4689 format!("route.bind response carried unexpected body: {other:?}");
4690 secondary_error = Some(control_error_frame(
4691 &frame,
4692 "invalid_control_body",
4693 message.clone(),
4694 )?);
4695 RouteBindRelayOutcome::ModuleGone(message)
4696 }
4697 Err(err) => {
4698 let message = format!("malformed route.bind response body: {err}");
4699 secondary_error = Some(control_error_frame(
4700 &frame,
4701 "invalid_control_body",
4702 message.clone(),
4703 )?);
4704 RouteBindRelayOutcome::ModuleGone(message)
4705 }
4706 }
4707 }
4708 Ok(probe) => {
4709 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
4710 {
4711 Ok(response) => ModuleControlRpcOutcome::Response(response),
4712 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4713 "malformed {} response body: {err}",
4714 probe.op
4715 )),
4716 };
4717 let completion = self
4718 .forwarding
4719 .complete_module_control_rpc(
4720 connection_id,
4721 frame.header.corr,
4722 Some(&probe.op),
4723 outcome,
4724 )
4725 .map_err(RouterError::Forwarding)?;
4726 if !self.observe_module_control_completion(completion) {
4727 debug!(
4728 connection_id = connection_id.get(),
4729 corr = frame.header.corr,
4730 op = %probe.op,
4731 "dropping late or unknown module-control RPC response"
4732 );
4733 }
4734 return Ok(Vec::new());
4735 }
4736 Err(err) => {
4737 if let Some(expected_op) = self
4738 .forwarding
4739 .pending_module_control_op(connection_id, frame.header.corr)
4740 .map_err(RouterError::Forwarding)?
4741 {
4742 let completion = self
4743 .forwarding
4744 .complete_module_control_rpc(
4745 connection_id,
4746 frame.header.corr,
4747 None,
4748 ModuleControlRpcOutcome::MalformedResponse(format!(
4749 "malformed {expected_op} response body: {err}"
4750 )),
4751 )
4752 .map_err(RouterError::Forwarding)?;
4753 if !self.observe_module_control_completion(completion) {
4754 debug!(
4755 connection_id = connection_id.get(),
4756 corr = frame.header.corr,
4757 "dropping late malformed module-control RPC response"
4758 );
4759 }
4760 return Ok(Vec::new());
4761 }
4762 let message = format!("malformed route.bind response body: {err}");
4763 secondary_error = Some(control_error_frame(
4764 &frame,
4765 "invalid_control_body",
4766 message.clone(),
4767 )?);
4768 RouteBindRelayOutcome::ModuleGone(message)
4769 }
4770 },
4771 FrameType::Error => {
4772 if self
4773 .forwarding
4774 .pending_module_control_op(connection_id, frame.header.corr)
4775 .map_err(RouterError::Forwarding)?
4776 .is_some()
4777 {
4778 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
4779 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
4780 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4781 "malformed module-control ERROR body: {err}"
4782 )),
4783 };
4784 let completion = self
4785 .forwarding
4786 .complete_module_control_rpc(
4787 connection_id,
4788 frame.header.corr,
4789 None,
4790 outcome,
4791 )
4792 .map_err(RouterError::Forwarding)?;
4793 if !self.observe_module_control_completion(completion) {
4794 debug!(
4795 connection_id = connection_id.get(),
4796 corr = frame.header.corr,
4797 "dropping late or unknown module-control RPC error"
4798 );
4799 }
4800 return Ok(Vec::new());
4801 }
4802 match serde_json::from_slice::<ErrorBody>(&frame.body) {
4803 Ok(body) => RouteBindRelayOutcome::Rejected(body),
4804 Err(err) => {
4805 let message = format!("malformed route.bind ERROR body: {err}");
4806 secondary_error = Some(control_error_frame(
4807 &frame,
4808 "invalid_control_body",
4809 message.clone(),
4810 )?);
4811 RouteBindRelayOutcome::ModuleGone(message)
4812 }
4813 }
4814 }
4815 ty => {
4816 return Ok(vec![control_error_frame(
4817 &frame,
4818 "unsupported_control_frame",
4819 format!("unsupported module channel-0 frame {ty:?}"),
4820 )?])
4821 }
4822 };
4823
4824 let settled =
4825 self.forwarding
4826 .complete_pending_relay(connection_id, frame.header.corr, outcome);
4827 let completion = match settled {
4828 Ok(completion) => completion,
4829 Err(err) => {
4830 self.refuse_to_end_module_connection_for_a_client(
4831 connection_id,
4832 frame.header.corr,
4833 err,
4834 )?;
4835 return Ok(secondary_error.into_iter().collect());
4836 }
4837 };
4838 if let Some(target) = completion.abandoned.as_ref() {
4839 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
4840 }
4841 if !completion.settled {
4842 debug!(
4843 connection_id = connection_id.get(),
4844 corr = frame.header.corr,
4845 frame_type = ?frame.header.ty,
4846 "dropping late or unknown route.bind relay response"
4847 );
4848 }
4849 Ok(secondary_error.into_iter().collect())
4850 }
4851
4852 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
4853 debug!(connection_id = connection_id.get(), "handling GOODBYE");
4854 let registrations = self
4855 .deregister_connection(connection_id)
4856 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
4857 let released_routes = self
4858 .forwarding
4859 .cleanup_connection(connection_id)
4860 .map_err(RouterError::Forwarding)?;
4861 self.emit_route_goodbyes(released_routes);
4862 if !registrations.is_empty() {
4864 crate::supervise::notify_registration_release();
4865 }
4866 Ok(Vec::new())
4867 }
4868}
4869
4870impl Default for ControlHandler {
4871 fn default() -> Self {
4872 Self::new(Arc::new(Registry::default()))
4873 }
4874}
4875
4876impl crate::supervise::SwapPromotionObserver for ControlHandler {
4877 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
4878 self.apply_registration_capabilities(registration);
4879 }
4880}
4881
4882fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
4883 CapabilityRequirementStatus {
4884 consumer: status.consumer,
4885 capability: status.capability,
4886 need: match status.need {
4887 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
4888 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
4889 },
4890 verdict: status.verdict.as_str().to_string(),
4891 episode_seq: status.episode_seq,
4892 config_satisfiable: status.config_satisfiable,
4893 runtime_available: status.runtime_available,
4894 detail: status.detail,
4895 }
4896}
4897
4898fn append_capability_problem_detail(
4899 detail: Option<String>,
4900 capability_detail: Option<String>,
4901) -> Option<String> {
4902 match (detail, capability_detail) {
4903 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
4904 (Some(detail), None) => Some(detail),
4905 (None, Some(capability_detail)) => Some(capability_detail),
4906 (None, None) => None,
4907 }
4908}
4909
4910fn subc_ops() -> Vec<String> {
4911 SUBC_CONTROL_OPS
4912 .iter()
4913 .map(|op| (*op).to_string())
4914 .collect()
4915}
4916
4917fn module_subc_ops() -> Vec<String> {
4918 SUBC_CONTROL_OPS
4919 .iter()
4920 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
4921 .map(|op| (*op).to_string())
4922 .collect()
4923}
4924
4925#[cfg(test)]
4926fn module_baseline_control_ops() -> Vec<String> {
4927 MODULE_BASELINE_CONTROL_OPS
4928 .iter()
4929 .map(|op| (*op).to_string())
4930 .collect()
4931}
4932
4933fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
4934 let mut seen = HashSet::new();
4935 let mut effective = Vec::new();
4936 for op in MODULE_BASELINE_CONTROL_OPS {
4937 if seen.insert((*op).to_string()) {
4938 effective.push((*op).to_string());
4939 }
4940 }
4941 for op in declared.unwrap_or_default() {
4942 if seen.insert(op.clone()) {
4943 effective.push(op);
4944 }
4945 }
4946 effective
4947}
4948
4949fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
4950 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
4951}
4952
4953fn target_module_id(target: &RouteTarget) -> &str {
4954 match target {
4955 RouteTarget::ToolProvider { module_id }
4956 | RouteTarget::ManagementSurface { module_id }
4957 | RouteTarget::InternalService { module_id, .. } => module_id,
4958 }
4959}
4960
4961fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
4962 roles.iter().any(|role| match (target, role) {
4963 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
4964 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
4965 (
4966 RouteTarget::InternalService { service_id, .. },
4967 ProviderRole::InternalService {
4968 service_id: provided,
4969 ..
4970 },
4971 ) => service_id == provided,
4972 _ => false,
4973 })
4974}
4975
4976fn is_routable_role(role: &ProviderRole) -> bool {
4977 matches!(
4978 role,
4979 ProviderRole::ToolProvider { .. }
4980 | ProviderRole::ManagementSurface { .. }
4981 | ProviderRole::InternalService { .. }
4982 )
4983}
4984
4985#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4986enum ControlRequestBodyError {
4987 UnknownOp,
4988 InvalidBody,
4989}
4990
4991#[derive(Debug, Deserialize)]
4992struct ControlOpProbe {
4993 op: String,
4994}
4995
4996const MODULE_PUSH_OPS: &[&str] = &["route.status"];
4999
5000fn is_known_module_push_op(body: &[u8]) -> bool {
5001 serde_json::from_slice::<ControlOpProbe>(body)
5002 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5003 .unwrap_or(false)
5004}
5005
5006fn is_known_module_request_op(body: &[u8]) -> bool {
5007 serde_json::from_slice::<ControlOpProbe>(body)
5008 .map(|probe| MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()))
5009 .unwrap_or(false)
5010}
5011
5012fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5013 debug!(
5014 op = %op,
5015 connection_id = connection_id.get(),
5016 corr,
5017 "control dispatch"
5018 );
5019}
5020
5021fn log_slow_control_dispatch(
5022 dispatch_started_at: Option<StdInstant>,
5023 op: &'static str,
5024 connection_id: ConnectionId,
5025 corr: u64,
5026) {
5027 let Some(dispatch_started_at) = dispatch_started_at else {
5028 return;
5029 };
5030 let elapsed = dispatch_started_at.elapsed();
5031 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5032 warn!(
5033 op = %op,
5034 connection_id = connection_id.get(),
5035 corr,
5036 elapsed_ms = elapsed.as_millis() as u64,
5037 "slow control dispatch"
5038 );
5039 }
5040}
5041
5042fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5043 match request {
5044 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5045 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5046 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5047 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5048 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5049 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5050 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5051 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5052 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5053 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5054 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5055 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5056 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5057 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5058 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5059 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5060 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5061 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5062 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5063 }
5064}
5065
5066fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5067 match request {
5068 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5069 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5070 }
5071}
5072
5073fn parse_client_control_request(
5074 body: &[u8],
5075) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5076 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5077 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5078 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5079 ControlRequestBodyError::InvalidBody
5080 }
5081 Ok(_) => ControlRequestBodyError::UnknownOp,
5082 Err(_) => ControlRequestBodyError::InvalidBody,
5083 };
5084 (err, classification)
5085 })
5086}
5087
5088fn parse_module_control_request_from_module(
5089 body: &[u8],
5090) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5091 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5092 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5093 Ok(probe) if MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5094 ControlRequestBodyError::InvalidBody
5095 }
5096 Ok(_) => ControlRequestBodyError::UnknownOp,
5097 Err(_) => ControlRequestBodyError::InvalidBody,
5098 };
5099 (err, classification)
5100 })
5101}
5102
5103#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5104enum ProviderRoleKind {
5105 ToolProvider,
5106 PipelineStage,
5107 ManagementSurface,
5108 InternalService,
5109}
5110
5111fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5112 match role {
5113 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5114 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5115 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5116 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5117 }
5118}
5119
5120fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5121 roles.iter().map(provider_role_kind).collect()
5122}
5123
5124fn capability_census_trigger(
5128 old: Option<&CapabilityDeclarations>,
5129 new: Option<&CapabilityDeclarations>,
5130) -> bool {
5131 let old_provides = old
5132 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5133 .unwrap_or_default();
5134 let old_denies = old
5135 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5136 .unwrap_or_default();
5137 let new = new.cloned().unwrap_or(CapabilityDeclarations {
5138 provides: Vec::new(),
5139 requires: Vec::new(),
5140 must_never_reach: Vec::new(),
5141 });
5142
5143 new.provides
5144 .iter()
5145 .any(|capability| !old_provides.contains(capability))
5146 || new
5147 .must_never_reach
5148 .iter()
5149 .any(|capability| !old_denies.contains(capability))
5150}
5151
5152fn denied_capability<'a>(
5155 opening_manifest: &'a ModuleManifest,
5156 target_manifest: &ModuleManifest,
5157) -> Option<&'a str> {
5158 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5159 let target_capabilities = target_manifest.capabilities.as_ref()?;
5160 opening_capabilities
5161 .must_never_reach
5162 .iter()
5163 .find(|denied| {
5164 target_capabilities
5165 .provides
5166 .iter()
5167 .any(|provided| provided == *denied)
5168 })
5169 .map(String::as_str)
5170}
5171
5172fn catalog_update_frozen_field_message(
5173 registered: &ModuleManifest,
5174 provides: &[ProviderRole],
5175) -> Option<String> {
5176 let old_has_provides = !registered.provides.is_empty();
5177 let new_has_provides = !provides.is_empty();
5178 if old_has_provides != new_has_provides {
5179 return Some(format!(
5180 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5181 registered.module_id
5182 ));
5183 }
5184
5185 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
5186 return Some(format!(
5187 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5188 registered.module_id
5189 ));
5190 }
5191
5192 let registered_concurrency = manifest_concurrency(registered);
5193 let mut candidate = registered.clone();
5194 candidate.provides = provides.to_vec();
5195 let candidate_concurrency = manifest_concurrency(&candidate);
5196 if candidate_concurrency != registered_concurrency {
5197 return Some(format!(
5198 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5199 registered.module_id, registered_concurrency, candidate_concurrency
5200 ));
5201 }
5202
5203 None
5206}
5207
5208fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5209 manifest.provides.iter().any(is_routable_role)
5210}
5211
5212fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5218 manifest
5219 .provides
5220 .iter()
5221 .find_map(|provider| match provider {
5222 ProviderRole::ToolProvider { concurrency, .. }
5223 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5224 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5225 })
5226 .unwrap_or(Concurrency::ModuleManaged)
5227}
5228
5229fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5236 let has_management_surface = manifest
5237 .provides
5238 .iter()
5239 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5240 if !has_management_surface {
5241 return false;
5242 }
5243 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5244 return false;
5245 };
5246 let Some(provides) = raw
5247 .get("manifest")
5248 .and_then(|manifest| manifest.get("provides"))
5249 .and_then(serde_json::Value::as_array)
5250 else {
5251 return false;
5252 };
5253 provides.iter().any(|role| {
5258 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5259 && role.get("concurrency").is_none()
5260 })
5261}
5262
5263fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5264 if peer_version != PROTOCOL_VERSION {
5265 return Err(format!(
5266 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5267 ));
5268 }
5269 Ok(PROTOCOL_VERSION)
5270}
5271
5272fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5273 Frame::build_with_version(
5274 response_version(frame),
5275 FrameType::Pong,
5276 frame.header.flags,
5277 0,
5278 0,
5279 frame.header.corr,
5280 Vec::new(),
5281 )
5282 .map_err(RouterError::FrameBuild)
5283}
5284
5285fn control_error_frame(
5286 frame: &Frame,
5287 code: &'static str,
5288 message: impl Into<String>,
5289) -> Result<Frame, RouterError> {
5290 control_error_body_frame(
5291 frame,
5292 ErrorBody {
5293 code: code.to_string(),
5294 message: message.into(),
5295 detail: None,
5296 },
5297 )
5298}
5299
5300fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5301 let body = serde_json::to_vec(&error).map_err(|err| {
5302 RouterError::backend(
5303 0,
5304 frame.header.corr,
5305 format!("failed to encode control ERROR: {err}"),
5306 )
5307 })?;
5308
5309 Frame::build_with_version(
5310 response_version(frame),
5311 FrameType::Error,
5312 control_flags(),
5313 0,
5314 0,
5315 frame.header.corr,
5316 body,
5317 )
5318 .map_err(RouterError::FrameBuild)
5319}
5320
5321fn control_response_body_frame<T: Serialize>(
5322 frame: &Frame,
5323 reply: &T,
5324 label: &'static str,
5325) -> Result<Frame, RouterError> {
5326 let body = serde_json::to_vec(reply).map_err(|err| {
5327 RouterError::backend(
5328 0,
5329 frame.header.corr,
5330 format!("failed to encode {label}: {err}"),
5331 )
5332 })?;
5333
5334 Frame::build_with_version(
5335 response_version(frame),
5336 FrameType::Response,
5337 control_flags(),
5338 0,
5339 0,
5340 frame.header.corr,
5341 body,
5342 )
5343 .map_err(RouterError::FrameBuild)
5344}
5345
5346fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5376 match err {
5377 ForwardingError::NoModuleConnection => "target_unavailable",
5378 ForwardingError::ModuleReloading { .. } => "module_reloading",
5379 ForwardingError::ClientRouteChannelExhausted { .. }
5380 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5381 ForwardingError::StaleModuleEndpoint
5382 | ForwardingError::UnknownReservation { .. }
5383 | ForwardingError::ConnectionClosing { .. }
5384 | ForwardingError::ClientEgressClosed { .. }
5385 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5386 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5389 ForwardingError::RelayCorrelationExhausted
5390 | ForwardingError::RouteOpenBuild(_)
5391 | ForwardingError::Poisoned => "forwarding_error",
5392 }
5393}
5394
5395fn response_version(frame: &Frame) -> u8 {
5396 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5397 frame.header.ver
5398 } else {
5399 PROTOCOL_VERSION
5400 }
5401}
5402
5403fn control_flags() -> Flags {
5404 Flags::new(false, Priority::Passive, false)
5405}
5406
5407fn send_goodbye_target_best_effort(
5412 counters: &DaemonCounters,
5413 target: &GoodbyeTarget,
5414 context: &'static str,
5415) {
5416 let Ok(frame) = Frame::build_with_version(
5417 target.negotiated_ver,
5418 FrameType::Goodbye,
5419 control_flags(),
5420 target.channel,
5421 target.epoch,
5422 0,
5423 Vec::new(),
5424 ) else {
5425 return;
5426 };
5427 crate::forwarding::send_module_route_goodbye(
5428 counters,
5429 &target.sink,
5430 frame,
5431 target.module_id.as_deref(),
5432 context,
5433 );
5434}
5435
5436pub(crate) fn send_route_control_pushes(
5437 forwarding: &ForwardingTable,
5438 routes: Vec<EndpointRoute>,
5439 push: ClientControlPush,
5440) {
5441 let body = match serde_json::to_vec(&push) {
5442 Ok(body) => body,
5443 Err(err) => {
5444 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5445 return;
5446 }
5447 };
5448 let mut targets = Vec::new();
5449 for route in routes {
5450 let target = route.goodbye_target;
5451 if let Some(existing) = targets
5452 .iter()
5453 .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5454 {
5455 debug_assert_eq!(
5456 existing.negotiated_ver, target.negotiated_ver,
5457 "one connection cannot negotiate multiple frame versions"
5458 );
5459 continue;
5460 }
5461 targets.push(target);
5462 }
5463 for target in targets {
5464 let frame = match Frame::build_with_version(
5465 target.negotiated_ver,
5466 FrameType::Push,
5467 control_flags(),
5468 0,
5469 0,
5470 0,
5471 body.clone(),
5472 ) {
5473 Ok(frame) => frame,
5474 Err(err) => {
5475 warn!(
5476 route_channel = target.channel,
5477 error = %err,
5478 "failed to build route lifecycle control PUSH frame"
5479 );
5480 continue;
5481 }
5482 };
5483 if let Err(err) = target.sink.try_send(frame) {
5484 if target.close_on_delivery_failure() {
5485 warn!(
5486 target_connection_id = target.connection_id.get(),
5487 route_channel = target.channel,
5488 error = %err,
5489 "route lifecycle control PUSH was not delivered to client; closing target connection"
5490 );
5491 let _ = forwarding.escalate_client_delivery_failure(
5492 target.connection_id,
5493 target.channel,
5494 target.epoch,
5495 CloseReason::new(
5496 "route_lifecycle_push_delivery_failed",
5497 format!(
5498 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
5499 target.channel
5500 ),
5501 ),
5502 crate::forwarding::UndeliveredFrame {
5503 module_id: target.module_id.as_deref(),
5504 sink: &target.sink,
5505 },
5506 );
5507 }
5508 }
5509 }
5510}
5511
5512#[cfg(test)]
5513mod tests {
5514 use std::{
5515 collections::BTreeMap,
5516 fmt,
5517 path::PathBuf,
5518 sync::{Arc, Mutex},
5519 time::Duration,
5520 };
5521 use subc_test_support::TestTempDir;
5522
5523 use serde_json::{json, Value};
5524 use subc_protocol::{
5525 manifest::{
5526 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
5527 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
5528 },
5529 session::HealthStatus,
5530 FrameType,
5531 };
5532
5533 use super::*;
5534 use crate::{
5535 forwarding::{DataRoute, DataRouteState},
5536 registry::ChannelState,
5537 router::FrameSink,
5538 stderr_tail::DEFAULT_MAX_LINE_BYTES,
5539 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
5540 RouteCtx, Router,
5541 };
5542 use tokio::{
5543 sync::mpsc,
5544 time::{sleep, Instant},
5545 };
5546 use tracing::{
5547 field::{Field, Visit},
5548 Event, Subscriber,
5549 };
5550 use tracing_subscriber::{layer::Context, prelude::*, Layer};
5551
5552 fn fake_aft_stub_path() -> PathBuf {
5571 let mut path = std::env::current_exe().expect("current_exe available in tests");
5572 path.pop(); path.pop(); path.push(if cfg!(windows) {
5575 "fake-aft-stub.exe"
5576 } else {
5577 "fake-aft-stub"
5578 });
5579 assert!(
5580 path.exists(),
5581 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
5582 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
5583 path.display()
5584 );
5585 path
5586 }
5587
5588 fn client_retries(code: &str) -> bool {
5593 subc_protocol::error_codes::is_retryable_route_open(code)
5594 }
5595
5596 #[test]
5607 fn retryability_of_forwarding_codes_matches_the_failure() {
5608 let transient = [
5611 ForwardingError::NoModuleConnection,
5612 ForwardingError::ModuleReloading {
5613 module_id: "m".into(),
5614 },
5615 ForwardingError::StaleModuleEndpoint,
5616 ForwardingError::UnknownReservation {
5617 client_channel: 1,
5618 module_channel: 1,
5619 },
5620 ForwardingError::ConnectionClosing {
5621 connection_id: ConnectionId::new(1),
5622 },
5623 ForwardingError::ClientEgressClosed {
5624 connection_id: ConnectionId::new(1),
5625 },
5626 ForwardingError::ModuleEgressUnavailable {
5627 connection_id: ConnectionId::new(1),
5628 },
5629 ];
5630 for err in transient {
5631 let code = forwarding_error_code(&err);
5632 assert!(
5633 client_retries(code),
5634 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
5635 );
5636 }
5637
5638 let permanent = [
5643 ForwardingError::ClientRouteChannelExhausted {
5644 connection_id: ConnectionId::new(1),
5645 },
5646 ForwardingError::ModuleRouteChannelExhausted {
5647 endpoint: ModuleEndpointId {
5648 connection_id: ConnectionId::new(1),
5649 generation: 1,
5650 },
5651 },
5652 ForwardingError::RelayCorrelationExhausted,
5653 ForwardingError::RouteOpenBuild("x".into()),
5654 ForwardingError::Poisoned,
5655 ];
5656 for err in permanent {
5657 let code = forwarding_error_code(&err);
5658 assert!(
5659 !client_retries(code),
5660 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
5661 );
5662 }
5663 }
5664
5665 #[tokio::test]
5693 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
5694 let handler = ControlHandler::default();
5695 let frame =
5696 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
5697
5698 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
5702 assert_eq!(
5703 stamped,
5704 Principal::Direct,
5705 "a caller that proved nothing must not be stamped as a supervised module"
5706 );
5707
5708 let forged = handler
5713 .route_open_principal(
5714 &frame,
5715 Some(ConsumerIdentity {
5716 module_id: "aft".to_string(),
5717 launch_nonce: "not-a-real-nonce".to_string(),
5718 }),
5719 )
5720 .unwrap();
5721 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
5722 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
5723 }
5724
5725 #[test]
5742 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
5743 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
5744 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
5745 let ClientControlRequest::RouteOpen {
5746 consumer_identity, ..
5747 } = parsed
5748 else {
5749 panic!("route.open body must parse as RouteOpen");
5750 };
5751 assert_eq!(
5752 consumer_identity,
5753 Some(ConsumerIdentity {
5754 module_id: "aft".to_string(),
5755 launch_nonce: "n".to_string(),
5756 }),
5757 "the wire field name must reach the value route_open_principal reads"
5758 );
5759 }
5760
5761 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
5762 ModuleManifest::builder(module_id, "0.1.0")
5763 .protocol_ver(protocol_ver)
5764 .provides(vec![ProviderRole::ToolProvider {
5765 tools: vec![Tool {
5766 name: "read".to_string(),
5767 description: None,
5768 execution_mode: ExecutionMode::Pure,
5769 schema: json!({"type": "object"}),
5770 }],
5771 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
5772 concurrency: Concurrency::ModuleManaged,
5773 emits_push: true,
5774 sub_supervises: true,
5775 }])
5776 .build()
5777 }
5778
5779 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
5780 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
5781 }
5782
5783 fn hello_frame_with_control_ops(
5784 module_id: &str,
5785 protocol_ver: u8,
5786 corr: u64,
5787 control_ops: Option<Vec<String>>,
5788 ) -> Frame {
5789 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
5790 }
5791
5792 fn hello_frame_with_nonce(
5793 module_id: &str,
5794 protocol_ver: u8,
5795 corr: u64,
5796 launch_nonce: Option<&str>,
5797 ) -> Frame {
5798 hello_frame_full(
5799 module_id,
5800 protocol_ver,
5801 corr,
5802 None,
5803 launch_nonce.map(ToOwned::to_owned),
5804 )
5805 }
5806
5807 fn hello_frame_full(
5808 module_id: &str,
5809 protocol_ver: u8,
5810 corr: u64,
5811 control_ops: Option<Vec<String>>,
5812 launch_nonce: Option<String>,
5813 ) -> Frame {
5814 let body = serde_json::to_vec(&ModuleHelloBody {
5815 manifest: manifest(module_id, protocol_ver),
5816 protocol_ver,
5817 control_ops,
5818 launch_nonce,
5819 })
5820 .unwrap();
5821 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5822 }
5823
5824 fn non_routable_hello_frame_with_control_ops(
5825 module_id: &str,
5826 corr: u64,
5827 control_ops: Option<Vec<String>>,
5828 ) -> Frame {
5829 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
5830 manifest.provides.clear();
5831 let body = serde_json::to_vec(&ModuleHelloBody {
5832 manifest,
5833 protocol_ver: PROTOCOL_VERSION,
5834 control_ops,
5835 launch_nonce: None,
5836 })
5837 .unwrap();
5838 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5839 }
5840
5841 fn capability_grammar_hello_frame(
5842 capabilities: Value,
5843 runtime_computed: Option<Value>,
5844 corr: u64,
5845 ) -> Frame {
5846 let mut body = serde_json::to_value(ModuleHelloBody {
5847 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
5848 protocol_ver: PROTOCOL_VERSION,
5849 control_ops: None,
5850 launch_nonce: None,
5851 })
5852 .expect("HELLO body serializes");
5853 body["manifest"]["capabilities"] = capabilities;
5854 if let Some(runtime_computed) = runtime_computed {
5855 body["runtime_computed"] = runtime_computed;
5856 }
5857 Frame::build(
5858 FrameType::Hello,
5859 control_flags(),
5860 0,
5861 0,
5862 corr,
5863 serde_json::to_vec(&body).expect("HELLO body reserializes"),
5864 )
5865 .expect("HELLO frame builds")
5866 }
5867
5868 fn channel_request(channel: u16, corr: u64) -> Frame {
5869 Frame::build(
5870 FrameType::Request,
5871 Flags::new(true, Priority::Interactive, false),
5872 channel,
5873 0,
5874 corr,
5875 b"opaque".to_vec(),
5876 )
5877 .unwrap()
5878 }
5879
5880 fn route_ctx(
5881 connection_id: ConnectionId,
5882 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
5883 let (tx, rx) = mpsc::channel(8);
5884 (
5885 RouteCtx {
5886 connection_id,
5887 egress: FrameSink::new(tx),
5888 },
5889 rx,
5890 )
5891 }
5892
5893 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
5894 serde_json::from_slice(&frame.body).unwrap()
5895 }
5896
5897 async fn hello_via_sink(
5902 handler: &ControlHandler,
5903 ctx: &RouteCtx,
5904 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
5905 hello: Frame,
5906 ) -> Frame {
5907 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
5908 assert!(
5909 replies.is_empty(),
5910 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
5911 );
5912 let ack = rx
5913 .try_recv()
5914 .expect("HELLO_ACK is queued on the module sink")
5915 .frame;
5916 assert_eq!(ack.header.ty, FrameType::HelloAck);
5917 ack
5918 }
5919
5920 fn parse_error(frame: &Frame) -> Value {
5921 serde_json::from_slice(&frame.body).unwrap()
5922 }
5923
5924 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
5925 serde_json::from_slice(&frame.body).unwrap()
5926 }
5927
5928 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
5929 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
5930 route_channel,
5931 route_epoch: 0,
5932 kind,
5933 })
5934 .unwrap();
5935 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5936 }
5937
5938 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
5939 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
5940 module_id: module_id.to_string(),
5941 })
5942 .unwrap();
5943 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5944 }
5945
5946 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
5947 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
5948 }
5949
5950 fn route_open_frame_with_consumer_capabilities(
5951 corr: u64,
5952 module_id: &str,
5953 project_root: TestTempDir,
5954 consumer_capabilities: Option<Vec<String>>,
5955 ) -> Frame {
5956 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5957 target: RouteTarget::ToolProvider {
5958 module_id: module_id.to_string(),
5959 },
5960 identity: BindIdentity::new(
5961 project_root.path().to_path_buf(),
5962 "unit".to_string(),
5963 "session".to_string(),
5964 ),
5965 consumer_identity: None,
5966 consumer_capabilities,
5967 admission_facts: None,
5968 })
5969 .unwrap();
5970 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5971 }
5972
5973 fn route_open_frame_with_admission_facts(
5974 corr: u64,
5975 module_id: &str,
5976 project_root: TestTempDir,
5977 consumer_identity: Option<subc_control::ConsumerIdentity>,
5978 facts: Option<Value>,
5979 ) -> Frame {
5980 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5981 target: RouteTarget::ToolProvider {
5982 module_id: module_id.to_string(),
5983 },
5984 identity: BindIdentity::new(
5985 project_root.path().to_path_buf(),
5986 "unit".to_string(),
5987 format!("session-{corr}"),
5988 ),
5989 consumer_identity,
5990 consumer_capabilities: None,
5991 admission_facts: facts,
5992 })
5993 .unwrap();
5994 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5995 }
5996
5997 #[derive(Clone, Default)]
5998 struct EventCapture {
5999 events: Arc<Mutex<Vec<CapturedEvent>>>,
6000 }
6001
6002 #[derive(Clone, Debug)]
6003 struct CapturedEvent {
6004 target: String,
6005 fields: BTreeMap<String, String>,
6006 }
6007
6008 impl EventCapture {
6009 fn events(&self) -> Vec<CapturedEvent> {
6010 self.events.lock().unwrap().clone()
6011 }
6012 }
6013
6014 impl<S> Layer<S> for EventCapture
6015 where
6016 S: Subscriber,
6017 {
6018 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6019 let mut visitor = EventFieldVisitor::default();
6020 event.record(&mut visitor);
6021 self.events.lock().unwrap().push(CapturedEvent {
6022 target: event.metadata().target().to_string(),
6023 fields: visitor.fields,
6024 });
6025 }
6026 }
6027
6028 #[derive(Default)]
6029 struct EventFieldVisitor {
6030 fields: BTreeMap<String, String>,
6031 }
6032
6033 impl Visit for EventFieldVisitor {
6034 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6035 self.fields
6036 .insert(field.name().to_string(), format!("{value:?}"));
6037 }
6038 }
6039
6040 fn health_response(corr: u64, status: HealthStatus) -> Frame {
6041 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6042 status,
6043 detail: Some("warming".to_string()),
6044 metrics: Some(json!({"queue_depth": 3})),
6045 })
6046 .unwrap();
6047 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6048 }
6049
6050 fn route_bind_ack(corr: u64) -> Frame {
6051 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6052 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6053 }
6054
6055 fn unique_project_root(label: &str) -> TestTempDir {
6056 TestTempDir::new(label)
6057 }
6058
6059 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6060 match parse_route_poll(frame) {
6061 ClientControlResponse::RoutePoll {
6062 status: None,
6063 live: Some(live),
6064 ..
6065 } => assert_eq!(live, expected_live),
6066 other => panic!("unexpected route.poll response: {other:?}"),
6067 }
6068 }
6069
6070 fn bind_liveness_route(
6071 registry: &Registry,
6072 forwarding: &ForwardingTable,
6073 module_id: &str,
6074 ) -> (RouteCtx, u16, u32) {
6075 let module_connection = ConnectionId::new(101);
6076 let client_connection = ConnectionId::new(202);
6077 let registration = registry
6078 .register_with_control_ops(
6079 manifest(module_id, PROTOCOL_VERSION),
6080 PROTOCOL_VERSION,
6081 module_connection,
6082 module_baseline_control_ops(),
6083 )
6084 .unwrap();
6085 let (module_tx, _module_rx) = mpsc::channel(8);
6086 let endpoint = forwarding
6087 .register_module_connection(
6088 module_connection,
6089 module_id.to_string(),
6090 PROTOCOL_VERSION,
6091 manifest_concurrency(®istration.manifest),
6092 FrameSink::new(module_tx),
6093 )
6094 .unwrap();
6095 let (client_ctx, _client_rx) = route_ctx(client_connection);
6096 let pending = forwarding
6097 .begin_route_bind_relay_for_test(
6098 client_connection,
6099 client_ctx.egress.clone(),
6100 1,
6101 module_id,
6102 )
6103 .unwrap();
6104 assert_eq!(pending.endpoint, endpoint);
6105 let route_channel = pending.client_channel;
6106 let route_epoch = pending.client_epoch;
6107 forwarding
6108 .complete_pending_relay(
6109 module_connection,
6110 pending.corr,
6111 RouteBindRelayOutcome::Accepted,
6112 )
6113 .unwrap();
6114 (client_ctx, route_channel, route_epoch)
6115 }
6116
6117 struct FakeProcessLiveness {
6118 live: Option<bool>,
6119 }
6120
6121 impl ModuleProcessLiveness for FakeProcessLiveness {
6122 fn process_live(&self, _module_id: &str) -> Option<bool> {
6123 self.live
6124 }
6125 }
6126
6127 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6128 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6129 {
6130 let registry = Arc::new(Registry::default());
6131 let supervisor_handle = SupervisorHandle::new();
6132 let supervisor = Supervisor::new(
6133 Arc::clone(®istry),
6134 RestartPolicy::new(1, Duration::from_millis(10)),
6135 )
6136 .with_handle(supervisor_handle.clone());
6137 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6138 let module = supervisor
6139 .spawn(ModuleSpec {
6140 module_id: "stderr-tail-wire".to_string(),
6141 program: fake_aft_stub_path(),
6142 args: Vec::new(),
6143 env: vec![
6144 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6145 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6146 ],
6147 reserved: false,
6148 reserved_prefixes: Vec::new(),
6149 protocol: ModuleProtocol::Subc,
6150 overlap: Default::default(),
6151 })
6152 .unwrap();
6153
6154 let deadline = Instant::now() + Duration::from_secs(5);
6155 loop {
6156 let tail = module.stderr_tail(None, None);
6157 if tail
6158 .entries
6159 .iter()
6160 .any(|entry| matches!(entry, TailEntry::ProcessStart))
6161 && tail.entries.iter().any(|entry| {
6162 matches!(
6163 entry,
6164 TailEntry::Line {
6165 truncated: true,
6166 ..
6167 }
6168 )
6169 })
6170 {
6171 break;
6172 }
6173 assert!(
6174 Instant::now() < deadline,
6175 "module did not produce a truncated line and restart boundary: {tail:?}"
6176 );
6177 sleep(Duration::from_millis(10)).await;
6178 }
6179
6180 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6181 let request = ClientControlRequest::SupervisorStderrTail {
6182 module_id: "stderr-tail-wire".to_string(),
6183 max_lines: None,
6184 max_bytes: None,
6185 };
6186 let frame = Frame::build(
6187 FrameType::Request,
6188 control_flags(),
6189 0,
6190 0,
6191 1,
6192 serde_json::to_vec(&request).unwrap(),
6193 )
6194 .unwrap();
6195 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6196 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6197 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6198 serde_json::from_slice(&responses[0].body).unwrap()
6199 else {
6200 panic!("expected supervisor.stderr_tail response");
6201 };
6202
6203 assert!(
6204 tail.entries
6205 .iter()
6206 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6207 "the control response lost the restart boundary"
6208 );
6209 let Some(StderrTailEntry::Line { text, truncated }) = tail.entries.iter().find(|entry| {
6210 matches!(
6211 entry,
6212 StderrTailEntry::Line {
6213 truncated: true,
6214 ..
6215 }
6216 )
6217 }) else {
6218 panic!("the control response lost the truncated line");
6219 };
6220 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6221 assert!(*truncated);
6222 }
6223
6224 #[tokio::test(flavor = "current_thread")]
6229 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6230 let dir = TestTempDir::new("terminals-off-worker");
6231 let journal_path = dir.join("terminals.jsonl");
6232 let registry = Arc::new(Registry::default());
6233 let supervisor_handle = SupervisorHandle::new();
6234 let supervisor =
6235 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6236 .with_handle(supervisor_handle.clone())
6237 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6238 let module = supervisor
6239 .spawn(ModuleSpec {
6240 module_id: "terminal-off-worker".to_string(),
6241 program: fake_aft_stub_path(),
6242 args: Vec::new(),
6243 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6244 reserved: false,
6245 reserved_prefixes: Vec::new(),
6246 protocol: ModuleProtocol::Subc,
6247 overlap: Default::default(),
6248 })
6249 .unwrap();
6250 let deadline = Instant::now() + Duration::from_secs(5);
6251 while module.terminal_history().entries.len() != 2 {
6252 assert!(Instant::now() < deadline, "module did not record two exits");
6253 sleep(Duration::from_millis(10)).await;
6254 }
6255
6256 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6257 let handler =
6258 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
6259 let frame = Frame::build(
6260 FrameType::Request,
6261 control_flags(),
6262 0,
6263 0,
6264 1,
6265 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6266 module_id: "terminal-off-worker".to_string(),
6267 })
6268 .unwrap(),
6269 )
6270 .unwrap();
6271 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6272 let spawned_at = std::time::Instant::now();
6273 let read = tokio::spawn({
6274 let handler = Arc::clone(&handler);
6275 async move { handler.handle_control_frame(&ctx, frame).await }
6276 });
6277 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6280 .await
6281 .unwrap()
6282 .expect("the history read reached its pause");
6283 let elapsed = spawned_at.elapsed();
6284 assert!(
6285 elapsed < Duration::from_secs(2) && !read.is_finished(),
6286 "this task could not run while the history read was paused \
6287 (resumed after {elapsed:?}, read finished: {})",
6288 read.is_finished()
6289 );
6290
6291 drop(release);
6292 let responses = read.await.unwrap().unwrap();
6293 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6294 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6295 panic!("expected supervisor.terminals response");
6296 };
6297 assert_eq!(terminals.entries.len(), 2);
6298 assert_eq!(terminals.journal_skipped_lines, 0);
6299 assert_eq!(terminals.journal_read_errors, 0);
6300 }
6301
6302 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6303 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6304 let registry = Arc::new(Registry::default());
6305 let supervisor_handle = SupervisorHandle::new();
6306 let supervisor =
6307 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6308 .with_handle(supervisor_handle.clone());
6309 let module = supervisor
6310 .spawn(ModuleSpec {
6311 module_id: "terminal-golden".to_string(),
6312 program: fake_aft_stub_path(),
6313 args: Vec::new(),
6314 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6315 reserved: false,
6316 reserved_prefixes: Vec::new(),
6317 protocol: ModuleProtocol::Subc,
6318 overlap: Default::default(),
6319 })
6320 .unwrap();
6321
6322 let deadline = Instant::now() + Duration::from_secs(5);
6323 while module.terminal_history().entries.len() != 2 {
6324 assert!(
6325 Instant::now() < deadline,
6326 "module did not retain two terminal exits: {:?}",
6327 module.terminal_history()
6328 );
6329 sleep(Duration::from_millis(10)).await;
6330 }
6331
6332 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6333 let request = ClientControlRequest::SupervisorTerminals {
6334 module_id: "terminal-golden".to_string(),
6335 };
6336 let frame = Frame::build(
6337 FrameType::Request,
6338 control_flags(),
6339 0,
6340 0,
6341 1,
6342 serde_json::to_vec(&request).unwrap(),
6343 )
6344 .unwrap();
6345 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6346 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6347 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6348 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6349 panic!("expected supervisor.terminals response");
6350 };
6351 assert_eq!(terminals.entries.len(), 2);
6352 assert_eq!(terminals.dropped, 0);
6353
6354 let mut rendered = serde_json::to_value(response).unwrap();
6355 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6358 for (index, entry) in rendered["entries"]
6359 .as_array_mut()
6360 .expect("terminal response entries array")
6361 .iter_mut()
6362 .enumerate()
6363 {
6364 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6365 }
6366
6367 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6368 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6369 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6370 if std::env::var_os("UPDATE_GOLDEN").is_some() {
6371 std::fs::write(&golden_path, &serialized).unwrap();
6372 }
6373 let expected: Value =
6374 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6375 assert_eq!(rendered, expected);
6376 }
6377
6378 #[test]
6379 fn hello_registers_manifest_and_returns_ack() {
6380 let registry = Arc::new(Registry::default());
6381 let handler = ControlHandler::new(Arc::clone(®istry));
6382 let conn = ConnectionId::new(1);
6383
6384 let responses = handler
6385 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6386 .unwrap();
6387
6388 assert_eq!(responses.len(), 1);
6389 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6390 assert_eq!(responses[0].header.channel, 0);
6391 assert_eq!(responses[0].header.corr, 7);
6392 let ack = parse_ack(&responses[0]);
6393 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6394 assert!(ack
6395 .subc_capabilities
6396 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6397 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6398 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6399 assert!(ack
6400 .subc_ops
6401 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6402 assert!(ack
6403 .subc_ops
6404 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6405
6406 let registration = registry.get_module("aft").unwrap().unwrap();
6407 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6408 assert_eq!(registration.state, ChannelState::Active);
6409 assert_eq!(registration.connection_id, conn);
6410 assert_eq!(registration.control_ops, module_baseline_control_ops());
6411 }
6412
6413 #[test]
6414 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
6415 let invalid_identifiers = [
6416 ("case_change", "credentials-Provider/v1"),
6417 ("leading_zero", "credentials-provider/v01"),
6418 ("trailing_hyphen", "credentials-provider-/v1"),
6419 ("consecutive_hyphens", "credentials--provider/v1"),
6420 ("uppercase", "Credentials-provider/v1"),
6421 ("missing_v", "credentials-provider/1"),
6422 ("whitespace", "credentials provider/v1"),
6423 ("zero_version", "credentials-provider/v0"),
6424 ("out_of_range_version", "credentials-provider/v4294967296"),
6425 (
6426 "overlength_name",
6427 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
6428 ),
6429 ];
6430 let mut cases = invalid_identifiers
6431 .into_iter()
6432 .map(|(name, identifier)| {
6433 (
6434 format!("identifier_{name}"),
6435 "capabilities.provides[0]".to_string(),
6436 identifier.to_string(),
6437 json!({ "provides": [identifier] }),
6438 None,
6439 )
6440 })
6441 .collect::<Vec<_>>();
6442 cases.extend([
6443 (
6444 "unknown_need".to_string(),
6445 "capabilities.requires[0].need".to_string(),
6446 "deferred".to_string(),
6447 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
6448 None,
6449 ),
6450 (
6451 "duplicate_provides".to_string(),
6452 "capabilities.provides[1]".to_string(),
6453 "credentials-provider/v1".to_string(),
6454 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
6455 None,
6456 ),
6457 (
6458 "duplicate_must_never_reach".to_string(),
6459 "capabilities.must_never_reach[1]".to_string(),
6460 "credentials-provider/v1".to_string(),
6461 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
6462 None,
6463 ),
6464 (
6465 "duplicate_requires_same_need".to_string(),
6466 "capabilities.requires[1]".to_string(),
6467 "credentials-provider/v1".to_string(),
6468 json!({ "requires": [
6469 { "capability": "credentials-provider/v1", "need": "required" },
6470 { "capability": "credentials-provider/v1", "need": "required" }
6471 ] }),
6472 None,
6473 ),
6474 (
6475 "duplicate_requires_conflicting_need".to_string(),
6476 "capabilities.requires[1]".to_string(),
6477 "credentials-provider/v1".to_string(),
6478 json!({ "requires": [
6479 { "capability": "credentials-provider/v1", "need": "required" },
6480 { "capability": "credentials-provider/v1", "need": "optional" }
6481 ] }),
6482 None,
6483 ),
6484 (
6485 "capabilities_root_pointer".to_string(),
6486 "runtime_computed[0]".to_string(),
6487 "/capabilities".to_string(),
6488 json!({}),
6489 Some(json!(["/capabilities"])),
6490 ),
6491 (
6492 "capabilities_descendant_pointer".to_string(),
6493 "runtime_computed[0]".to_string(),
6494 "/capabilities/provides".to_string(),
6495 json!({}),
6496 Some(json!(["/capabilities/provides"])),
6497 ),
6498 (
6499 "malformed_pointer_without_leading_slash".to_string(),
6500 "runtime_computed[0]".to_string(),
6501 "capabilities".to_string(),
6502 json!({}),
6503 Some(json!(["capabilities"])),
6504 ),
6505 (
6506 "malformed_pointer_escape".to_string(),
6507 "runtime_computed[0]".to_string(),
6508 "/roles/~2/tools".to_string(),
6509 json!({}),
6510 Some(json!(["/roles/~2/tools"])),
6511 ),
6512 (
6513 "unknown_capabilities_field".to_string(),
6514 "capabilities.future".to_string(),
6515 "<array>".to_string(),
6516 json!({ "future": [] }),
6517 None,
6518 ),
6519 ]);
6520
6521 for (index, (name, field, value, capabilities, runtime_computed)) in
6522 cases.into_iter().enumerate()
6523 {
6524 let registry = Arc::new(Registry::default());
6525 let handler = ControlHandler::new(Arc::clone(®istry));
6526 let response = handler
6527 .handle_control(
6528 ConnectionId::new((index + 1) as u64),
6529 capability_grammar_hello_frame(
6530 capabilities,
6531 runtime_computed,
6532 index as u64 + 1,
6533 ),
6534 )
6535 .expect("invalid HELLO returns a refusal");
6536
6537 assert_eq!(response.len(), 1, "{name} must emit one refusal");
6538 let error = parse_error(&response[0]);
6539 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
6540 let message = error["message"]
6541 .as_str()
6542 .expect("error message is a string");
6543 assert!(
6544 message.contains(&field),
6545 "{name}: field missing from {message}"
6546 );
6547 assert!(
6548 message.contains(&value),
6549 "{name}: value missing from {message}"
6550 );
6551 assert_eq!(
6552 registry
6553 .active_registration_count()
6554 .expect("registry reads"),
6555 0,
6556 "{name}: refused HELLO must not create a catalog entry"
6557 );
6558 }
6559 }
6560
6561 #[test]
6562 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
6563 let registry = Arc::new(Registry::default());
6564 let handler = ControlHandler::new(Arc::clone(®istry));
6565 let capabilities = json!({
6566 "provides": ["credentials-provider/v1"],
6567 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
6568 "must_never_reach": ["federation-transport/v1"]
6569 });
6570 let response = handler
6571 .handle_control(
6572 ConnectionId::new(99),
6573 capability_grammar_hello_frame(
6574 capabilities.clone(),
6575 Some(json!(["/roles/0/tools"])),
6576 99,
6577 ),
6578 )
6579 .expect("valid HELLO registers");
6580 assert_eq!(response[0].header.ty, FrameType::HelloAck);
6581
6582 let request = Frame::build(
6583 FrameType::Request,
6584 control_flags(),
6585 0,
6586 0,
6587 100,
6588 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6589 .expect("catalog request serializes"),
6590 )
6591 .expect("catalog request frame builds");
6592 let response = handler
6593 .handle_catalog_list(request, None)
6594 .expect("catalog list succeeds");
6595 let ClientControlResponse::CatalogList { modules, .. } =
6596 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
6597 else {
6598 panic!("catalog request must return catalog.list");
6599 };
6600 assert_eq!(modules.len(), 1);
6601 assert_eq!(
6602 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
6603 capabilities
6604 );
6605 }
6606
6607 #[test]
6608 fn catalog_list_mirrors_management_operation_description() {
6609 let registry = Arc::new(Registry::default());
6610 let handler = ControlHandler::new(Arc::clone(®istry));
6611 let description = "List managed records and return their identifiers and metadata.";
6612 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
6613 manifest.provides = vec![ProviderRole::ManagementSurface {
6614 operations: vec![ManagementOperation {
6615 name: "records.list".to_string(),
6616 kind: ManagementOperationKind::Query,
6617 description: Some(description.to_string()),
6618 }],
6619 config_schema: json!({"type": "object"}),
6620 observability: vec![ObservabilitySurface {
6621 name: "records.stats".to_string(),
6622 kind: ObservabilityKind::Snapshot,
6623 }],
6624 identity_scope: vec![IdentityScope::Project],
6625 concurrency: Concurrency::ModuleManaged,
6626 }];
6627 registry
6628 .register_with_control_ops(
6629 manifest,
6630 PROTOCOL_VERSION,
6631 ConnectionId::new(99),
6632 Vec::new(),
6633 )
6634 .expect("described management manifest registers");
6635
6636 let request = Frame::build(
6637 FrameType::Request,
6638 control_flags(),
6639 0,
6640 0,
6641 100,
6642 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6643 .expect("catalog request serializes"),
6644 )
6645 .expect("catalog request frame builds");
6646 let response = handler
6647 .handle_catalog_list(request, None)
6648 .expect("catalog list succeeds");
6649 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6650 assert_eq!(
6651 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
6652 "catalog.list must preserve the declared operation description verbatim"
6653 );
6654 }
6655
6656 #[test]
6657 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
6658 let registry = Arc::new(Registry::default());
6659 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
6660 [("vault".to_string(), true), ("squatter".to_string(), true)],
6661 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
6662 );
6663 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
6664 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6665 provides: vec!["credentials-provider/v1".to_string()],
6666 requires: Vec::new(),
6667 must_never_reach: Vec::new(),
6668 });
6669 let frame = Frame::build(
6670 FrameType::Hello,
6671 control_flags(),
6672 0,
6673 0,
6674 77,
6675 serde_json::to_vec(&ModuleHelloBody {
6676 manifest: squatter,
6677 protocol_ver: PROTOCOL_VERSION,
6678 control_ops: None,
6679 launch_nonce: None,
6680 })
6681 .expect("HELLO serializes"),
6682 )
6683 .expect("HELLO frame builds");
6684 let response = handler
6685 .handle_control(ConnectionId::new(77), frame)
6686 .expect("reserved claim receives a typed refusal");
6687 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
6688 assert_eq!(
6689 registry
6690 .active_registration_count()
6691 .expect("registry reads"),
6692 0,
6693 "a reserved capability refusal must not leave a catalog entry"
6694 );
6695 }
6696
6697 #[test]
6698 fn server_describe_surfaces_required_capability_verdict_fields() {
6699 let registry = Arc::new(Registry::default());
6700 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
6701 [
6702 ("consumer".to_string(), true),
6703 ("provider".to_string(), false),
6704 ],
6705 BTreeMap::new(),
6706 );
6707 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
6708 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6709 provides: Vec::new(),
6710 requires: vec![subc_protocol::manifest::CapabilityRequirement {
6711 capability: "credentials-provider/v1".to_string(),
6712 need: subc_protocol::manifest::CapabilityNeed::Required,
6713 }],
6714 must_never_reach: Vec::new(),
6715 });
6716 let hello = Frame::build(
6717 FrameType::Hello,
6718 control_flags(),
6719 0,
6720 0,
6721 78,
6722 serde_json::to_vec(&ModuleHelloBody {
6723 manifest: consumer,
6724 protocol_ver: PROTOCOL_VERSION,
6725 control_ops: None,
6726 launch_nonce: None,
6727 })
6728 .expect("HELLO serializes"),
6729 )
6730 .expect("HELLO frame builds");
6731 handler
6732 .handle_control(ConnectionId::new(78), hello)
6733 .expect("consumer registers");
6734 let describe = Frame::build(
6735 FrameType::Request,
6736 control_flags(),
6737 0,
6738 0,
6739 79,
6740 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
6741 .expect("request serializes"),
6742 )
6743 .expect("describe frame builds");
6744 let response = handler
6745 .handle_server_describe(describe)
6746 .expect("server.describe succeeds");
6747 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
6748 let requirement = &rendered["capability_requirements"][0];
6749 assert_eq!(requirement["consumer"], "consumer");
6750 assert_eq!(requirement["verdict"], "never_provided");
6751 assert_eq!(requirement["episode_seq"], 1);
6752 assert_eq!(requirement["config_satisfiable"], false);
6753 assert_eq!(requirement["runtime_available"], false);
6754 assert!(requirement["detail"]
6755 .as_str()
6756 .expect("detail string")
6757 .contains("credentials-provider/v1"));
6758 }
6759
6760 #[test]
6761 fn catalog_list_omits_capabilities_for_legacy_manifest() {
6762 let registry = Arc::new(Registry::default());
6763 let handler = ControlHandler::new(Arc::clone(®istry));
6764 let hello = handler
6765 .handle_control(
6766 ConnectionId::new(101),
6767 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
6768 )
6769 .expect("legacy HELLO registers");
6770 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
6771
6772 let request = Frame::build(
6773 FrameType::Request,
6774 control_flags(),
6775 0,
6776 0,
6777 102,
6778 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6779 .expect("catalog request serializes"),
6780 )
6781 .expect("catalog request frame builds");
6782 let response = handler
6783 .handle_catalog_list(request, None)
6784 .expect("catalog list succeeds");
6785 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6786 assert!(
6787 body["modules"][0].get("capabilities").is_none(),
6788 "legacy manifest must retain an absent capabilities field on catalog.list"
6789 );
6790 }
6791
6792 #[test]
6793 fn hello_ack_omits_storage_when_no_storage_config() {
6794 let registry = Arc::new(Registry::default());
6795 let handler = ControlHandler::new(Arc::clone(®istry));
6796 let responses = handler
6797 .handle_control(
6798 ConnectionId::new(1),
6799 hello_frame("aft", PROTOCOL_VERSION, 7),
6800 )
6801 .unwrap();
6802 let ack = parse_ack(&responses[0]);
6803 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
6804 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
6805 }
6806
6807 #[tokio::test]
6808 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
6809 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
6810 let registry = Arc::new(Registry::default());
6811 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
6812 let responses = handler
6813 .handle_control(
6814 ConnectionId::new(1),
6815 hello_frame("aft", PROTOCOL_VERSION, 7),
6816 )
6817 .unwrap();
6818 let ack = parse_ack(&responses[0]);
6819 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
6820
6821 let described = handler
6822 .handle_control_frame(
6823 &route_ctx(ConnectionId::new(2)).0,
6824 Frame::build(
6825 FrameType::Request,
6826 control_flags(),
6827 0,
6828 0,
6829 9,
6830 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
6831 )
6832 .unwrap(),
6833 )
6834 .await
6835 .unwrap();
6836 let ClientControlResponse::ServerDescribe { machine_id, .. } =
6837 serde_json::from_slice(&described[0].body).unwrap()
6838 else {
6839 panic!("server.describe answered with another shape");
6840 };
6841 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
6842 }
6843
6844 #[test]
6845 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
6846 let registry = Arc::new(Registry::default());
6849 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
6850 crate::daemon_config::StorageConfig::Sqlite {
6851 data_home: std::path::PathBuf::from("/data"),
6852 },
6853 ));
6854
6855 let responses = handler
6856 .handle_control(
6857 ConnectionId::new(1),
6858 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
6859 )
6860 .unwrap();
6861 let ack = parse_ack(&responses[0]);
6862 assert_eq!(
6863 ack.storage,
6864 Some(serde_json::json!({
6865 "module_id": "alfonso-routing",
6866 "storage_namespace": "default",
6867 "isolation": { "kind": "module" },
6868 "backend": {
6869 "backend": "sqlite",
6870 "path": "/data/cortexkit/alfonso-routing/store.db"
6871 }
6872 })),
6873 "the delivered descriptor is the module's own sqlite store path"
6874 );
6875 }
6876
6877 #[test]
6878 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
6879 let registry = Arc::new(Registry::default());
6880 let handler = ControlHandler::new(Arc::clone(®istry));
6881 let conn = ConnectionId::new(1);
6882 let responses = handler
6883 .handle_control(
6884 conn,
6885 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6886 )
6887 .unwrap();
6888 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6889 let registration = registry.get_module("aft").unwrap().unwrap();
6890 assert_eq!(registration.control_ops, module_baseline_control_ops());
6891
6892 let frame =
6893 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
6894 assert!(handler
6895 .guard_module_control_op(&frame, "aft", "route.bind")
6896 .unwrap()
6897 .is_none());
6898 let error = handler
6899 .guard_module_control_op(&frame, "aft", "test.synthetic")
6900 .unwrap()
6901 .expect("synthetic ungranted op should be rejected");
6902 assert_eq!(error.header.ty, FrameType::Error);
6903 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
6904 }
6905
6906 #[test]
6907 fn hello_control_ops_some_adds_optional_grants() {
6908 let registry = Arc::new(Registry::default());
6909 let handler = ControlHandler::new(Arc::clone(®istry));
6910 handler
6911 .handle_control(
6912 ConnectionId::new(1),
6913 hello_frame_with_control_ops(
6914 "aft",
6915 PROTOCOL_VERSION,
6916 7,
6917 Some(vec![
6918 "future.synthetic".to_string(),
6919 "route.bind".to_string(),
6920 ]),
6921 ),
6922 )
6923 .unwrap();
6924 let registration = registry.get_module("aft").unwrap().unwrap();
6925 assert_eq!(
6926 registration.control_ops,
6927 vec![
6928 "route.bind".to_string(),
6929 "route.status".to_string(),
6930 "future.synthetic".to_string(),
6931 ]
6932 );
6933 let frame =
6934 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
6935 assert!(handler
6936 .guard_module_control_op(&frame, "aft", "future.synthetic")
6937 .unwrap()
6938 .is_none());
6939 }
6940
6941 #[tokio::test]
6942 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
6943 let registry = Arc::new(Registry::default());
6944 let forwarding = Arc::new(ForwardingTable::default());
6945 let handler =
6946 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
6947 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
6948 hello_via_sink(
6949 &handler,
6950 &module_ctx,
6951 &mut module_rx,
6952 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6953 )
6954 .await;
6955
6956 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
6957 let responses = handler
6958 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
6959 .await
6960 .unwrap();
6961 assert_eq!(responses.len(), 1);
6962 assert_eq!(responses[0].header.ty, FrameType::Error);
6963 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
6964 assert!(module_rx.try_recv().is_err());
6965 }
6966
6967 #[tokio::test]
6968 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
6969 let registry = Arc::new(Registry::default());
6970 let forwarding = Arc::new(ForwardingTable::default());
6971 let handler =
6972 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
6973 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
6974 hello_via_sink(
6975 &handler,
6976 &module_ctx,
6977 &mut module_rx,
6978 hello_frame_with_control_ops(
6979 "aft",
6980 PROTOCOL_VERSION,
6981 7,
6982 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
6983 ),
6984 )
6985 .await;
6986
6987 let project_root = unique_project_root("demux");
6988 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
6989 let route_handler = handler.clone();
6990 let route_task = tokio::spawn(async move {
6991 route_handler
6992 .handle_control_frame(
6993 &route_client_ctx,
6994 route_open_frame(100, "aft", project_root),
6995 )
6996 .await
6997 .unwrap()
6998 });
6999 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7000 .await
7001 .unwrap()
7002 .unwrap();
7003 assert!(matches!(
7004 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
7005 ModuleControlRequest::RouteBind { .. }
7006 ));
7007
7008 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
7009 let health_handler = handler.clone();
7010 let health_task = tokio::spawn(async move {
7011 health_handler
7012 .handle_control_frame(
7013 &health_client_ctx,
7014 supervisor_health_probe_frame(101, "aft"),
7015 )
7016 .await
7017 .unwrap()
7018 });
7019 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7020 .await
7021 .unwrap()
7022 .unwrap();
7023 assert_eq!(
7024 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
7025 ModuleControlRequest::HealthCheck {}
7026 );
7027
7028 handler
7029 .handle_control_frame(
7030 &module_ctx,
7031 health_response(health_frame.header.corr, HealthStatus::Degraded),
7032 )
7033 .await
7034 .unwrap();
7035 let health_response = health_task.await.unwrap();
7036 assert_eq!(health_response.len(), 1);
7037 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
7038 ClientControlResponse::SupervisorHealthProbe {
7039 module_id,
7040 status,
7041 detail,
7042 metrics,
7043 } => {
7044 assert_eq!(module_id, "aft");
7045 assert_eq!(status, HealthStatus::Degraded);
7046 assert_eq!(detail.as_deref(), Some("warming"));
7047 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
7048 }
7049 other => panic!("unexpected health response: {other:?}"),
7050 }
7051
7052 handler
7053 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7054 .await
7055 .unwrap();
7056 let route_response = route_task.await.unwrap();
7057 assert!(route_response.is_empty());
7058 let published = route_client_rx.recv().await.unwrap();
7059 assert!(matches!(
7060 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7061 ClientControlResponse::RouteOpen { .. }
7062 ));
7063 }
7064
7065 async fn relay_route_open(
7070 handler: &ControlHandler,
7071 client_connection: ConnectionId,
7072 client_egress: &FrameSink,
7073 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7074 corr: u64,
7075 module_id: &str,
7076 project_root_label: &str,
7077 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7078 let ctx = RouteCtx {
7079 connection_id: client_connection,
7080 egress: client_egress.clone(),
7081 };
7082 let handler = handler.clone();
7083 let project_root = unique_project_root(project_root_label);
7084 let module_id = module_id.to_string();
7085 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7086 let task = tokio::spawn(async move {
7087 let _guard = tracing::dispatcher::set_default(&dispatch);
7088 handler
7089 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
7090 .await
7091 .unwrap()
7092 });
7093 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
7094 .await
7095 .expect("module receives the relayed route.bind")
7096 .expect("module egress is open");
7097 (task, bind.frame)
7098 }
7099
7100 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
7101 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7102 ModuleControlRequest::RouteBind {
7103 route_channel,
7104 epoch,
7105 ..
7106 } => (route_channel, epoch),
7107 other => panic!("expected a route.bind request, got {other:?}"),
7108 }
7109 }
7110
7111 fn published_route(frame: &Frame) -> (u16, u32) {
7112 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7113 ClientControlResponse::RouteOpen {
7114 route_channel,
7115 route_epoch,
7116 } => (route_channel, route_epoch),
7117 other => panic!("expected a route.open response, got {other:?}"),
7118 }
7119 }
7120
7121 #[tokio::test]
7141 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7142 let registry = Arc::new(Registry::default());
7143 let forwarding = Arc::new(ForwardingTable::default());
7144 let handler =
7145 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7146
7147 let module_connection = ConnectionId::new(30);
7148 let (module_ctx, mut module_rx) = route_ctx(module_connection);
7149 hello_via_sink(
7150 &handler,
7151 &module_ctx,
7152 &mut module_rx,
7153 hello_frame("aft", PROTOCOL_VERSION, 7),
7154 )
7155 .await;
7156
7157 let dying_client = ConnectionId::new(31);
7158 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7159
7160 let (first_task, first_bind) = relay_route_open(
7163 &handler,
7164 dying_client,
7165 &dying_ctx.egress,
7166 &mut module_rx,
7167 100,
7168 "aft",
7169 "closing-first",
7170 )
7171 .await;
7172 handler
7173 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7174 .await
7175 .unwrap();
7176 assert!(first_task.await.unwrap().is_empty());
7177 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7178
7179 let (second_task, second_bind) = relay_route_open(
7181 &handler,
7182 dying_client,
7183 &dying_ctx.egress,
7184 &mut module_rx,
7185 101,
7186 "aft",
7187 "closing-second",
7188 )
7189 .await;
7190 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7191
7192 assert!(forwarding
7195 .escalate_client_delivery_failure(
7196 dying_client,
7197 first_channel,
7198 first_epoch,
7199 CloseReason::new(
7200 "module_to_client_delivery_failed",
7201 "client egress refused a module frame",
7202 ),
7203 crate::forwarding::UndeliveredFrame {
7204 module_id: None,
7205 sink: &dying_ctx.egress,
7206 },
7207 )
7208 .unwrap());
7209 assert!(!dying_ctx.egress.is_closed());
7210
7211 let ack = handler
7213 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7214 .await;
7215 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7216 if module_loop_error.is_some() {
7217 handler.cleanup_connection(module_connection).unwrap();
7221 }
7222 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7226 .await
7227 .ok()
7228 .flatten();
7229
7230 assert!(
7232 registry
7233 .get_module_by_connection(module_connection)
7234 .unwrap()
7235 .is_some(),
7236 "one client's closing connection ended the shared module connection: \
7237 {module_loop_error:?}"
7238 );
7239 let cotenant = ConnectionId::new(32);
7241 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7242 let (cotenant_task, cotenant_bind) = relay_route_open(
7243 &handler,
7244 cotenant,
7245 &cotenant_ctx.egress,
7246 &mut module_rx,
7247 102,
7248 "aft",
7249 "closing-cotenant",
7250 )
7251 .await;
7252 handler
7253 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7254 .await
7255 .unwrap();
7256 assert!(cotenant_task.await.unwrap().is_empty());
7257 let (cotenant_channel, cotenant_epoch) =
7258 published_route(&cotenant_rx.recv().await.unwrap());
7259 assert!(matches!(
7260 forwarding
7261 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7262 .unwrap(),
7263 DataRoute::Client(DataRouteState::Bound(_))
7264 ));
7265
7266 let goodbye = post_ack_module_frame
7269 .expect("module receives a GOODBYE for the abandoned route channel");
7270 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7271 assert_eq!(goodbye.header.channel, abandoned_channel);
7272 assert_eq!(goodbye.header.epoch, abandoned_epoch);
7273
7274 assert!(dying_rx.try_recv().is_err());
7278 let second_response = second_task.await.unwrap();
7279 assert_eq!(second_response.len(), 1);
7280 assert_eq!(
7281 parse_error(&second_response[0])["code"],
7282 "target_unavailable"
7283 );
7284 }
7285
7286 #[test]
7293 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7294 let handler = ControlHandler::default();
7295 let module_connection = ConnectionId::new(30);
7296 let client_connection = ConnectionId::new(31);
7297
7298 handler
7299 .refuse_to_end_module_connection_for_a_client(
7300 module_connection,
7301 77,
7302 ForwardingError::ConnectionClosing {
7303 connection_id: client_connection,
7304 },
7305 )
7306 .expect("a closing client must never end the module connection");
7307
7308 assert!(matches!(
7309 handler.refuse_to_end_module_connection_for_a_client(
7310 module_connection,
7311 78,
7312 ForwardingError::ConnectionClosing {
7313 connection_id: module_connection,
7314 },
7315 ),
7316 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7317 connection_id
7318 })) if connection_id == module_connection
7319 ));
7320 assert!(matches!(
7321 handler.refuse_to_end_module_connection_for_a_client(
7322 module_connection,
7323 79,
7324 ForwardingError::Poisoned,
7325 ),
7326 Err(RouterError::Forwarding(ForwardingError::Poisoned))
7327 ));
7328 assert!(matches!(
7329 handler.refuse_to_end_module_connection_for_a_client(
7330 module_connection,
7331 80,
7332 ForwardingError::StaleModuleEndpoint,
7333 ),
7334 Err(RouterError::Forwarding(
7335 ForwardingError::StaleModuleEndpoint
7336 ))
7337 ));
7338 }
7339
7340 #[tokio::test]
7353 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7354 let registry = Arc::new(Registry::default());
7355 let forwarding = Arc::new(ForwardingTable::default());
7356 let supervisor = SupervisorHandle::new();
7357 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7358 let handler =
7359 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7360 .with_supervisor(supervisor);
7361
7362 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7363 hello_via_sink(
7364 &handler,
7365 &target_ctx,
7366 &mut target_rx,
7367 hello_frame("target", PROTOCOL_VERSION, 1),
7368 )
7369 .await;
7370
7371 let wrong_nonce = handler
7375 .handle_control_frame(
7376 &route_ctx(ConnectionId::new(91)).0,
7377 route_open_frame_with_admission_facts(
7378 20,
7379 "target",
7380 unique_project_root("admission-facts"),
7381 Some(subc_control::ConsumerIdentity {
7382 module_id: "fed".to_string(),
7383 launch_nonce: "not-the-real-nonce".to_string(),
7384 }),
7385 None,
7386 ),
7387 )
7388 .await
7389 .unwrap();
7390 assert_eq!(
7391 parse_error(&wrong_nonce[0])["code"],
7392 "bad_consumer_identity",
7393 "a mismatched launch nonce must be refused, not stamped Reserved"
7394 );
7395
7396 let never_spawned = handler
7400 .handle_control_frame(
7401 &route_ctx(ConnectionId::new(92)).0,
7402 route_open_frame_with_admission_facts(
7403 21,
7404 "target",
7405 unique_project_root("admission-facts"),
7406 Some(subc_control::ConsumerIdentity {
7407 module_id: "never-spawned".to_string(),
7408 launch_nonce: "any-nonce".to_string(),
7409 }),
7410 None,
7411 ),
7412 )
7413 .await
7414 .unwrap();
7415 assert_eq!(
7416 parse_error(&never_spawned[0])["code"],
7417 "bad_consumer_identity",
7418 "an unspawned module_id must be refused rather than accepted for lack of a record"
7419 );
7420 }
7421
7422 #[tokio::test]
7445 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
7446 let registry = Arc::new(Registry::default());
7447 let forwarding = Arc::new(ForwardingTable::default());
7448 let supervisor = SupervisorHandle::new();
7449 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7450 let handler =
7451 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7452 .with_supervisor(supervisor);
7453
7454 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
7455 hello_via_sink(
7456 &handler,
7457 &target_ctx,
7458 &mut target_rx,
7459 hello_frame("target", PROTOCOL_VERSION, 1),
7460 )
7461 .await;
7462
7463 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
7464 let route_handler = handler.clone();
7465 let route_task = tokio::spawn(async move {
7466 route_handler
7467 .handle_control_frame(
7468 &client_ctx,
7469 route_open_frame_with_admission_facts(
7470 30,
7471 "target",
7472 unique_project_root("admission-facts"),
7473 Some(subc_control::ConsumerIdentity {
7474 module_id: "fed".to_string(),
7475 launch_nonce: "fed-nonce".to_string(),
7476 }),
7477 None,
7478 ),
7479 )
7480 .await
7481 .unwrap()
7482 });
7483
7484 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7491 .await
7492 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
7493 .expect("module control channel closed before route.bind");
7494 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7495 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
7496 panic!("expected route.bind")
7497 };
7498 assert_eq!(
7499 principal,
7500 Some(Principal::Reserved {
7501 module_id: "fed".to_string()
7502 }),
7503 "a correctly attested consumer must be stamped Reserved for its own id"
7504 );
7505
7506 handler
7507 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
7508 .await
7509 .unwrap();
7510 assert!(route_task.await.unwrap().is_empty());
7511 assert!(
7512 matches!(
7513 serde_json::from_slice::<ClientControlResponse>(
7514 &client_rx.recv().await.unwrap().body
7515 )
7516 .unwrap(),
7517 ClientControlResponse::RouteOpen { .. }
7518 ),
7519 "the route must actually open, not merely avoid an error"
7520 );
7521 }
7522
7523 #[tokio::test(start_paused = true)]
7524 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
7525 let registry = Arc::new(Registry::default());
7526 let forwarding = Arc::new(ForwardingTable::default());
7527 let supervisor = SupervisorHandle::new();
7528 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7529 let handler =
7530 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7531 .with_supervisor(supervisor);
7532
7533 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
7534 hello_via_sink(
7535 &handler,
7536 &target_ctx,
7537 &mut target_rx,
7538 hello_frame("target", PROTOCOL_VERSION, 1),
7539 )
7540 .await;
7541
7542 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
7543 let direct_handler = handler.clone();
7544 let direct_open = tokio::spawn(async move {
7545 direct_handler
7546 .handle_control_frame(
7547 &direct_ctx,
7548 route_open_frame(2, "target", unique_project_root("route-census-direct")),
7549 )
7550 .await
7551 .unwrap()
7552 });
7553 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7554 .await
7555 .expect("no direct route.bind within 5s")
7556 .expect("target control channel closed before direct route.bind");
7557 handler
7558 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
7559 .await
7560 .unwrap();
7561 assert!(direct_open.await.unwrap().is_empty());
7562 let _ = direct_rx.recv().await.unwrap();
7563
7564 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
7565 let reserved_handler = handler.clone();
7566 let reserved_open = tokio::spawn(async move {
7567 reserved_handler
7568 .handle_control_frame(
7569 &reserved_ctx,
7570 route_open_frame_with_admission_facts(
7571 3,
7572 "target",
7573 unique_project_root("admission-facts"),
7574 Some(ConsumerIdentity {
7575 module_id: "fed".to_string(),
7576 launch_nonce: "fed-nonce".to_string(),
7577 }),
7578 None,
7579 ),
7580 )
7581 .await
7582 .unwrap()
7583 });
7584 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7585 .await
7586 .expect("no reserved route.bind within 5s")
7587 .expect("target control channel closed before reserved route.bind");
7588 handler
7589 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
7590 .await
7591 .unwrap();
7592 assert!(reserved_open.await.unwrap().is_empty());
7593 let _ = reserved_rx.recv().await.unwrap();
7594
7595 forwarding
7596 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
7597 .unwrap();
7598 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
7599 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
7600 module_id: Some("target".to_string()),
7601 })
7602 .unwrap();
7603 let census_frame =
7604 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
7605 let response = handler
7606 .handle_control_frame(&census_ctx, census_frame)
7607 .await
7608 .unwrap()
7609 .pop()
7610 .unwrap();
7611 let actual: Value = serde_json::from_slice(&response.body).unwrap();
7612 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
7613 assert!(matches!(
7614 decoded,
7615 ClientControlResponse::SupervisorRoutes { .. }
7616 ));
7617 let routes = actual["modules"][0]["routes"].as_array().unwrap();
7618 assert_eq!(routes.len(), 2);
7619 assert!(routes.iter().all(|route| route["draining"] == true));
7620 assert!(
7623 routes.iter().all(|route| route["drain_reason"] == "reload"),
7624 "draining routes must name the drain's reason: {routes:?}"
7625 );
7626 assert!(routes.iter().any(|route| {
7627 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
7628 }));
7629 assert!(routes.iter().any(|route| {
7630 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
7631 }));
7632
7633 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7634 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
7635 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7636 std::fs::write(
7637 &golden_path,
7638 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
7639 )
7640 .unwrap();
7641 }
7642 let expected: Value =
7643 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
7644 assert_eq!(actual, expected);
7645 }
7646
7647 async fn query_live_roots(
7648 handler: &ControlHandler,
7649 module_ctx: &RouteCtx,
7650 ) -> ModuleControlResponseToModule {
7651 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
7652 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
7653 let response = handler
7654 .handle_control_frame(module_ctx, frame)
7655 .await
7656 .unwrap()
7657 .pop()
7658 .unwrap();
7659 serde_json::from_slice(&response.body).unwrap()
7660 }
7661
7662 #[tokio::test(start_paused = true)]
7663 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
7664 let registry = Arc::new(Registry::default());
7665 let forwarding = Arc::new(ForwardingTable::default());
7666 let handler = ControlHandler::with_forwarding(registry, forwarding);
7667 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
7668 hello_via_sink(
7669 &handler,
7670 &target_ctx,
7671 &mut target_rx,
7672 hello_frame("target", PROTOCOL_VERSION, 1),
7673 )
7674 .await;
7675 let root = unique_project_root("live-roots-known");
7676 let path = ProjectRootId::from_path_allowing_missing(root.path())
7677 .unwrap()
7678 .as_path()
7679 .to_path_buf();
7680 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
7681 let open_handler = handler.clone();
7682 let opened = tokio::spawn(async move {
7683 open_handler
7684 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
7685 .await
7686 .unwrap()
7687 });
7688 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7689 .await
7690 .unwrap()
7691 .unwrap();
7692 handler
7693 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
7694 .await
7695 .unwrap();
7696 assert!(opened.await.unwrap().is_empty());
7697 let _ = client_rx.recv().await.unwrap();
7698
7699 let root = unique_project_root("live-roots-pending");
7700 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
7701 .unwrap()
7702 .as_path()
7703 .to_path_buf();
7704 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
7705 let open_handler = handler.clone();
7706 let pending = tokio::spawn(async move {
7707 open_handler
7708 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
7709 .await
7710 .unwrap()
7711 });
7712 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7713 .await
7714 .unwrap()
7715 .unwrap();
7716 let actual = query_live_roots(&handler, &target_ctx).await;
7717 let ModuleControlResponseToModule::LiveRoots {
7718 roots,
7719 unknown_root_bindings,
7720 total_bindings,
7721 } = actual
7722 else {
7723 panic!("expected live roots")
7724 };
7725 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
7726 assert_eq!(unknown_root_bindings, 0);
7727 assert_eq!(
7728 roots.len(),
7729 2,
7730 "root-known arm must retain each canonical root"
7731 );
7732 assert_eq!(
7733 total_bindings,
7734 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7735 );
7736 let counts = roots
7737 .iter()
7738 .map(|root| (root.project_root.clone(), root.bound, root.pending))
7739 .collect::<Vec<_>>();
7740 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
7741 expected.sort_by(|a, b| a.0.cmp(&b.0));
7742 assert_eq!(
7743 counts, expected,
7744 "roots must sort by path and count pending separately"
7745 );
7746 handler
7747 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
7748 .await
7749 .unwrap();
7750 assert!(pending.await.unwrap().is_empty());
7751 }
7752
7753 #[tokio::test(start_paused = true)]
7754 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
7755 let forwarding = Arc::new(ForwardingTable::default());
7756 let handler =
7757 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7758 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
7759 hello_via_sink(
7760 &handler,
7761 &target_ctx,
7762 &mut target_rx,
7763 hello_frame("target", PROTOCOL_VERSION, 1),
7764 )
7765 .await;
7766 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
7767 let pending = forwarding
7768 .begin_route_bind_relay_for_test(
7769 client_ctx.connection_id,
7770 client_ctx.egress.clone(),
7771 2,
7772 "target",
7773 )
7774 .unwrap();
7775 forwarding
7776 .complete_pending_relay(
7777 target_ctx.connection_id,
7778 pending.corr,
7779 RouteBindRelayOutcome::Accepted,
7780 )
7781 .unwrap();
7782 let actual = query_live_roots(&handler, &target_ctx).await;
7783 let ModuleControlResponseToModule::LiveRoots {
7784 roots,
7785 unknown_root_bindings,
7786 total_bindings,
7787 } = actual
7788 else {
7789 panic!("expected live roots")
7790 };
7791 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
7792 assert_eq!(
7793 unknown_root_bindings, 1,
7794 "unknown-root arm must not read as no bindings"
7795 );
7796 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
7797 assert_eq!(
7798 total_bindings,
7799 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7800 );
7801 }
7802
7803 #[tokio::test(start_paused = true)]
7811 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
7812 let forwarding = Arc::new(ForwardingTable::default());
7813 let handler =
7814 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7815 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
7816 let replies = handler
7817 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
7818 .await
7819 .unwrap();
7820 let queued_by_hello = module_rx.len();
7821
7822 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
7823 let open_handler = handler.clone();
7824 let open = tokio::spawn(async move {
7825 open_handler
7826 .handle_control_frame(
7827 &client_ctx,
7828 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
7829 )
7830 .await
7831 .unwrap()
7832 });
7833 let mut spins = 0;
7835 while module_rx.len() == queued_by_hello {
7836 spins += 1;
7837 assert!(spins < 10_000, "route.open never queued a route.bind");
7838 tokio::task::yield_now().await;
7839 }
7840
7841 for reply in replies {
7843 module_ctx.egress.send(reply).await.unwrap();
7844 }
7845
7846 let first = module_rx.recv().await.unwrap().frame;
7847 assert_eq!(
7848 first.header.ty,
7849 FrameType::HelloAck,
7850 "the first frame a registering module reads must be its HELLO_ACK"
7851 );
7852 assert_eq!(first.header.corr, 7);
7853 let second = module_rx.recv().await.unwrap().frame;
7854 assert_eq!(second.header.ty, FrameType::Request);
7855 assert!(
7856 matches!(
7857 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
7858 ModuleControlRequest::RouteBind { .. }
7859 ),
7860 "the route.bind follows the ack"
7861 );
7862 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
7863
7864 handler
7865 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
7866 .await
7867 .unwrap();
7868 assert!(open.await.unwrap().is_empty());
7869 let _ = client_rx.recv().await.unwrap();
7870 }
7871
7872 #[tokio::test(start_paused = true)]
7873 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
7874 let handler = ControlHandler::with_forwarding(
7875 Arc::new(Registry::default()),
7876 Arc::new(ForwardingTable::default()),
7877 );
7878 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
7879 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
7880 hello_via_sink(
7881 &handler,
7882 &first_ctx,
7883 &mut first_rx,
7884 hello_frame("first", PROTOCOL_VERSION, 1),
7885 )
7886 .await;
7887 hello_via_sink(
7888 &handler,
7889 &second_ctx,
7890 &mut second_rx,
7891 hello_frame("second", PROTOCOL_VERSION, 2),
7892 )
7893 .await;
7894 let root = unique_project_root("second-only");
7895 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
7896 let cloned = handler.clone();
7897 let open = tokio::spawn(async move {
7898 cloned
7899 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
7900 .await
7901 .unwrap()
7902 });
7903 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
7904 .await
7905 .unwrap()
7906 .unwrap();
7907 let first = query_live_roots(&handler, &first_ctx).await;
7908 let second = query_live_roots(&handler, &second_ctx).await;
7909 assert!(
7910 matches!(
7911 first,
7912 ModuleControlResponseToModule::LiveRoots {
7913 total_bindings: 0,
7914 ..
7915 }
7916 ),
7917 "cross-module scope must not expose another module's roots"
7918 );
7919 assert!(
7920 matches!(
7921 second,
7922 ModuleControlResponseToModule::LiveRoots {
7923 total_bindings: 1,
7924 ..
7925 }
7926 ),
7927 "second module must see its pending route"
7928 );
7929 handler
7930 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
7931 .await
7932 .unwrap();
7933 assert!(open.await.unwrap().is_empty());
7934 }
7935
7936 #[tokio::test(start_paused = true)]
7937 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
7938 let handler = ControlHandler::with_forwarding(
7939 Arc::new(Registry::default()),
7940 Arc::new(ForwardingTable::default()),
7941 );
7942 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
7943 hello_via_sink(
7944 &handler,
7945 &target_ctx,
7946 &mut target_rx,
7947 hello_frame("target", PROTOCOL_VERSION, 1),
7948 )
7949 .await;
7950 let actual = query_live_roots(&handler, &target_ctx).await;
7951 let ModuleControlResponseToModule::LiveRoots {
7952 roots,
7953 unknown_root_bindings,
7954 total_bindings,
7955 } = actual
7956 else {
7957 panic!("expected live roots")
7958 };
7959 assert!(roots.is_empty());
7960 assert_eq!(unknown_root_bindings, 0);
7961 assert_eq!(total_bindings, 0);
7962 assert_eq!(
7963 total_bindings,
7964 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7965 );
7966 }
7967
7968 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
7975 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7976 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
7977 let text = std::fs::read_to_string(&path)
7978 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
7979 let vectors: Vec<(String, Value)> = text
7980 .lines()
7981 .filter(|line| !line.trim().is_empty())
7982 .map(|line| {
7983 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
7984 let id = entry["corpus_id"]
7985 .as_str()
7986 .expect("every vector carries a corpus_id")
7987 .to_string();
7988 (id, entry["package"].clone())
7989 })
7990 .collect();
7991 assert_eq!(
7995 vectors.len(),
7996 3,
7997 "vendored fed corpus changed size; re-sync from subc-federation"
7998 );
7999
8000 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
8015 let richest = vectors
8016 .iter()
8017 .filter_map(|(_, package)| package.as_object())
8018 .map(|object| {
8019 object
8020 .keys()
8021 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
8022 .count()
8023 })
8024 .max()
8025 .unwrap_or(0);
8026 assert!(
8027 richest >= 2,
8028 "vendored corpus no longer carries a package with unmodelled fields, \
8029 so the relay test can no longer distinguish a verbatim relay from a lossy one"
8030 );
8031
8032 vectors
8033 }
8034
8035 #[tokio::test]
8043 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
8044 for (corpus_id, package) in fed_admission_facts_vectors() {
8045 let registry = Arc::new(Registry::default());
8046 let forwarding = Arc::new(ForwardingTable::default());
8047 let supervisor = SupervisorHandle::new();
8048 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8049 let handler =
8050 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8051 .with_supervisor(supervisor)
8052 .with_admission_facts_config(
8053 Some("fed".to_string()),
8054 Some(vec!["target".to_string()]),
8055 );
8056
8057 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8058 hello_via_sink(
8059 &handler,
8060 &target_ctx,
8061 &mut target_rx,
8062 hello_frame("target", PROTOCOL_VERSION, 1),
8063 )
8064 .await;
8065
8066 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8067 let route_handler = handler.clone();
8068 let expected = package.clone();
8069 let route_task = tokio::spawn(async move {
8070 route_handler
8071 .handle_control_frame(
8072 &client_ctx,
8073 route_open_frame_with_admission_facts(
8074 20,
8075 "target",
8076 unique_project_root("admission-facts"),
8077 Some(subc_control::ConsumerIdentity {
8078 module_id: "fed".to_string(),
8079 launch_nonce: "fed-nonce".to_string(),
8080 }),
8081 Some(package),
8082 ),
8083 )
8084 .await
8085 .unwrap()
8086 });
8087
8088 let bind_frame = target_rx.recv().await.unwrap();
8089 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8090 let ModuleControlRequest::RouteBind {
8091 admission_facts, ..
8092 } = bind
8093 else {
8094 panic!("{corpus_id}: expected route.bind")
8095 };
8096 assert_eq!(
8097 admission_facts,
8098 Some(expected),
8099 "{corpus_id}: relay must not add, drop or reshape any field"
8100 );
8101
8102 handler
8103 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8104 .await
8105 .unwrap();
8106 route_task.await.unwrap();
8107 }
8108 }
8109
8110 #[tokio::test]
8111 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8112 let registry = Arc::new(Registry::default());
8113 let forwarding = Arc::new(ForwardingTable::default());
8114 let supervisor = SupervisorHandle::new();
8115 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8116 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8117 let handler =
8118 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8119 .with_supervisor(supervisor)
8120 .with_admission_facts_config(
8121 Some("fed".to_string()),
8122 Some(vec!["target".to_string()]),
8123 );
8124
8125 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8126 hello_via_sink(
8127 &handler,
8128 &target_ctx,
8129 &mut target_rx,
8130 hello_frame("target", PROTOCOL_VERSION, 1),
8131 )
8132 .await;
8133 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8134 hello_via_sink(
8135 &handler,
8136 &other_ctx,
8137 &mut other_rx,
8138 hello_frame("other", PROTOCOL_VERSION, 2),
8139 )
8140 .await;
8141
8142 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8143 let expected_facts = facts.clone();
8144 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8145 let route_handler = handler.clone();
8146 let route_task = tokio::spawn(async move {
8147 route_handler
8148 .handle_control_frame(
8149 &client_ctx,
8150 route_open_frame_with_admission_facts(
8151 10,
8152 "target",
8153 unique_project_root("admission-facts"),
8154 Some(subc_control::ConsumerIdentity {
8155 module_id: "fed".to_string(),
8156 launch_nonce: "fed-nonce".to_string(),
8157 }),
8158 Some(facts.clone()),
8159 ),
8160 )
8161 .await
8162 .unwrap()
8163 });
8164 let bind_frame = target_rx.recv().await.unwrap();
8165 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8166 let ModuleControlRequest::RouteBind {
8167 admission_facts, ..
8168 } = bind
8169 else {
8170 panic!("expected route.bind")
8171 };
8172 assert_eq!(admission_facts, Some(expected_facts));
8173 handler
8174 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8175 .await
8176 .unwrap();
8177 assert!(route_task.await.unwrap().is_empty());
8178 assert!(matches!(
8179 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8180 .unwrap(),
8181 ClientControlResponse::RouteOpen { .. }
8182 ));
8183
8184 let direct = handler
8185 .handle_control_frame(
8186 &route_ctx(ConnectionId::new(73)).0,
8187 route_open_frame_with_admission_facts(
8188 11,
8189 "target",
8190 unique_project_root("admission-facts"),
8191 None,
8192 Some(json!({"x": 1})),
8193 ),
8194 )
8195 .await
8196 .unwrap();
8197 assert_eq!(
8198 parse_error(&direct[0])["code"],
8199 "admission_facts_not_permitted"
8200 );
8201
8202 let different_reserved = handler
8203 .handle_control_frame(
8204 &route_ctx(ConnectionId::new(77)).0,
8205 route_open_frame_with_admission_facts(
8206 15,
8207 "target",
8208 unique_project_root("admission-facts"),
8209 Some(subc_control::ConsumerIdentity {
8210 module_id: "other".to_string(),
8211 launch_nonce: "other-nonce".to_string(),
8212 }),
8213 Some(json!({"x": 1})),
8214 ),
8215 )
8216 .await
8217 .unwrap();
8218 assert_eq!(
8219 parse_error(&different_reserved[0])["code"],
8220 "admission_facts_not_permitted"
8221 );
8222
8223 let other_target = handler
8224 .handle_control_frame(
8225 &route_ctx(ConnectionId::new(74)).0,
8226 route_open_frame_with_admission_facts(
8227 12,
8228 "other",
8229 unique_project_root("admission-facts"),
8230 Some(subc_control::ConsumerIdentity {
8231 module_id: "fed".to_string(),
8232 launch_nonce: "fed-nonce".to_string(),
8233 }),
8234 Some(json!({"x": 1})),
8235 ),
8236 )
8237 .await
8238 .unwrap();
8239 assert_eq!(
8240 parse_error(&other_target[0])["code"],
8241 "admission_facts_target_not_allowed"
8242 );
8243
8244 let nonexistent = handler
8245 .handle_control_frame(
8246 &route_ctx(ConnectionId::new(75)).0,
8247 route_open_frame_with_admission_facts(
8248 13,
8249 "missing",
8250 unique_project_root("admission-facts"),
8251 None,
8252 Some(json!({"x": 1})),
8253 ),
8254 )
8255 .await
8256 .unwrap();
8257 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8258
8259 let described = handler
8260 .handle_control_frame(
8261 &route_ctx(ConnectionId::new(76)).0,
8262 Frame::build(
8263 FrameType::Request,
8264 control_flags(),
8265 0,
8266 0,
8267 14,
8268 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8269 )
8270 .unwrap(),
8271 )
8272 .await
8273 .unwrap();
8274 let ClientControlResponse::ServerDescribe { capabilities, .. } =
8275 serde_json::from_slice(&described[0].body).unwrap()
8276 else {
8277 panic!("expected server.describe response")
8278 };
8279 assert!(capabilities
8280 .iter()
8281 .any(|cap| cap == "admission_facts_relay_v1"));
8282 }
8283
8284 #[tokio::test]
8285 async fn admission_facts_without_configured_carrier_are_rejected() {
8286 let registry = Arc::new(Registry::default());
8287 let forwarding = Arc::new(ForwardingTable::default());
8288 let handler = ControlHandler::with_forwarding(registry, forwarding);
8289 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8290 hello_via_sink(
8291 &handler,
8292 &target_ctx,
8293 &mut target_rx,
8294 hello_frame("target", PROTOCOL_VERSION, 1),
8295 )
8296 .await;
8297
8298 let responses = handler
8299 .handle_control_frame(
8300 &route_ctx(ConnectionId::new(79)).0,
8301 route_open_frame_with_admission_facts(
8302 16,
8303 "target",
8304 unique_project_root("admission-facts"),
8305 None,
8306 Some(json!({"x": 1})),
8307 ),
8308 )
8309 .await
8310 .unwrap();
8311 assert_eq!(
8312 parse_error(&responses[0])["code"],
8313 "admission_facts_not_permitted"
8314 );
8315 }
8316
8317 #[tokio::test]
8318 async fn route_open_relays_consumer_capabilities_verbatim() {
8319 let registry = Arc::new(Registry::default());
8320 let forwarding = Arc::new(ForwardingTable::default());
8321 let handler =
8322 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8323 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8324 hello_via_sink(
8325 &handler,
8326 &module_ctx,
8327 &mut module_rx,
8328 hello_frame("aft", PROTOCOL_VERSION, 7),
8329 )
8330 .await;
8331
8332 let expected = vec!["elicitation".to_string(), "roots".to_string()];
8333 let expected_for_request = expected.clone();
8334 let project_root = unique_project_root("consumer-capabilities-present");
8335 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8336 let route_handler = handler.clone();
8337 let route_task = tokio::spawn(async move {
8338 route_handler
8339 .handle_control_frame(
8340 &client_ctx,
8341 route_open_frame_with_consumer_capabilities(
8342 401,
8343 "aft",
8344 project_root,
8345 Some(expected_for_request),
8346 ),
8347 )
8348 .await
8349 .unwrap()
8350 });
8351 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8352 .await
8353 .unwrap()
8354 .unwrap();
8355 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8356 let ModuleControlRequest::RouteBind {
8357 consumer_capabilities,
8358 ..
8359 } = bind
8360 else {
8361 panic!("expected route.bind request, got {bind:?}");
8362 };
8363 assert_eq!(consumer_capabilities, Some(expected.clone()));
8364
8365 handler
8366 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8367 .await
8368 .unwrap();
8369 let route_response = route_task.await.unwrap();
8370 assert!(route_response.is_empty());
8371 let published = client_rx.recv().await.unwrap();
8372 assert!(matches!(
8373 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8374 ClientControlResponse::RouteOpen { .. }
8375 ));
8376 }
8377
8378 #[tokio::test]
8379 async fn route_open_without_consumer_capabilities_relays_none() {
8380 let registry = Arc::new(Registry::default());
8381 let forwarding = Arc::new(ForwardingTable::default());
8382 let handler =
8383 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8384 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8385 hello_via_sink(
8386 &handler,
8387 &module_ctx,
8388 &mut module_rx,
8389 hello_frame("aft", PROTOCOL_VERSION, 7),
8390 )
8391 .await;
8392
8393 let project_root = unique_project_root("consumer-capabilities-absent");
8394 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8395 let route_handler = handler.clone();
8396 let route_task = tokio::spawn(async move {
8397 route_handler
8398 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
8399 .await
8400 .unwrap()
8401 });
8402 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8403 .await
8404 .unwrap()
8405 .unwrap();
8406 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8407 let ModuleControlRequest::RouteBind {
8408 consumer_capabilities,
8409 ..
8410 } = bind
8411 else {
8412 panic!("expected route.bind request, got {bind:?}");
8413 };
8414 assert_eq!(consumer_capabilities, None);
8415
8416 handler
8417 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8418 .await
8419 .unwrap();
8420 let route_response = route_task.await.unwrap();
8421 assert!(route_response.is_empty());
8422 let published = client_rx.recv().await.unwrap();
8423 assert!(matches!(
8424 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8425 ClientControlResponse::RouteOpen { .. }
8426 ));
8427 }
8428
8429 #[tokio::test]
8430 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
8431 let registry = Arc::new(Registry::default());
8432 let forwarding = Arc::new(ForwardingTable::default());
8433 let handler =
8434 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8435 .with_health_probe_timeout(Duration::from_secs(5));
8436 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
8437 hello_via_sink(
8438 &handler,
8439 &module_ctx,
8440 &mut module_rx,
8441 non_routable_hello_frame_with_control_ops(
8442 "mcp",
8443 300,
8444 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8445 ),
8446 )
8447 .await;
8448 assert!(registry
8449 .get_module("mcp")
8450 .unwrap()
8451 .unwrap()
8452 .manifest
8453 .provides
8454 .is_empty());
8455
8456 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
8457 let route_response = handler
8458 .handle_control_frame(
8459 &route_client_ctx,
8460 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
8461 )
8462 .await
8463 .unwrap();
8464 assert_eq!(route_response[0].header.ty, FrameType::Error);
8465 assert_eq!(
8466 parse_error(&route_response[0])["code"],
8467 "target_unavailable"
8468 );
8469 assert!(parse_error(&route_response[0])["message"]
8470 .as_str()
8471 .unwrap()
8472 .contains("does not provide the requested target"));
8473 assert!(module_rx.try_recv().is_err());
8474
8475 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
8476 let health_handler = handler.clone();
8477 let health_task = tokio::spawn(async move {
8478 health_handler
8479 .handle_control_frame(
8480 &health_client_ctx,
8481 supervisor_health_probe_frame(302, "mcp"),
8482 )
8483 .await
8484 .unwrap()
8485 });
8486 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8487 .await
8488 .unwrap()
8489 .unwrap();
8490 assert_eq!(
8491 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8492 ModuleControlRequest::HealthCheck {}
8493 );
8494 handler
8495 .handle_control_frame(
8496 &module_ctx,
8497 health_response(health_frame.header.corr, HealthStatus::Ok),
8498 )
8499 .await
8500 .unwrap();
8501 let health_response = health_task.await.unwrap();
8502 assert_eq!(health_response[0].header.ty, FrameType::Response);
8503 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8504 ClientControlResponse::SupervisorHealthProbe {
8505 module_id, status, ..
8506 } => {
8507 assert_eq!(module_id, "mcp");
8508 assert_eq!(status, HealthStatus::Ok);
8509 }
8510 other => panic!("unexpected health response: {other:?}"),
8511 }
8512
8513 forwarding
8518 .cleanup_connection(module_ctx.connection_id)
8519 .unwrap();
8520 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
8521 let cleanup_response = tokio::time::timeout(
8522 Duration::from_millis(200),
8523 handler.handle_control_frame(
8524 &cleanup_probe_ctx,
8525 supervisor_health_probe_frame(303, "mcp"),
8526 ),
8527 )
8528 .await
8529 .expect("probe should fail immediately when the control lane is gone")
8530 .unwrap();
8531 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
8532 assert_eq!(
8533 parse_error(&cleanup_response[0])["code"],
8534 "target_unavailable"
8535 );
8536 assert!(parse_error(&cleanup_response[0])["message"]
8537 .as_str()
8538 .unwrap()
8539 .contains("no module connection"));
8540
8541 handler
8542 .cleanup_connection(module_ctx.connection_id)
8543 .unwrap();
8544 }
8545
8546 #[tokio::test]
8547 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
8548 let registry = Arc::new(Registry::default());
8549 let supervisor_handle = SupervisorHandle::new();
8550 let supervisor =
8551 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8552 .with_handle(supervisor_handle.clone())
8553 .with_connection_file_path(
8554 std::env::temp_dir()
8555 .join(format!("subc-route-open-warming-{}", std::process::id())),
8556 );
8557 let module = supervisor
8558 .supervise_configured(
8559 ModuleSpec {
8560 module_id: "warming".to_string(),
8561 program: fake_aft_stub_path(),
8562 args: Vec::new(),
8563 env: Vec::new(),
8564 reserved: false,
8565 reserved_prefixes: Vec::new(),
8566 protocol: ModuleProtocol::Subc,
8567 overlap: Default::default(),
8568 },
8569 true,
8570 )
8571 .unwrap();
8572 assert_eq!(module.state().unwrap(), ModuleState::Running);
8573
8574 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8575 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
8576 let response = handler
8577 .handle_control_frame(
8578 &ctx,
8579 route_open_frame(304, "warming", unique_project_root("warming")),
8580 )
8581 .await
8582 .unwrap();
8583 module.stop().await.unwrap();
8584
8585 assert_eq!(response[0].header.ty, FrameType::Error);
8586 let error = parse_error(&response[0]);
8587 assert_eq!(error["code"], "module_warming");
8588 assert!(error["message"]
8589 .as_str()
8590 .unwrap()
8591 .contains("state=running, enabled=true, live=false"));
8592 }
8593
8594 #[test]
8595 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
8596 let handler = ControlHandler::new(Arc::new(Registry::default()));
8597 let capture = EventCapture::default();
8598 let _subscriber =
8599 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8600 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
8601 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
8602 let pending = (0..limit).collect::<Vec<_>>();
8603 let response = handler
8604 .route_open_capacity_refusal(
8605 &ctx,
8606 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
8607 "busy",
8608 pending.len(),
8609 limit,
8610 )
8611 .unwrap();
8612 assert_eq!(parse_error(&response)["code"], "target_unavailable");
8613 let event = capture
8614 .events()
8615 .into_iter()
8616 .find(|event| {
8617 event.target == "control"
8618 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
8619 })
8620 .expect("connection admission refusal event");
8621 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
8622 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
8623 }
8624
8625 #[test]
8626 fn route_open_target_cap_logs_admission_reason_and_capacity() {
8627 let handler = ControlHandler::new(Arc::new(Registry::default()));
8628 let capture = EventCapture::default();
8629 let _subscriber =
8630 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8631 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
8632 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
8633 let guards = (0..limit)
8634 .map(|_| {
8635 handler
8636 .route_bind_concurrency
8637 .try_admit("busy", limit)
8638 .unwrap()
8639 })
8640 .collect::<Vec<_>>();
8641 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
8642 Err(in_flight) => in_flight,
8643 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
8644 };
8645 let response = handler
8646 .route_open_target_capacity_refusal(
8647 &ctx,
8648 &route_open_frame(397, "busy", unique_project_root("target-cap")),
8649 "busy",
8650 in_flight,
8651 )
8652 .unwrap();
8653 assert_eq!(parse_error(&response)["code"], "target_unavailable");
8654 let event = capture
8655 .events()
8656 .into_iter()
8657 .find(|event| {
8658 event.target == "control"
8659 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
8660 })
8661 .expect("target admission refusal event");
8662 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
8663 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
8664 drop(guards);
8665 }
8666
8667 #[tokio::test]
8672 async fn route_open_refusal_names_the_check_that_refused() {
8673 let handler = ControlHandler::new(Arc::new(Registry::default()));
8674 let capture = EventCapture::default();
8675 let _subscriber =
8676 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8677 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8678 let response = handler
8679 .handle_control_frame(
8680 &ctx,
8681 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
8682 )
8683 .await
8684 .unwrap();
8685
8686 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8687 let event = capture
8688 .events()
8689 .into_iter()
8690 .find(|event| {
8691 event.target == "control"
8692 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8693 })
8694 .expect("route.open refusal event");
8695 assert_eq!(
8696 event.fields.get("reason"),
8697 Some(&"\"not_registered\"".to_string())
8698 );
8699 }
8700
8701 #[tokio::test]
8702 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
8703 let registry = Arc::new(Registry::default());
8704 let supervisor_handle = SupervisorHandle::new();
8705 let supervisor =
8706 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8707 .with_handle(supervisor_handle.clone())
8708 .with_connection_file_path(std::env::temp_dir().join(format!(
8709 "subc-route-open-refusal-info-{}",
8710 std::process::id()
8711 )));
8712 let module = supervisor
8713 .supervise_configured(
8714 ModuleSpec {
8715 module_id: "warming".to_string(),
8716 program: fake_aft_stub_path(),
8717 args: Vec::new(),
8718 env: Vec::new(),
8719 reserved: false,
8720 reserved_prefixes: Vec::new(),
8721 protocol: ModuleProtocol::Subc,
8722 overlap: Default::default(),
8723 },
8724 true,
8725 )
8726 .unwrap();
8727 assert_eq!(module.state().unwrap(), ModuleState::Running);
8728
8729 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8730 assert!(handler
8731 .counters()
8732 .snapshot()
8733 .get("route_open_refused_by_code")
8734 .is_none());
8735 let capture = EventCapture::default();
8736 let _subscriber =
8737 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8738 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
8739 let response = handler
8740 .handle_control_frame(
8741 &ctx,
8742 route_open_frame(394, "warming", unique_project_root("refusal-info")),
8743 )
8744 .await
8745 .unwrap();
8746 module.stop().await.unwrap();
8747
8748 assert_eq!(parse_error(&response[0])["code"], "module_warming");
8749 let event = capture
8750 .events()
8751 .into_iter()
8752 .find(|event| {
8753 event.target == "control"
8754 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
8755 })
8756 .expect("route.open refusal event");
8757 assert_eq!(
8758 event.fields.get("module_id"),
8759 Some(&"\"warming\"".to_string())
8760 );
8761 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
8762 assert_eq!(
8763 event.fields.get("reason"),
8764 Some(&"\"supervised_not_registered\"".to_string())
8765 );
8766 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
8767 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
8768 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
8769 assert_eq!(
8770 handler.counters().snapshot()["route_open_refused_by_code"],
8771 json!({ "module_warming": 1 })
8772 );
8773 }
8774
8775 #[tokio::test(flavor = "current_thread")]
8776 async fn route_open_unknown_module_escapes_target_module_id() {
8777 let handler = ControlHandler::new(Arc::new(Registry::default()));
8778 let capture = EventCapture::default();
8779 let _subscriber =
8780 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8781 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
8782 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8783 let response = handler
8784 .handle_control_frame(
8785 &ctx,
8786 route_open_frame(
8787 395,
8788 hostile_module_id,
8789 unique_project_root("hostile-target-module-id"),
8790 ),
8791 )
8792 .await
8793 .unwrap();
8794
8795 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8796 let event = capture
8797 .events()
8798 .into_iter()
8799 .find(|event| {
8800 event.target == "control"
8801 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8802 })
8803 .expect("route.open unknown-module refusal event");
8804 let logged = event.fields.get("module_id").expect("module_id field");
8805 assert!(!logged.bytes().any(|byte| byte < 0x20));
8806 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8807 }
8808
8809 #[tokio::test(flavor = "current_thread")]
8810 async fn route_open_module_rejection_uses_daemon_counter_key() {
8811 let registry = Arc::new(Registry::default());
8812 let forwarding = Arc::new(ForwardingTable::default());
8813 let handler =
8814 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8815 let module_connection = ConnectionId::new(95);
8816 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8817 hello_via_sink(
8818 &handler,
8819 &module_ctx,
8820 &mut module_rx,
8821 hello_frame("aft", PROTOCOL_VERSION, 395),
8822 )
8823 .await;
8824
8825 let client_connection = ConnectionId::new(96);
8826 let (client_ctx, _client_rx) = route_ctx(client_connection);
8827 let capture = EventCapture::default();
8828 let _subscriber =
8829 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8830 let (route_task, bind) = relay_route_open(
8831 &handler,
8832 client_connection,
8833 &client_ctx.egress,
8834 &mut module_rx,
8835 396,
8836 "aft",
8837 "hostile-module-code",
8838 )
8839 .await;
8840 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
8841 let rejection = Frame::build(
8842 FrameType::Error,
8843 control_flags(),
8844 0,
8845 0,
8846 bind.header.corr,
8847 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
8848 )
8849 .unwrap();
8850 handler
8851 .handle_control_frame(&module_ctx, rejection)
8852 .await
8853 .unwrap();
8854
8855 let response = route_task.await.unwrap();
8856 assert_eq!(parse_error(&response[0])["code"], hostile_code);
8857 let counters = handler.counters().snapshot();
8858 assert_eq!(
8859 counters["route_open_refused_by_code"],
8860 json!({ "module_rejected": 1 })
8861 );
8862 assert!(counters["route_open_refused_by_code"]
8863 .get(hostile_code)
8864 .is_none());
8865
8866 let event = capture
8867 .events()
8868 .into_iter()
8869 .find(|event| {
8870 event.target == "control"
8871 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
8872 })
8873 .expect("route.open module-rejection refusal event");
8874 let logged = event.fields.get("module_code").expect("module_code field");
8875 assert!(!logged.bytes().any(|byte| byte < 0x20));
8876 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8877 }
8878
8879 #[tokio::test]
8880 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
8881 let registry = Arc::new(Registry::default());
8882 let supervisor_handle = SupervisorHandle::new();
8883 let missing_program = std::env::temp_dir().join(format!(
8884 "subc-route-open-missing-program-{}",
8885 std::process::id()
8886 ));
8887 let supervisor =
8888 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8889 .with_handle(supervisor_handle.clone());
8890 let module = supervisor
8891 .supervise_configured(
8892 ModuleSpec {
8893 module_id: "failed".to_string(),
8894 program: missing_program,
8895 args: Vec::new(),
8896 env: Vec::new(),
8897 reserved: false,
8898 reserved_prefixes: Vec::new(),
8899 protocol: ModuleProtocol::Subc,
8900 overlap: Default::default(),
8901 },
8902 true,
8903 )
8904 .unwrap();
8905 assert_eq!(module.state().unwrap(), ModuleState::Failed);
8906
8907 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8908 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
8909 let response = handler
8910 .handle_control_frame(
8911 &ctx,
8912 route_open_frame(305, "failed", unique_project_root("failed")),
8913 )
8914 .await
8915 .unwrap();
8916
8917 assert_eq!(response[0].header.ty, FrameType::Error);
8918 let error = parse_error(&response[0]);
8919 assert_eq!(error["code"], "target_unavailable");
8920 assert!(error["message"]
8921 .as_str()
8922 .unwrap()
8923 .contains("state=failed, enabled=true, live=false"));
8924 }
8925
8926 #[tokio::test]
8927 async fn route_open_role_mismatch_remains_target_unavailable() {
8928 let registry = Arc::new(Registry::default());
8929 let handler = ControlHandler::new(Arc::clone(®istry));
8930 handler
8931 .handle_control(
8932 ConnectionId::new(41),
8933 non_routable_hello_frame_with_control_ops("health-only", 306, None),
8934 )
8935 .unwrap();
8936
8937 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
8938 let response = handler
8939 .handle_control_frame(
8940 &ctx,
8941 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
8942 )
8943 .await
8944 .unwrap();
8945
8946 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8947 assert!(parse_error(&response[0])["message"]
8948 .as_str()
8949 .unwrap()
8950 .contains("does not provide the requested target"));
8951 }
8952
8953 #[tokio::test]
8954 async fn route_open_inactive_registration_remains_target_unavailable() {
8955 let registry = Arc::new(Registry::default());
8956 let handler = ControlHandler::new(Arc::clone(®istry));
8957 handler
8958 .handle_control(
8959 ConnectionId::new(43),
8960 hello_frame("inactive", PROTOCOL_VERSION, 308),
8961 )
8962 .unwrap();
8963 assert!(registry
8964 .set_module_state_for_test("inactive", ChannelState::Closed)
8965 .unwrap());
8966
8967 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
8968 let response = handler
8969 .handle_control_frame(
8970 &ctx,
8971 route_open_frame(309, "inactive", unique_project_root("inactive")),
8972 )
8973 .await
8974 .unwrap();
8975
8976 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8977 assert!(parse_error(&response[0])["message"]
8978 .as_str()
8979 .unwrap()
8980 .contains("is not active"));
8981 }
8982
8983 #[tokio::test]
8984 async fn late_health_reply_is_recorded_through_the_module_response_path() {
8985 let registry = Arc::new(Registry::default());
8986 let forwarding = Arc::new(ForwardingTable::default());
8987 let supervisor_handle = SupervisorHandle::new();
8988 let supervisor = Supervisor::new(Arc::clone(®istry), crate::RestartPolicy::default())
8989 .with_forwarding(Arc::clone(&forwarding))
8990 .with_handle(supervisor_handle.clone());
8991 let module = supervisor
8992 .supervise_configured(
8993 crate::ModuleSpec {
8994 module_id: "late-health-response".to_string(),
8995 program: PathBuf::from("disabled-module"),
8996 args: Vec::new(),
8997 env: Vec::new(),
8998 reserved: false,
8999 reserved_prefixes: Vec::new(),
9000 protocol: ModuleProtocol::Subc,
9001 overlap: Default::default(),
9002 },
9003 false,
9004 )
9005 .unwrap();
9006 let handler =
9007 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9008 .with_supervisor(supervisor_handle);
9009 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
9010 handler
9011 .handle_control_frame(
9012 &module_ctx,
9013 hello_frame_with_control_ops(
9014 "late-health-response",
9015 PROTOCOL_VERSION,
9016 7,
9017 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9018 ),
9019 )
9020 .await
9021 .unwrap();
9022 let probe_started_at = Instant::now() - Duration::from_millis(80);
9023 let pending = forwarding
9024 .begin_health_probe_rpc_for(
9025 "late-health-response",
9026 MODULE_CONTROL_OP_HEALTH_CHECK,
9027 probe_started_at,
9028 Instant::now() - Duration::from_millis(1),
9029 )
9030 .unwrap();
9031 assert!(forwarding
9032 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
9033 .unwrap());
9034
9035 let responses = handler
9036 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
9037 .await
9038 .unwrap();
9039
9040 assert!(responses.is_empty());
9041 let health = module.status().unwrap().health;
9042 assert_eq!(health.late_answer_count, 1);
9043 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
9044 }
9045
9046 #[tokio::test]
9047 async fn health_probe_timeout_and_module_death_are_typed() {
9048 let registry = Arc::new(Registry::default());
9049 let forwarding = Arc::new(ForwardingTable::default());
9050 let handler =
9051 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9052 .with_health_probe_timeout(Duration::from_millis(50));
9053 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9054 hello_via_sink(
9055 &handler,
9056 &module_ctx,
9057 &mut module_rx,
9058 hello_frame_with_control_ops(
9059 "aft",
9060 PROTOCOL_VERSION,
9061 7,
9062 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9063 ),
9064 )
9065 .await;
9066
9067 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9068 let responses = handler
9069 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
9070 .await
9071 .unwrap();
9072 assert_eq!(responses[0].header.ty, FrameType::Error);
9073 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
9074 let _ = module_rx.try_recv();
9075
9076 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
9077 let health_handler = handler.clone();
9078 let death_task = tokio::spawn(async move {
9079 health_handler
9080 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
9081 .await
9082 .unwrap()
9083 });
9084 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9085 .await
9086 .unwrap()
9087 .unwrap();
9088 handler
9089 .cleanup_connection(module_ctx.connection_id)
9090 .unwrap();
9091 let responses = death_task.await.unwrap();
9092 assert_eq!(responses[0].header.ty, FrameType::Error);
9093 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
9094 }
9095
9096 #[test]
9097 fn hello_requires_exact_protocol_version() {
9098 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
9099 let registry = Arc::new(Registry::default());
9100 let handler = ControlHandler::new(Arc::clone(®istry));
9101 let responses = handler
9102 .handle_control(
9103 ConnectionId::new(connection),
9104 hello_frame("aft", offered, 9),
9105 )
9106 .unwrap();
9107
9108 assert_eq!(responses.len(), 1);
9109 assert_eq!(responses[0].header.ty, FrameType::Error);
9110 let error = parse_error(&responses[0]);
9111 assert_eq!(error["code"], "version_unsupported");
9112 assert!(registry.get_module("aft").unwrap().is_none());
9113 assert_eq!(registry.active_registration_count().unwrap(), 0);
9114 }
9115 }
9116
9117 #[test]
9118 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
9119 let registry = Arc::new(Registry::default());
9120 let forwarding = Arc::new(ForwardingTable::default());
9121 let handler =
9122 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9123 let module_connection = ConnectionId::new(301);
9124 let registration = registry
9125 .register_with_control_ops(
9126 manifest("aft-push", PROTOCOL_VERSION),
9127 PROTOCOL_VERSION,
9128 module_connection,
9129 module_baseline_control_ops(),
9130 )
9131 .unwrap();
9132 let (module_tx, _module_rx) = mpsc::channel(8);
9133 let endpoint = forwarding
9134 .register_module_connection(
9135 module_connection,
9136 "aft-push".to_string(),
9137 PROTOCOL_VERSION,
9138 manifest_concurrency(®istration.manifest),
9139 FrameSink::new(module_tx),
9140 )
9141 .unwrap();
9142
9143 let unknown = Frame::build(
9145 FrameType::Push,
9146 control_flags(),
9147 0,
9148 0,
9149 5,
9150 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
9151 )
9152 .unwrap();
9153 let out = handler.handle_status_update(endpoint, unknown).unwrap();
9154 assert!(
9155 out.is_empty(),
9156 "unknown push op must be ignored, got {out:?}"
9157 );
9158
9159 let malformed = Frame::build(
9161 FrameType::Push,
9162 control_flags(),
9163 0,
9164 0,
9165 6,
9166 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
9167 )
9168 .unwrap();
9169 let out = handler.handle_status_update(endpoint, malformed).unwrap();
9170 assert_eq!(out.len(), 1);
9171 assert_eq!(out[0].header.ty, FrameType::Error);
9172 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
9173 }
9174
9175 #[test]
9176 fn hello_rejected_when_connection_already_owns_client_routes() {
9177 let registry = Arc::new(Registry::default());
9178 let forwarding = Arc::new(ForwardingTable::default());
9179 let handler =
9180 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9181 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
9183 let client_connection = ConnectionId::new(202);
9184
9185 let responses = handler
9188 .handle_control(
9189 client_connection,
9190 hello_frame("aft-second", PROTOCOL_VERSION, 9),
9191 )
9192 .unwrap();
9193 assert_eq!(responses[0].header.ty, FrameType::Error);
9194 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
9195 assert!(registry.get_module("aft-second").unwrap().is_none());
9196 }
9197
9198 #[test]
9199 fn reserved_module_hello_requires_matching_launch_nonce() {
9200 let registry = Arc::new(Registry::default());
9201 let supervisor = SupervisorHandle::new();
9202 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9205 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9206
9207 let no_nonce = handler
9209 .handle_control(
9210 ConnectionId::new(1),
9211 hello_frame("vault", PROTOCOL_VERSION, 1),
9212 )
9213 .unwrap();
9214 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
9215 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
9216 assert!(registry.get_module("vault").unwrap().is_none());
9217
9218 let wrong = handler
9220 .handle_control(
9221 ConnectionId::new(2),
9222 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
9223 )
9224 .unwrap();
9225 assert_eq!(wrong[0].header.ty, FrameType::Error);
9226 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
9227 assert!(registry.get_module("vault").unwrap().is_none());
9228
9229 let ok = handler
9231 .handle_control(
9232 ConnectionId::new(3),
9233 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
9234 )
9235 .unwrap();
9236 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
9237 assert!(registry.get_module("vault").unwrap().is_some());
9238 }
9239
9240 #[test]
9241 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
9242 let registry = Arc::new(Registry::default());
9243 let supervisor = SupervisorHandle::new();
9244 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9245 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9246 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9247
9248 let squat = handler
9249 .handle_control(
9250 ConnectionId::new(1),
9251 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
9252 )
9253 .unwrap();
9254 assert_eq!(squat[0].header.ty, FrameType::Error);
9255 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
9256 assert!(parse_error(&squat[0])["message"]
9257 .as_str()
9258 .unwrap()
9259 .contains("fed:"));
9260
9261 let accepted_peer = handler
9262 .handle_control(
9263 ConnectionId::new(2),
9264 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
9265 )
9266 .unwrap();
9267 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
9268
9269 let accepted_short = handler
9270 .handle_control(
9271 ConnectionId::new(3),
9272 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
9273 )
9274 .unwrap();
9275 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
9276
9277 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
9278 let response = handler
9279 .handle_control(
9280 ConnectionId::new(conn),
9281 hello_frame(module_id, PROTOCOL_VERSION, conn),
9282 )
9283 .unwrap();
9284 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
9285 }
9286 }
9287
9288 #[test]
9289 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
9290 let registry = Arc::new(Registry::default());
9291 let supervisor = SupervisorHandle::new();
9292 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9293 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9294 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
9295 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9296
9297 let owner_nonce = handler
9298 .handle_control(
9299 ConnectionId::new(1),
9300 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
9301 )
9302 .unwrap();
9303 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
9304 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
9305 assert!(registry.get_module("fed:special").unwrap().is_none());
9306
9307 let exact_nonce = handler
9308 .handle_control(
9309 ConnectionId::new(2),
9310 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
9311 )
9312 .unwrap();
9313 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
9314 assert!(registry.get_module("fed:special").unwrap().is_some());
9315 }
9316
9317 #[test]
9318 fn non_reserved_module_ignores_launch_nonce() {
9319 let registry = Arc::new(Registry::default());
9320 let handler = ControlHandler::new(Arc::clone(®istry));
9323 let no_nonce = handler
9324 .handle_control(
9325 ConnectionId::new(1),
9326 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
9327 )
9328 .unwrap();
9329 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
9330 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
9331
9332 let echoed_nonce = handler
9333 .handle_control(
9334 ConnectionId::new(2),
9335 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
9336 )
9337 .unwrap();
9338 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
9339 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
9340 }
9341
9342 #[test]
9343 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
9344 let handler = ControlHandler::default();
9345 let conn = ConnectionId::new(1);
9346 let malformed = Frame::build(
9347 FrameType::Hello,
9348 control_flags(),
9349 0,
9350 0,
9351 3,
9352 b"{not json".to_vec(),
9353 )
9354 .unwrap();
9355
9356 let error = handler.handle_control(conn, malformed).unwrap();
9357 assert_eq!(error[0].header.ty, FrameType::Error);
9358 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
9359
9360 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
9361 let pong = handler.handle_control(conn, ping).unwrap();
9362 assert_eq!(pong[0].header.ty, FrameType::Pong);
9363 assert_eq!(pong[0].header.corr, 4);
9364 }
9365
9366 #[test]
9367 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
9368 let registry = Arc::new(Registry::default());
9369 let handler = ControlHandler::new(Arc::clone(®istry));
9370
9371 handler
9372 .handle_control(
9373 ConnectionId::new(1),
9374 hello_frame("aft", PROTOCOL_VERSION, 1),
9375 )
9376 .unwrap();
9377 let duplicate = handler
9378 .handle_control(
9379 ConnectionId::new(2),
9380 hello_frame("aft", PROTOCOL_VERSION, 2),
9381 )
9382 .unwrap();
9383
9384 assert_eq!(duplicate[0].header.ty, FrameType::Error);
9385 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9386 let registration = registry.get_module("aft").unwrap().unwrap();
9387 assert_eq!(registration.connection_id, ConnectionId::new(1));
9388 }
9389
9390 #[test]
9391 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
9392 let registry = Arc::new(Registry::default());
9393 let forwarding = Arc::new(ForwardingTable::default());
9394 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
9395 let handler =
9396 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9397 .with_process_liveness(process_liveness);
9398 let (ctx, route_channel, route_epoch) =
9399 bind_liveness_route(®istry, &forwarding, "aft-dead");
9400 let responses = handler
9401 .handle_route_poll(
9402 &ctx,
9403 route_poll_frame(41, PollKind::Liveness, route_channel),
9404 route_channel,
9405 route_epoch,
9406 PollKind::Liveness,
9407 )
9408 .unwrap();
9409
9410 assert_eq!(responses.len(), 1);
9411 assert_eq!(responses[0].header.ty, FrameType::Response);
9412 assert_route_poll_liveness(&responses[0], false);
9413 }
9414
9415 #[test]
9416 fn liveness_poll_without_process_source_uses_bound_route() {
9417 let registry = Arc::new(Registry::default());
9418 let forwarding = Arc::new(ForwardingTable::default());
9419 let handler =
9420 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9421 let (ctx, route_channel, route_epoch) =
9422 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
9423 let responses = handler
9424 .handle_route_poll(
9425 &ctx,
9426 route_poll_frame(42, PollKind::Liveness, route_channel),
9427 route_channel,
9428 route_epoch,
9429 PollKind::Liveness,
9430 )
9431 .unwrap();
9432
9433 assert_route_poll_liveness(&responses[0], true);
9434 }
9435
9436 #[test]
9437 fn liveness_poll_untracked_process_source_uses_bound_route() {
9438 let registry = Arc::new(Registry::default());
9439 let forwarding = Arc::new(ForwardingTable::default());
9440 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
9441 let handler =
9442 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9443 .with_process_liveness(process_liveness);
9444 let (ctx, route_channel, route_epoch) =
9445 bind_liveness_route(®istry, &forwarding, "aft-untracked");
9446 let responses = handler
9447 .handle_route_poll(
9448 &ctx,
9449 route_poll_frame(43, PollKind::Liveness, route_channel),
9450 route_channel,
9451 route_epoch,
9452 PollKind::Liveness,
9453 )
9454 .unwrap();
9455
9456 assert_route_poll_liveness(&responses[0], true);
9457 }
9458
9459 #[tokio::test]
9460 async fn unknown_op_returns_unknown_control_op() {
9461 let handler = ControlHandler::default();
9462 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
9463 let request = Frame::build(
9464 FrameType::Request,
9465 control_flags(),
9466 0,
9467 0,
9468 55,
9469 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
9470 )
9471 .unwrap();
9472
9473 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9474
9475 assert_eq!(response.len(), 1);
9476 assert_eq!(response[0].header.ty, FrameType::Error);
9477 assert_eq!(response[0].header.corr, 55);
9478 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
9479 }
9480
9481 #[tokio::test]
9482 async fn supervisor_provenance_rejects_unknown_exact_module() {
9483 let handler = ControlHandler::default();
9484 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
9485 let request = Frame::build(
9486 FrameType::Request,
9487 control_flags(),
9488 0,
9489 0,
9490 57,
9491 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
9492 )
9493 .unwrap();
9494
9495 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9496
9497 assert_eq!(response.len(), 1);
9498 assert_eq!(response[0].header.ty, FrameType::Error);
9499 assert_eq!(response[0].header.corr, 57);
9500 let error = parse_error(&response[0]);
9501 assert_eq!(error["code"], "unknown_module");
9502 assert_eq!(error["message"], "module_id 'missing' is not supervised");
9503 }
9504
9505 #[test]
9506 fn provenance_probe_override_keeps_handler_tests_deterministic() {
9507 let expected = subc_control::RunningImageAgreement::Unavailable {
9508 reason: subc_control::RunningImageUnavailableReason::HashFailed,
9509 };
9510 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
9511 assert_eq!(handler.provenance_probe_override, Some(expected));
9512 }
9513
9514 #[test]
9515 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
9516 let verdict = reload_verdict(
9517 std::path::Path::new("/bin/new"),
9518 Some(std::path::Path::new("/bin/old")),
9519 subc_control::RunningImageAgreement::Unavailable {
9520 reason: subc_control::RunningImageUnavailableReason::HashFailed,
9521 },
9522 );
9523 assert!(matches!(
9524 verdict.path,
9525 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
9526 if configured == std::path::Path::new("/bin/new")
9527 && spawned_from == std::path::Path::new("/bin/old")
9528 ));
9529 }
9530
9531 #[test]
9532 fn reload_verdict_detects_replaced_image_at_same_path() {
9533 let image = subc_control::RunningImageAgreement::Mismatch {
9534 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
9535 digest: "old".into(),
9536 },
9537 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
9538 digest: "new".into(),
9539 },
9540 };
9541 let verdict = reload_verdict(
9542 std::path::Path::new("/bin/same"),
9543 Some(std::path::Path::new("/bin/same")),
9544 image.clone(),
9545 );
9546 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9547 assert_eq!(verdict.image, image);
9548 }
9549
9550 #[test]
9551 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
9552 let image = subc_control::RunningImageAgreement::Unavailable {
9553 reason: subc_control::RunningImageUnavailableReason::NotRunning,
9554 };
9555 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
9556 assert_eq!(
9557 verdict.path,
9558 subc_control::ReloadPathAgreement::Unavailable {
9559 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
9560 }
9561 );
9562 assert_eq!(verdict.image, image);
9563
9564 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
9565 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
9566 };
9567 let verdict = reload_verdict(
9568 std::path::Path::new("/bin/same"),
9569 Some(std::path::Path::new("/bin/same")),
9570 unconfirmed.clone(),
9571 );
9572 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9573 assert_eq!(verdict.image, unconfirmed);
9574 }
9575
9576 #[test]
9577 fn reload_verdict_preserves_each_image_unavailability_reason() {
9578 use subc_control::RunningImageUnavailableReason as Reason;
9579
9580 for reason in [
9581 Reason::NotRunning,
9582 Reason::UnsupportedPlatform,
9583 Reason::RunningExecutableUnreadable,
9584 Reason::SpawnedPathUnreadable,
9585 Reason::HashFailed,
9586 Reason::ProcessIdentityUnconfirmed,
9587 Reason::Unknown("future_probe_reason".to_string()),
9588 ] {
9589 let image = subc_control::RunningImageAgreement::Unavailable {
9590 reason: reason.clone(),
9591 };
9592 let verdict = reload_verdict(
9593 std::path::Path::new("/bin/same"),
9594 Some(std::path::Path::new("/bin/same")),
9595 image.clone(),
9596 );
9597 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9598 assert_eq!(verdict.image, image, "{reason:?}");
9599 }
9600 }
9601
9602 #[tokio::test]
9603 async fn malformed_control_bodies_return_invalid_control_body() {
9604 let handler = ControlHandler::default();
9605 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
9606
9607 for (corr, body) in [
9608 (56, br#"{"route_channel":1}"#.as_slice()),
9609 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
9610 (
9611 58,
9612 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
9613 ),
9614 ] {
9615 let request = Frame::build(
9616 FrameType::Request,
9617 control_flags(),
9618 0,
9619 0,
9620 corr,
9621 body.to_vec(),
9622 )
9623 .unwrap();
9624 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9625
9626 assert_eq!(response.len(), 1);
9627 assert_eq!(response[0].header.ty, FrameType::Error);
9628 assert_eq!(response[0].header.corr, corr);
9629 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
9630 }
9631 }
9632
9633 #[tokio::test]
9634 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
9635 let registry = Arc::new(Registry::default());
9636 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
9637 let router = Router::with_control_handler(Arc::clone(&control));
9638 let connection = router.begin_connection();
9639 let (ctx, mut rx) = route_ctx(connection.id());
9640
9641 router
9642 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
9643 .await
9644 .unwrap();
9645 let response = rx.recv().await.unwrap();
9646 let ack = parse_ack(&response);
9647 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9648 let channel = 1;
9649
9650 let goodbye =
9651 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
9652 router.route_for_connection(&ctx, goodbye).await.unwrap();
9653 assert!(rx.try_recv().is_err());
9654 assert!(registry.get_module("aft").unwrap().is_none());
9655
9656 router
9657 .route_for_connection(&ctx, channel_request(channel, 13))
9658 .await
9659 .unwrap();
9660 let error_frame = rx.recv().await.unwrap();
9661 assert_eq!(error_frame.header.ty, FrameType::Error);
9662 assert_eq!(error_frame.header.channel, channel);
9663 }
9664
9665 #[tokio::test]
9666 async fn dropping_router_connection_releases_registration() {
9667 let registry = Arc::new(Registry::default());
9668 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
9669 let router = Router::with_control_handler(control);
9670 let connection = router.begin_connection();
9671 let (ctx, mut rx) = route_ctx(connection.id());
9672
9673 router
9674 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
9675 .await
9676 .unwrap();
9677 let response = rx.recv().await.unwrap();
9678 let ack = parse_ack(&response);
9679 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9680 assert!(registry.get_module("aft").unwrap().is_some());
9681
9682 drop(connection);
9683
9684 assert!(registry.get_module("aft").unwrap().is_none());
9685 assert_eq!(registry.active_registration_count().unwrap(), 0);
9686 }
9687
9688 fn capability_manifest(
9689 module_id: &str,
9690 provides: &[&str],
9691 must_never_reach: &[&str],
9692 ) -> ModuleManifest {
9693 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9694 manifest.capabilities = Some(CapabilityDeclarations {
9695 provides: provides
9696 .iter()
9697 .map(|capability| (*capability).to_string())
9698 .collect(),
9699 requires: Vec::new(),
9700 must_never_reach: must_never_reach
9701 .iter()
9702 .map(|capability| (*capability).to_string())
9703 .collect(),
9704 });
9705 manifest
9706 }
9707
9708 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
9709 Frame::build(
9710 FrameType::Hello,
9711 control_flags(),
9712 0,
9713 0,
9714 corr,
9715 serde_json::to_vec(&ModuleHelloBody {
9716 protocol_ver: manifest.protocol_ver,
9717 manifest,
9718 control_ops: None,
9719 launch_nonce: None,
9720 })
9721 .expect("capability test HELLO serializes"),
9722 )
9723 .expect("capability test HELLO frame builds")
9724 }
9725
9726 fn catalog_update_with_capabilities_frame(
9727 corr: u64,
9728 capabilities: CapabilityDeclarations,
9729 ) -> Frame {
9730 Frame::build(
9731 FrameType::Request,
9732 control_flags(),
9733 0,
9734 0,
9735 corr,
9736 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
9737 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
9738 capabilities: Some(capabilities),
9739 ready: None,
9740 })
9741 .expect("capability catalog.update serializes"),
9742 )
9743 .expect("capability catalog.update frame builds")
9744 }
9745
9746 async fn register_capability_manifest(
9747 handler: &ControlHandler,
9748 ctx: &RouteCtx,
9749 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9750 manifest: ModuleManifest,
9751 corr: u64,
9752 ) {
9753 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
9754 }
9755
9756 async fn open_route_for_capability_test(
9757 handler: &ControlHandler,
9758 target_ctx: &RouteCtx,
9759 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9760 client_connection_id: u64,
9761 corr: u64,
9762 target_module_id: &str,
9763 consumer_identity: Option<ConsumerIdentity>,
9764 ) -> (
9765 mpsc::Receiver<crate::router::OutboundFrame>,
9766 ModuleControlRequest,
9767 ) {
9768 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
9769 let route_handler = handler.clone();
9770 let target_module_id = target_module_id.to_string();
9771 let route_task = tokio::spawn(async move {
9772 route_handler
9773 .handle_control_frame(
9774 &client_ctx,
9775 route_open_frame_with_admission_facts(
9776 corr,
9777 &target_module_id,
9778 unique_project_root("admission-facts"),
9779 consumer_identity,
9780 None,
9781 ),
9782 )
9783 .await
9784 .expect("capability test route.open succeeds")
9785 });
9786 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
9787 .await
9788 .expect("capability test route.open must reach route.bind")
9789 .expect("target control receiver stays open");
9790 let bind_request: ModuleControlRequest =
9791 serde_json::from_slice(&bind.body).expect("route.bind decodes");
9792 handler
9793 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
9794 .await
9795 .expect("capability test route.bind ACK succeeds");
9796 assert!(route_task.await.expect("route.open task joins").is_empty());
9797 let opened = client_rx
9798 .recv()
9799 .await
9800 .expect("successful route.open publishes a response");
9801 assert!(matches!(
9802 serde_json::from_slice::<ClientControlResponse>(&opened.body),
9803 Ok(ClientControlResponse::RouteOpen { .. })
9804 ));
9805 (client_rx, bind_request)
9806 }
9807
9808 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
9809 assert_eq!(frame.header.ty, FrameType::Push);
9810 assert_eq!(frame.header.channel, 0);
9811 assert_eq!(
9812 serde_json::from_slice::<ClientControlPush>(&frame.body)
9813 .expect("route.closed control push decodes"),
9814 ClientControlPush::RouteClosed {
9815 module_id: target_module_id.to_string(),
9816 reason: RouteCloseReason::CapabilityDenied,
9817 drained: false,
9818 abandoned: 0,
9819 excluded_subscriptions: 0,
9820 terminal: Some(false),
9821 }
9822 );
9823 }
9824
9825 #[tokio::test]
9826 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
9827 let registry = Arc::new(Registry::default());
9828 let forwarding = Arc::new(ForwardingTable::default());
9829 let supervisor = SupervisorHandle::new();
9830 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9831 let handler =
9832 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9833 .with_supervisor(supervisor);
9834 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
9835 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
9836 register_capability_manifest(
9837 &handler,
9838 &target_ctx,
9839 &mut target_rx,
9840 capability_manifest("target", &["credentials-provider/v1"], &[]),
9841 1,
9842 )
9843 .await;
9844 register_capability_manifest(
9845 &handler,
9846 &opener_ctx,
9847 &mut opener_rx,
9848 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9849 2,
9850 )
9851 .await;
9852
9853 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
9854 let replies = handler
9855 .handle_control_frame(
9856 &client_ctx,
9857 route_open_frame_with_admission_facts(
9858 3,
9859 "target",
9860 unique_project_root("admission-facts"),
9861 Some(ConsumerIdentity {
9862 module_id: "opener".to_string(),
9863 launch_nonce: "opener-nonce".to_string(),
9864 }),
9865 None,
9866 ),
9867 )
9868 .await
9869 .expect("denied route.open returns a typed frame");
9870 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
9871 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9872 assert!(
9873 target_rx.try_recv().is_err(),
9874 "forbidden route.open must not relay route.bind"
9875 );
9876 }
9877
9878 #[tokio::test]
9879 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
9880 let registry = Arc::new(Registry::default());
9881 let forwarding = Arc::new(ForwardingTable::default());
9882 let supervisor = SupervisorHandle::new();
9883 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9884 let handler =
9885 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9886 .with_supervisor(supervisor);
9887 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
9888 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
9889 register_capability_manifest(
9890 &handler,
9891 &target_ctx,
9892 &mut target_rx,
9893 capability_manifest("target", &["credentials-provider/v1"], &[]),
9894 1,
9895 )
9896 .await;
9897 register_capability_manifest(
9898 &handler,
9899 &old_opener_ctx,
9900 &mut old_opener_rx,
9901 capability_manifest("opener", &[], &[]),
9902 2,
9903 )
9904 .await;
9905 let (mut client_rx, _) = open_route_for_capability_test(
9906 &handler,
9907 &target_ctx,
9908 &mut target_rx,
9909 712,
9910 3,
9911 "target",
9912 Some(ConsumerIdentity {
9913 module_id: "opener".to_string(),
9914 launch_nonce: "opener-nonce".to_string(),
9915 }),
9916 )
9917 .await;
9918 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9919
9920 handler
9921 .cleanup_connection(old_opener_ctx.connection_id)
9922 .expect("old opener registration cleans up");
9923 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
9924 register_capability_manifest(
9925 &handler,
9926 &new_opener_ctx,
9927 &mut new_opener_rx,
9928 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9929 4,
9930 )
9931 .await;
9932
9933 assert_capability_denied_push(
9934 client_rx
9935 .try_recv()
9936 .expect("HELLO deny addition must emit route.closed")
9937 .frame,
9938 "target",
9939 );
9940 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9941 assert!(matches!(
9942 target_rx.try_recv(),
9943 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9944 ));
9945 }
9946
9947 #[tokio::test]
9948 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
9949 let registry = Arc::new(Registry::default());
9950 let forwarding = Arc::new(ForwardingTable::default());
9951 let supervisor = SupervisorHandle::new();
9952 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9953 let handler =
9954 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9955 .with_supervisor(supervisor);
9956 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
9957 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
9958 register_capability_manifest(
9959 &handler,
9960 &target_ctx,
9961 &mut target_rx,
9962 capability_manifest("target", &[], &[]),
9963 1,
9964 )
9965 .await;
9966 register_capability_manifest(
9967 &handler,
9968 &opener_ctx,
9969 &mut opener_rx,
9970 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9971 2,
9972 )
9973 .await;
9974 let (mut client_rx, _) = open_route_for_capability_test(
9975 &handler,
9976 &target_ctx,
9977 &mut target_rx,
9978 722,
9979 3,
9980 "target",
9981 Some(ConsumerIdentity {
9982 module_id: "opener".to_string(),
9983 launch_nonce: "opener-nonce".to_string(),
9984 }),
9985 )
9986 .await;
9987 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9988
9989 let replies = handler
9990 .handle_control_frame(
9991 &target_ctx,
9992 catalog_update_with_capabilities_frame(
9993 4,
9994 CapabilityDeclarations {
9995 provides: vec!["credentials-provider/v1".to_string()],
9996 requires: Vec::new(),
9997 must_never_reach: Vec::new(),
9998 },
9999 ),
10000 )
10001 .await
10002 .expect("claim catalog.update succeeds");
10003 assert!(matches!(
10004 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
10005 Ok(ModuleControlResponseToModule::CatalogUpdate {})
10006 ));
10007 assert_capability_denied_push(
10008 client_rx
10009 .try_recv()
10010 .expect("claim addition must emit route.closed")
10011 .frame,
10012 "target",
10013 );
10014 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10015 assert!(matches!(
10016 target_rx.try_recv(),
10017 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10018 ));
10019 }
10020
10021 #[tokio::test]
10022 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
10023 let registry = Arc::new(Registry::default());
10024 let forwarding = Arc::new(ForwardingTable::default());
10025 let supervisor = SupervisorHandle::new();
10026 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10027 let handler =
10028 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10029 .with_supervisor(supervisor);
10030 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
10031 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
10032 register_capability_manifest(
10033 &handler,
10034 &target_ctx,
10035 &mut target_rx,
10036 capability_manifest("target", &["credentials-provider/v1"], &[]),
10037 1,
10038 )
10039 .await;
10040 register_capability_manifest(
10041 &handler,
10042 &opener_ctx,
10043 &mut opener_rx,
10044 capability_manifest("opener", &[], &[]),
10045 2,
10046 )
10047 .await;
10048 let (mut client_rx, _) = open_route_for_capability_test(
10049 &handler,
10050 &target_ctx,
10051 &mut target_rx,
10052 732,
10053 3,
10054 "target",
10055 Some(ConsumerIdentity {
10056 module_id: "opener".to_string(),
10057 launch_nonce: "opener-nonce".to_string(),
10058 }),
10059 )
10060 .await;
10061 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10062
10063 handler
10064 .handle_control_frame(
10065 &target_ctx,
10066 catalog_update_with_capabilities_frame(
10067 4,
10068 CapabilityDeclarations {
10069 provides: Vec::new(),
10070 requires: Vec::new(),
10071 must_never_reach: Vec::new(),
10072 },
10073 ),
10074 )
10075 .await
10076 .expect("claim removal catalog.update succeeds");
10077 assert_eq!(
10078 forwarding.active_binding_count().unwrap(),
10079 1,
10080 "removing an attested target claim must leave the route census unchanged"
10081 );
10082 assert!(
10083 client_rx.try_recv().is_err(),
10084 "claim removal must not emit route.closed capability_denied"
10085 );
10086 assert!(
10087 target_rx.try_recv().is_err(),
10088 "claim removal must not send the target a route GOODBYE"
10089 );
10090 }
10091
10092 #[tokio::test]
10095 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
10096 let registry = Arc::new(Registry::default());
10097 let forwarding = Arc::new(ForwardingTable::default());
10098 let supervisor = SupervisorHandle::new();
10099 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10100 let handler =
10101 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10102 .with_supervisor(supervisor);
10103 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
10104 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
10105 register_capability_manifest(
10106 &handler,
10107 &target_ctx,
10108 &mut target_rx,
10109 capability_manifest("target", &["credentials-provider/v1"], &[]),
10110 1,
10111 )
10112 .await;
10113 register_capability_manifest(
10114 &handler,
10115 &opener_ctx,
10116 &mut opener_rx,
10117 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10118 2,
10119 )
10120 .await;
10121
10122 let (_client_rx, bind) = open_route_for_capability_test(
10123 &handler,
10124 &target_ctx,
10125 &mut target_rx,
10126 742,
10127 3,
10128 "target",
10129 None,
10130 )
10131 .await;
10132 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10133 panic!("direct scope-honesty route must bind");
10134 };
10135 assert_eq!(principal, Some(Principal::Direct));
10136 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10137 }
10138
10139 #[tokio::test]
10142 async fn must_never_reach_self_route_is_capability_forbidden() {
10143 let registry = Arc::new(Registry::default());
10144 let forwarding = Arc::new(ForwardingTable::default());
10145 let supervisor = SupervisorHandle::new();
10146 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
10147 let handler =
10148 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10149 .with_supervisor(supervisor);
10150 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
10151 register_capability_manifest(
10152 &handler,
10153 &self_ctx,
10154 &mut self_rx,
10155 capability_manifest(
10156 "self-provider",
10157 &["credentials-provider/v1"],
10158 &["credentials-provider/v1"],
10159 ),
10160 1,
10161 )
10162 .await;
10163
10164 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
10165 let replies = handler
10166 .handle_control_frame(
10167 &client_ctx,
10168 route_open_frame_with_admission_facts(
10169 2,
10170 "self-provider",
10171 unique_project_root("admission-facts"),
10172 Some(ConsumerIdentity {
10173 module_id: "self-provider".to_string(),
10174 launch_nonce: "self-nonce".to_string(),
10175 }),
10176 None,
10177 ),
10178 )
10179 .await
10180 .expect("self-route refusal returns a typed frame");
10181 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10182 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10183 assert!(
10184 self_rx.try_recv().is_err(),
10185 "self denial must not relay route.bind"
10186 );
10187 }
10188
10189 #[test]
10190 fn unsupported_channel_zero_frame_returns_error() {
10191 let handler = ControlHandler::default();
10192 let request = Frame::build(
10193 FrameType::Request,
10194 control_flags(),
10195 0,
10196 0,
10197 21,
10198 b"opaque".to_vec(),
10199 )
10200 .unwrap();
10201
10202 let response = handler
10203 .handle_control(ConnectionId::new(1), request)
10204 .unwrap();
10205
10206 assert_eq!(response[0].header.ty, FrameType::Error);
10207 assert_eq!(
10208 parse_error(&response[0])["code"],
10209 "unsupported_control_frame"
10210 );
10211 }
10212
10213 mod swap {
10218 use super::*;
10219
10220 const INCUMBENT: ConnectionId = ConnectionId::new(30);
10221 const CANDIDATE: ConnectionId = ConnectionId::new(40);
10222
10223 struct Swap {
10224 registry: Arc<Registry>,
10225 forwarding: Arc<ForwardingTable>,
10226 handler: ControlHandler,
10227 incumbent_ctx: RouteCtx,
10228 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10229 candidate_ctx: RouteCtx,
10230 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10231 }
10232
10233 async fn swap_with_incumbent() -> Swap {
10234 let registry = Arc::new(Registry::default());
10235 let forwarding = Arc::new(ForwardingTable::default());
10236 let handler =
10237 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10238 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
10239 hello_via_sink(
10240 &handler,
10241 &incumbent_ctx,
10242 &mut incumbent_rx,
10243 hello_frame("aft", PROTOCOL_VERSION, 7),
10244 )
10245 .await;
10246 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
10247 Swap {
10248 registry,
10249 forwarding,
10250 handler,
10251 incumbent_ctx,
10252 incumbent_rx,
10253 candidate_ctx,
10254 candidate_rx,
10255 }
10256 }
10257
10258 fn register_candidate(swap: &Swap, ready: Option<bool>) {
10259 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
10260 candidate_manifest.ready = ready;
10261 let registration = swap
10262 .registry
10263 .register_candidate_with_control_ops(
10264 candidate_manifest,
10265 PROTOCOL_VERSION,
10266 CANDIDATE,
10267 module_baseline_control_ops(),
10268 )
10269 .unwrap();
10270 swap.forwarding
10271 .register_candidate_module_connection(
10272 CANDIDATE,
10273 "aft".to_string(),
10274 PROTOCOL_VERSION,
10275 manifest_concurrency(®istration.manifest),
10276 swap.candidate_ctx.egress.clone(),
10277 )
10278 .unwrap();
10279 }
10280
10281 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
10282 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
10283 swap.registry.promote_candidate("aft").unwrap().unwrap();
10284 cutover.incumbent.unwrap()
10285 }
10286
10287 fn keyed_total(counters: &Value, key: &str) -> u64 {
10288 counters[key]
10289 .as_object()
10290 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
10291 .unwrap_or(0)
10292 }
10293
10294 #[tokio::test]
10300 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
10301 let mut swap = swap_with_incumbent().await;
10302 let handler = swap.handler.clone();
10303
10304 let cotenant = ConnectionId::new(31);
10306 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10307 let (cotenant_task, cotenant_bind) = relay_route_open(
10308 &handler,
10309 cotenant,
10310 &cotenant_ctx.egress,
10311 &mut swap.incumbent_rx,
10312 100,
10313 "aft",
10314 "swap-cotenant",
10315 )
10316 .await;
10317 handler
10318 .handle_control_frame(
10319 &swap.incumbent_ctx,
10320 route_bind_ack(cotenant_bind.header.corr),
10321 )
10322 .await
10323 .unwrap();
10324 assert!(cotenant_task.await.unwrap().is_empty());
10325 let (cotenant_channel, cotenant_epoch) =
10326 published_route(&cotenant_rx.recv().await.unwrap());
10327
10328 let caller = ConnectionId::new(32);
10330 let (caller_ctx, mut caller_rx) = route_ctx(caller);
10331 let (caller_task, caller_bind) = relay_route_open(
10332 &handler,
10333 caller,
10334 &caller_ctx.egress,
10335 &mut swap.incumbent_rx,
10336 101,
10337 "aft",
10338 "swap-caller",
10339 )
10340 .await;
10341 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
10342
10343 register_candidate(&swap, None);
10344 cutover(&swap);
10345
10346 let ack = handler
10348 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
10349 .await;
10350 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10351 if module_loop_error.is_some() {
10352 handler.cleanup_connection(INCUMBENT).unwrap();
10355 }
10356
10357 assert!(
10359 cotenant_rx.try_recv().is_err(),
10360 "the co-tenant route on the incumbent was torn down by one late ack: \
10361 {module_loop_error:?}"
10362 );
10363 assert!(matches!(
10364 swap.forwarding
10365 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10366 .unwrap(),
10367 DataRoute::Client(DataRouteState::Bound(_))
10368 ));
10369 assert_eq!(module_loop_error, None);
10370 assert!(swap
10371 .registry
10372 .get_module_by_connection(INCUMBENT)
10373 .unwrap()
10374 .is_some());
10375
10376 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
10378 .await
10379 .expect("the incumbent is told to drop the abandoned binding")
10380 .unwrap()
10381 .frame;
10382 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10383 assert_eq!(goodbye.header.channel, abandoned_channel);
10384 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10385 assert!(swap.incumbent_rx.try_recv().is_err());
10386
10387 let response = caller_task.await.unwrap();
10389 assert_eq!(response.len(), 1);
10390 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
10391 assert!(caller_rx.try_recv().is_err());
10392
10393 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
10397 let counters = handler.counters().snapshot();
10398 assert_eq!(
10399 keyed_total(&counters, "route_open_accepted_by_principal"),
10400 1
10401 );
10402 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
10403 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
10404 }
10405
10406 #[tokio::test]
10410 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
10411 let mut swap = swap_with_incumbent().await;
10412 register_candidate(&swap, None);
10413 let incumbent = cutover(&swap);
10414 swap.forwarding
10415 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
10416 .unwrap()
10417 .expect("the incumbent is still registered");
10418
10419 let client = ConnectionId::new(33);
10420 let (client_ctx, mut client_rx) = route_ctx(client);
10421 let route_handler = swap.handler.clone();
10422 let open_ctx = RouteCtx {
10423 connection_id: client,
10424 egress: client_ctx.egress.clone(),
10425 };
10426 let mut route_task = tokio::spawn(async move {
10427 route_handler
10428 .handle_control_frame(
10429 &open_ctx,
10430 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
10431 )
10432 .await
10433 .unwrap()
10434 });
10435 let bind = tokio::select! {
10436 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
10437 response = &mut route_task => {
10438 let response = response.unwrap();
10439 panic!(
10440 "post-cutover route.open was refused instead of relayed to the candidate: {}",
10441 parse_error(&response[0])["code"]
10442 );
10443 }
10444 };
10445 swap.handler
10446 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
10447 .await
10448 .unwrap();
10449 assert!(route_task.await.unwrap().is_empty());
10450 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
10451 match swap
10452 .forwarding
10453 .lookup_data_route(client, channel, epoch)
10454 .unwrap()
10455 {
10456 DataRoute::Client(DataRouteState::Bound(route)) => {
10457 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
10458 }
10459 other => panic!("expected a bound route on the candidate, got {other:?}"),
10460 }
10461 assert!(swap.incumbent_rx.try_recv().is_err());
10462 }
10463
10464 #[tokio::test]
10469 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
10470 let swap = swap_with_incumbent().await;
10471 register_candidate(&swap, Some(false));
10472 let update = Frame::build(
10473 FrameType::Request,
10474 control_flags(),
10475 0,
10476 0,
10477 55,
10478 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10479 provides: manifest("aft", PROTOCOL_VERSION).provides,
10480 capabilities: None,
10481 ready: Some(true),
10482 })
10483 .unwrap(),
10484 )
10485 .unwrap();
10486
10487 let replies = swap
10488 .handler
10489 .handle_control_frame(&swap.candidate_ctx, update)
10490 .await
10491 .unwrap();
10492
10493 assert_eq!(replies.len(), 1);
10494 assert_eq!(
10495 replies[0].header.ty,
10496 FrameType::Response,
10497 "candidate catalog.update was refused: {:?}",
10498 serde_json::from_slice::<Value>(&replies[0].body).ok()
10499 );
10500 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
10501 assert_eq!(
10502 swap.registry
10503 .get_module("aft")
10504 .unwrap()
10505 .unwrap()
10506 .connection_id,
10507 INCUMBENT
10508 );
10509 }
10510 }
10511
10512 mod swap_admission {
10516 use super::*;
10517
10518 const INCUMBENT_NONCE: &str = "incumbent-nonce";
10519 const CANDIDATE_NONCE: &str = "candidate-nonce";
10520
10521 fn handler_with_incumbent(
10522 module_id: &str,
10523 reserved: bool,
10524 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
10525 let registry = Arc::new(Registry::default());
10526 let supervisor = SupervisorHandle::new();
10527 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
10528 if reserved {
10529 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
10530 }
10531 let handler =
10532 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
10533 let incumbent = handler
10534 .handle_control(
10535 ConnectionId::new(1),
10536 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
10537 )
10538 .unwrap();
10539 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
10540 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
10541 (registry, supervisor, handler)
10542 }
10543
10544 #[test]
10550 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
10551 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
10552
10553 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
10554 let replies = handler
10555 .handle_control(
10556 ConnectionId::new(connection),
10557 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
10558 )
10559 .unwrap();
10560 assert_eq!(replies[0].header.ty, FrameType::Error);
10561 assert_eq!(
10562 parse_error(&replies[0])["code"],
10563 "swap_token_invalid",
10564 "nonce {nonce:?}"
10565 );
10566 }
10567 assert!(registry.get_candidate("aft").unwrap().is_none());
10568 assert_eq!(
10569 registry.get_module("aft").unwrap().unwrap().connection_id,
10570 ConnectionId::new(1)
10571 );
10572
10573 let admitted = handler
10576 .handle_control(
10577 ConnectionId::new(4),
10578 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
10579 )
10580 .unwrap();
10581 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
10582 assert_eq!(
10583 registry
10584 .get_candidate("aft")
10585 .unwrap()
10586 .unwrap()
10587 .connection_id,
10588 ConnectionId::new(4)
10589 );
10590 assert_eq!(
10591 registry.get_module("aft").unwrap().unwrap().connection_id,
10592 ConnectionId::new(1),
10593 "the candidate must not take the active slot"
10594 );
10595 let replayed = handler
10596 .handle_control(
10597 ConnectionId::new(5),
10598 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
10599 )
10600 .unwrap();
10601 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
10602
10603 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
10607 let squatter = handler
10608 .handle_control(
10609 ConnectionId::new(6),
10610 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
10611 )
10612 .unwrap();
10613 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
10614 assert!(
10615 registry.get_module("aft").unwrap().is_none(),
10616 "a squatter took the active slot of an id being swapped"
10617 );
10618 }
10619
10620 #[test]
10625 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
10626 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
10627
10628 let replies = handler
10629 .handle_control(
10630 ConnectionId::new(2),
10631 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10632 )
10633 .unwrap();
10634
10635 assert_eq!(
10636 replies[0].header.ty,
10637 FrameType::HelloAck,
10638 "reserved candidate refused: {:?}",
10639 serde_json::from_slice::<Value>(&replies[0].body).ok()
10640 );
10641 assert_eq!(
10642 registry
10643 .get_candidate("vault")
10644 .unwrap()
10645 .unwrap()
10646 .connection_id,
10647 ConnectionId::new(2)
10648 );
10649 }
10650
10651 #[test]
10654 fn without_an_open_swap_the_ordinary_gates_decide() {
10655 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
10656 supervisor.close_swap("vault");
10657
10658 let candidate = handler
10659 .handle_control(
10660 ConnectionId::new(2),
10661 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10662 )
10663 .unwrap();
10664 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
10665 let duplicate = handler
10666 .handle_control(
10667 ConnectionId::new(3),
10668 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
10669 )
10670 .unwrap();
10671 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10672 assert!(registry.get_candidate("vault").unwrap().is_none());
10673 }
10674 }
10675}
10676
10677#[cfg(test)]
10678mod concurrency_default_exposure_tests {
10679 use super::*;
10680
10681 fn hello_body(role_json: &str) -> Vec<u8> {
10682 format!(
10683 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
10684 )
10685 .into_bytes()
10686 }
10687
10688 fn manifest_from(body: &[u8]) -> ModuleManifest {
10689 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
10690 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
10691 .expect("manifest parses")
10692 }
10693
10694 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
10695
10696 #[test]
10697 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
10698 let body = hello_body(&format!(
10699 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
10700 ));
10701 let manifest = manifest_from(&body);
10702 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10705 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
10706 }
10707
10708 #[test]
10709 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
10710 let body = hello_body(&format!(
10711 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
10712 ));
10713 let manifest = manifest_from(&body);
10714 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10715 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10716 }
10717
10718 #[test]
10719 fn non_management_roles_are_never_reported() {
10720 let body = hello_body(
10721 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
10722 );
10723 let manifest = manifest_from(&body);
10724 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10725 }
10726}