1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24 ProviderRole,
25 },
26 session::{
27 HealthReport, ModuleControlPush, ModuleControlRequest, ModuleControlRequestFromModule,
28 ModuleControlResponse, ModuleControlResponseToModule, MODULE_CONTROL_OP_HEALTH_CHECK,
29 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
30 },
31 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
32 Priority, RouteTarget, PROTOCOL_VERSION,
33};
34use tokio::time::{timeout_at, Instant};
35use tracing::{debug, info, warn};
36
37use crate::{
38 capability_requirements::{
39 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
40 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
41 RuntimeModule,
42 },
43 daemon_config::RestartRequiredSection,
44 forwarding::{
45 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
46 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
47 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
48 },
49 observability::{
50 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
51 },
52 provenance::{
53 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
54 },
55 registry::{ChannelState, ConnectionId, Registry, RegistryError},
56 router::{RouteCtx, RouterError},
57 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
58 stderr_tail::{CaptureState, TailEntry},
59 supervise::{
60 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
61 SupervisorHandle, SwapHelloAdmission,
62 },
63 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
64};
65
66pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
72
73const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
74const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
75const CAP_PING_PONG: &str = "ping_pong_v1";
76const CAP_SESSION_ATTACH: &str = "session_attach_v1";
77const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
78
79const SUBC_CONTROL_OPS: &[&str] = &[
80 ops::SERVER_DESCRIBE,
81 ops::CATALOG_LIST,
82 ops::ROUTE_OPEN,
83 ops::ROUTE_POLL,
84 ops::ROUTE_CLOSING,
85 ops::ROUTE_CLOSED,
86 ops::SUPERVISOR_LIST,
87 ops::SUPERVISOR_RESTART,
88 ops::SUPERVISOR_SWAP,
89 ops::SUPERVISOR_RELOAD,
90 ops::SUPERVISOR_RESCAN,
91 ops::SUPERVISOR_RELEASE_RESERVED,
92 ops::SUPERVISOR_SET_ENABLED,
93 ops::SUPERVISOR_HEALTH_PROBE,
94 ops::SUPERVISOR_HEALTH,
95 ops::SUPERVISOR_STDERR_TAIL,
96 ops::SUPERVISOR_TERMINALS,
97 ops::SUPERVISOR_ROUTES,
98 ops::SUPERVISOR_PROVENANCE,
99 ops::SUPERVISOR_SPAWN_SNAPSHOT,
100 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
101];
102
103const MODULE_TO_SUBC_CONTROL_OPS: &[&str] =
104 &[MODULE_TO_SUBC_OP_CATALOG_UPDATE, "supervisor.live_roots"];
105
106const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
107
108pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
116
117pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
132
133pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
151
152const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
153const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
154
155fn reload_verdict(
156 configured: &Path,
157 spawned_from: Option<&Path>,
158 image: subc_control::RunningImageAgreement,
159) -> PendingReloadVerdict {
160 let path = match spawned_from {
161 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
162 Some(spawned_from) => ReloadPathAgreement::Mismatch {
163 configured: configured.to_path_buf(),
164 spawned_from: spawned_from.to_path_buf(),
165 },
166 None => ReloadPathAgreement::Unavailable {
167 reason: if matches!(
168 image,
169 subc_control::RunningImageAgreement::Unavailable {
170 reason: subc_control::RunningImageUnavailableReason::NotRunning
171 }
172 ) {
173 ReloadPathUnavailableReason::NotRunning
174 } else {
175 ReloadPathUnavailableReason::SpawnedPathUnavailable
176 },
177 },
178 };
179 PendingReloadVerdict { path, image }
180}
181
182#[derive(Clone)]
183struct DaemonProvenanceFacts {
184 build: DaemonBuildProvenance,
185 pid: Option<u32>,
186 started_at_ms: Option<u64>,
187 start_clock: Option<crate::clock::StartClock>,
188 executable_path: Option<PathBuf>,
189 executable_identity: Option<SpawnedFileIdentity>,
190 process_start_time: Option<u64>,
191 probe: ExecutableIdentityProbe,
192}
193
194impl Default for DaemonProvenanceFacts {
195 fn default() -> Self {
196 Self {
197 build: DaemonBuildProvenance {
198 build_git_sha: None,
199 build_lock_digest: None,
200 },
201 pid: None,
202 started_at_ms: None,
203 start_clock: None,
204 executable_path: None,
205 executable_identity: None,
206 process_start_time: None,
207 probe: ExecutableIdentityProbe::default(),
208 }
209 }
210}
211
212#[derive(Debug, Clone)]
213struct SupervisorRescanContext {
214 supervisor: Supervisor,
215 config_path: PathBuf,
216 configured_port: Option<u16>,
217 storage_config: Option<crate::daemon_config::StorageConfig>,
218 admission_facts_carrier_module_id: Option<String>,
219 admission_facts_targets: Option<Vec<String>>,
220}
221
222#[derive(Clone)]
224pub struct ControlHandler {
225 registry: Arc<Registry>,
226 forwarding: Arc<ForwardingTable>,
227 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
228 supervisor: SupervisorHandle,
229 subc_capabilities: Arc<[String]>,
230 route_bind_relay_timeout: Duration,
234 route_bind_relay_timeouts: BTreeMap<String, Duration>,
238 route_bind_breakers: RouteBindBreakers,
241 route_bind_concurrency: RouteBindConcurrency,
244 route_bind_breaker_threshold: u32,
246 route_bind_breaker_cooldown: Duration,
248 health_probe_timeout: Duration,
249 storage_config: Option<crate::daemon_config::StorageConfig>,
252 machine_id: Option<crate::machine_id::MachineId>,
256 admission_facts_carrier_module_id: Option<String>,
257 admission_facts_targets: Option<Vec<String>>,
258 rescan: Option<SupervisorRescanContext>,
259 connected_clients: ConnectedClients,
260 counters: DaemonCounters,
261 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
262 daemon_provenance: DaemonProvenanceFacts,
263 #[cfg(test)]
264 control_dispatch_delay: Option<Duration>,
265 #[cfg(test)]
266 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
267}
268
269impl fmt::Debug for ControlHandler {
270 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
271 f.debug_struct("ControlHandler")
272 .field("registry", &self.registry)
273 .field("forwarding", &self.forwarding)
274 .field("process_liveness", &self.process_liveness.is_some())
275 .field("supervisor", &self.supervisor)
276 .field("subc_capabilities", &self.subc_capabilities)
277 .finish()
278 }
279}
280
281struct RouteOpenRequest {
282 target: RouteTarget,
283 identity: BindIdentity,
284 consumer_identity: Option<ConsumerIdentity>,
285 consumer_capabilities: Option<Vec<String>>,
286 admission_facts: Option<serde_json::Value>,
287}
288
289struct RouteBindReservationGuard {
290 forwarding: Arc<ForwardingTable>,
291 endpoint: ModuleEndpointId,
292 relay_corr: u64,
293 armed: bool,
294}
295
296struct ModuleControlRpcGuard {
297 forwarding: Arc<ForwardingTable>,
298 endpoint: ModuleEndpointId,
299 corr: u64,
300 armed: bool,
301}
302
303impl ModuleControlRpcGuard {
304 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
305 Self {
306 forwarding,
307 endpoint,
308 corr,
309 armed: true,
310 }
311 }
312
313 fn disarm(&mut self) {
314 self.armed = false;
315 }
316}
317
318impl Drop for ModuleControlRpcGuard {
319 fn drop(&mut self) {
320 if self.armed {
321 let _ = self
322 .forwarding
323 .cancel_module_control_rpc(self.endpoint, self.corr);
324 }
325 }
326}
327
328impl RouteBindReservationGuard {
329 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
330 Self {
331 forwarding,
332 endpoint,
333 relay_corr,
334 armed: true,
335 }
336 }
337
338 fn release_and_disarm(&mut self) {
339 if !self.armed {
340 return;
341 }
342 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
343 self.endpoint,
344 self.relay_corr,
345 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
346 ) {
347 send_goodbye_target_best_effort(
348 &self.forwarding.counters(),
349 &target,
350 "canceled route.bind",
351 );
352 }
353 self.armed = false;
354 }
355
356 fn disarm(&mut self) {
357 self.armed = false;
358 }
359}
360
361impl Drop for RouteBindReservationGuard {
362 fn drop(&mut self) {
363 self.release_and_disarm();
364 }
365}
366
367#[derive(Debug, Clone, Default)]
397pub(crate) struct RouteBindBreakers {
398 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
399}
400
401#[derive(Debug, Clone, Default)]
402pub(crate) struct RouteBindConcurrency {
403 modules: Arc<Mutex<HashMap<String, usize>>>,
404}
405
406struct RouteBindConcurrencyGuard {
407 concurrency: RouteBindConcurrency,
408 module_id: String,
409}
410
411impl RouteBindConcurrency {
412 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
415 let mut modules = self
416 .modules
417 .lock()
418 .expect("route.bind concurrency mutex poisoned");
419 let in_flight = modules.entry(module_id.to_string()).or_default();
420 if *in_flight >= limit {
421 return Err(*in_flight);
422 }
423 *in_flight += 1;
424 Ok(RouteBindConcurrencyGuard {
425 concurrency: self.clone(),
426 module_id: module_id.to_string(),
427 })
428 }
429}
430
431impl Drop for RouteBindConcurrencyGuard {
432 fn drop(&mut self) {
433 let mut modules = self
434 .concurrency
435 .modules
436 .lock()
437 .expect("route.bind concurrency mutex poisoned");
438 let remove = {
439 let in_flight = modules
440 .get_mut(&self.module_id)
441 .expect("admitted route.bind has a concurrency entry");
442 *in_flight -= 1;
443 *in_flight == 0
444 };
445 if remove {
446 modules.remove(&self.module_id);
447 }
448 }
449}
450
451#[derive(Debug, Default)]
452struct ModuleBreakerState {
453 consecutive_timeouts: u32,
455 cooldown_until: Option<Instant>,
458 probe_in_flight: bool,
463}
464
465enum RouteBindAdmission<'a> {
467 Admitted {
468 guard: RouteBindBreakerGuard<'a>,
469 probe: bool,
472 },
473 Refused {
474 consecutive_timeouts: u32,
475 retry_in: Duration,
479 probe_in_flight: bool,
480 },
481}
482
483struct RouteBindBreakerGuard<'a> {
490 breakers: RouteBindBreakers,
491 module_id: &'a str,
492 settled: bool,
493}
494
495impl RouteBindBreakerGuard<'_> {
496 fn record_accepted(&mut self) -> bool {
500 self.settled = true;
501 self.breakers.record_accepted(self.module_id)
502 }
503
504 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
507 self.settled = true;
508 self.breakers
509 .record_timeout(self.module_id, threshold, cooldown)
510 }
511
512 fn record_inconclusive(&mut self) {
521 self.settled = true;
522 self.breakers.record_inconclusive(self.module_id);
523 }
524}
525
526impl Drop for RouteBindBreakerGuard<'_> {
527 fn drop(&mut self) {
528 if !self.settled {
529 self.breakers.record_inconclusive(self.module_id);
530 }
531 }
532}
533
534struct BreakerOpened {
538 consecutive_timeouts: u32,
539 reopened_after_probe: bool,
542}
543
544impl RouteBindBreakers {
545 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
546 self.modules
547 .lock()
548 .expect("route.bind breaker mutex poisoned")
549 }
550
551 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
554 let admitted = |probe| RouteBindAdmission::Admitted {
555 guard: RouteBindBreakerGuard {
556 breakers: self.clone(),
557 module_id,
558 settled: false,
559 },
560 probe,
561 };
562
563 let mut modules = self.lock();
564 let Some(state) = modules.get_mut(module_id) else {
565 return admitted(false);
566 };
567 let Some(cooldown_until) = state.cooldown_until else {
568 return admitted(false);
569 };
570 if state.probe_in_flight {
571 return RouteBindAdmission::Refused {
572 consecutive_timeouts: state.consecutive_timeouts,
573 retry_in: Duration::ZERO,
574 probe_in_flight: true,
575 };
576 }
577 let now = Instant::now();
578 if now < cooldown_until {
579 return RouteBindAdmission::Refused {
580 consecutive_timeouts: state.consecutive_timeouts,
581 retry_in: cooldown_until - now,
582 probe_in_flight: false,
583 };
584 }
585 state.probe_in_flight = true;
586 admitted(true)
587 }
588
589 fn record_accepted(&self, module_id: &str) -> bool {
590 self.lock()
591 .remove(module_id)
592 .is_some_and(|state| state.cooldown_until.is_some())
593 }
594
595 fn record_timeout(
596 &self,
597 module_id: &str,
598 threshold: u32,
599 cooldown: Duration,
600 ) -> Option<BreakerOpened> {
601 let mut modules = self.lock();
602 let state = modules.entry(module_id.to_string()).or_default();
603 let was_open = state.cooldown_until.is_some();
604 let was_probe = state.probe_in_flight;
605 state.probe_in_flight = false;
606 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
607 if state.consecutive_timeouts < threshold {
608 return None;
609 }
610 state.cooldown_until = Some(Instant::now() + cooldown);
611 Some(BreakerOpened {
612 consecutive_timeouts: state.consecutive_timeouts,
613 reopened_after_probe: was_open && was_probe,
614 })
615 }
616
617 fn record_inconclusive(&self, module_id: &str) {
618 if let Some(state) = self.lock().get_mut(module_id) {
619 state.probe_in_flight = false;
620 }
621 }
622
623 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
638 self.lock()
639 .remove(module_id)
640 .map(|state| state.consecutive_timeouts)
641 .filter(|discarded| *discarded > 0)
642 }
643
644 fn open_snapshot(&self) -> Option<serde_json::Value> {
651 let now = Instant::now();
652 let modules = self.lock();
653 let open = modules
654 .iter()
655 .filter_map(|(module_id, state)| {
656 let cooldown_until = state.cooldown_until?;
657 Some((
658 module_id.clone(),
659 serde_json::json!({
660 "consecutive_timeouts": state.consecutive_timeouts,
661 "cooldown_remaining_ms":
662 cooldown_until.saturating_duration_since(now).as_millis() as u64,
663 "probe_in_flight": state.probe_in_flight,
664 }),
665 ))
666 })
667 .collect::<serde_json::Map<String, serde_json::Value>>();
668 (!open.is_empty()).then_some(serde_json::Value::Object(open))
669 }
670}
671
672impl ControlHandler {
673 pub fn new(registry: Arc<Registry>) -> Self {
674 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
675 }
676
677 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
678 let counters = forwarding.counters();
679 let route_bind_breakers = forwarding.route_bind_breakers();
683 let route_bind_concurrency = forwarding.route_bind_concurrency();
684 Self {
685 registry,
686 forwarding,
687 process_liveness: None,
688 supervisor: SupervisorHandle::new(),
689 subc_capabilities: Arc::from([
690 CAP_MANIFEST_REGISTRATION.to_string(),
691 CAP_CHANNEL_LIFECYCLE.to_string(),
692 CAP_PING_PONG.to_string(),
693 CAP_SESSION_ATTACH.to_string(),
694 CAP_ADMISSION_FACTS_RELAY.to_string(),
695 ]),
696 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
697 route_bind_relay_timeouts: BTreeMap::new(),
698 route_bind_breakers,
699 route_bind_concurrency,
700 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
701 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
702 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
703 storage_config: None,
704 machine_id: None,
705 admission_facts_carrier_module_id: None,
706 admission_facts_targets: None,
707 rescan: None,
708 connected_clients: ConnectedClients::new(),
709 counters,
710 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
711 daemon_provenance: DaemonProvenanceFacts::default(),
712 #[cfg(test)]
713 control_dispatch_delay: None,
714 #[cfg(test)]
715 provenance_probe_override: None,
716 }
717 }
718
719 pub fn with_storage_config(
722 mut self,
723 storage_config: Option<crate::daemon_config::StorageConfig>,
724 ) -> Self {
725 self.storage_config = storage_config;
726 self
727 }
728
729 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
732 self.machine_id = machine_id;
733 self
734 }
735
736 pub fn with_admission_facts_config(
740 mut self,
741 carrier_module_id: Option<String>,
742 targets: Option<Vec<String>>,
743 ) -> Self {
744 self.admission_facts_carrier_module_id = carrier_module_id;
745 self.admission_facts_targets = targets;
746 self
747 }
748
749 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
752 self.route_bind_relay_timeout = timeout;
753 self
754 }
755
756 pub fn with_route_bind_relay_timeouts(
762 mut self,
763 timeouts: impl IntoIterator<Item = (String, Duration)>,
764 ) -> Self {
765 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
766 self
767 }
768
769 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
775 self.route_bind_relay_timeouts
776 .get(module_id)
777 .copied()
778 .unwrap_or(self.route_bind_relay_timeout)
779 }
780
781 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
789 self.route_bind_breaker_threshold = threshold.max(1);
790 self.route_bind_breaker_cooldown = cooldown;
791 self
792 }
793
794 #[cfg(test)]
795 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
796 self.health_probe_timeout = timeout;
797 self
798 }
799
800 #[cfg(test)]
801 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
802 self.control_dispatch_delay = Some(delay);
803 self
804 }
805
806 pub fn with_process_liveness(
807 mut self,
808 process_liveness: Arc<dyn ModuleProcessLiveness>,
809 ) -> Self {
810 self.process_liveness = Some(process_liveness);
811 self
812 }
813
814 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
815 self.supervisor = supervisor;
816 self
817 }
818
819 pub fn with_daemon_provenance(
820 mut self,
821 pid: u32,
822 started_at_ms: u64,
823 executable_path: Option<PathBuf>,
824 build_git_sha: Option<String>,
825 build_lock_digest: Option<String>,
826 ) -> Self {
827 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
828 let process_start_time = process_start_time(pid);
829 self.daemon_provenance = DaemonProvenanceFacts {
830 build: DaemonBuildProvenance {
831 build_git_sha,
832 build_lock_digest,
833 },
834 pid: Some(pid),
835 started_at_ms: Some(started_at_ms),
836 start_clock: None,
837 executable_path,
838 executable_identity,
839 process_start_time,
840 probe: ExecutableIdentityProbe::default(),
841 };
842 self
843 }
844
845 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
846 self.daemon_provenance.start_clock = Some(clock);
847 self
848 }
849
850 #[cfg(test)]
851 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
852 self.provenance_probe_override = Some(result);
853 self
854 }
855
856 pub fn with_capability_config(
860 self,
861 modules: impl IntoIterator<Item = (String, bool)>,
862 reserved_capabilities: BTreeMap<String, String>,
863 ) -> Self {
864 self.capability_evaluator
865 .configure(modules, reserved_capabilities);
866 self
867 }
868
869 pub fn with_supervisor_rescan(
870 mut self,
871 supervisor: Supervisor,
872 config_path: impl Into<PathBuf>,
873 configured_port: Option<u16>,
874 ) -> Self {
875 self.rescan = Some(SupervisorRescanContext {
876 supervisor,
877 config_path: config_path.into(),
878 configured_port,
879 storage_config: self.storage_config.clone(),
880 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
881 admission_facts_targets: self.admission_facts_targets.clone(),
882 });
883 self
884 }
885
886 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
887 self.connected_clients = connected_clients;
888 self
889 }
890
891 pub fn forwarding(&self) -> Arc<ForwardingTable> {
892 Arc::clone(&self.forwarding)
893 }
894
895 pub(crate) fn counters(&self) -> DaemonCounters {
896 self.counters.clone()
897 }
898
899 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
902 tokio::spawn(async move {
903 loop {
904 self.capability_evaluator
905 .wait_for_change_or_deadline()
906 .await;
907 self.refresh_capability_requirements();
908 }
909 });
910 }
911
912 fn runtime_capability_snapshot(
913 &self,
914 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
915 let runtime = self
916 .supervisor
917 .list()
918 .into_iter()
919 .map(|module| {
920 let status = module.status().map_err(|err| {
921 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
922 })?;
923 Ok(RuntimeModule {
924 module_id: status.module_id,
925 state: status.state,
926 enabled: status.enabled,
927 })
928 })
929 .collect::<Result<Vec<_>, RouterError>>()?;
930 let (_, registrations) = self.registry.list_modules().map_err(|err| {
931 RouterError::backend(
932 0,
933 0,
934 format!("failed to list capability registrations: {err}"),
935 )
936 })?;
937 let registrations = registrations
938 .into_iter()
939 .map(|registration| RegisteredModule {
940 module_id: registration.manifest.module_id,
941 module_version: registration.manifest.module_version,
942 capabilities: registration.manifest.capabilities,
943 })
944 .collect();
945 Ok((runtime, registrations))
946 }
947
948 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
955 let cached_registration = RegisteredModule {
956 module_id: registration.manifest.module_id.clone(),
957 module_version: registration.manifest.module_version.clone(),
958 capabilities: registration.manifest.capabilities.clone(),
959 };
960 if self.capability_evaluator.record_hello(&cached_registration) {
961 warn!(
962 module_id = %cached_registration.module_id,
963 "capability claims drifted from the cached manifest"
964 );
965 }
966 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
967 self.enforce_capability_denies();
968 }
969 self.refresh_capability_requirements();
970 }
971
972 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
976 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
977 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
978 self.supervisor.set_swap_promotion_observer(observer);
979 }
980
981 pub fn refresh_capability_requirements(&self) {
982 match self.runtime_capability_snapshot() {
983 Ok((runtime, registrations)) => {
984 log_requirement_events(
985 self.capability_evaluator
986 .evaluate_now(&runtime, ®istrations),
987 );
988 }
989 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
990 }
991 }
992
993 fn enforce_capability_denies(&self) {
997 let (_, registrations) = match self.registry.list_modules() {
998 Ok(snapshot) => snapshot,
999 Err(err) => {
1000 warn!(error = %err, "failed to read registrations for capability deny census");
1001 return;
1002 }
1003 };
1004 let manifests = registrations
1005 .into_iter()
1006 .map(|registration| {
1007 (
1008 registration.manifest.module_id.clone(),
1009 registration.manifest,
1010 )
1011 })
1012 .collect::<BTreeMap<_, _>>();
1013 let census = match self.forwarding.route_census(None) {
1014 Ok(census) => census,
1015 Err(err) => {
1016 warn!(error = %err, "failed to read route census for capability deny enforcement");
1017 return;
1018 }
1019 };
1020
1021 for (target_module_id, routes) in census {
1022 let Some(target_manifest) = manifests.get(&target_module_id) else {
1023 continue;
1024 };
1025 let mut closed_routes = Vec::new();
1026 let mut module_goodbyes = Vec::new();
1027 for route in routes {
1028 let Principal::Reserved {
1029 module_id: opening_module_id,
1030 } = &route.principal
1031 else {
1032 continue;
1033 };
1034 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1035 continue;
1036 };
1037 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1038 continue;
1039 };
1040
1041 match self.forwarding.release_client_route(
1042 route.goodbye_target.connection_id,
1043 route.goodbye_target.channel,
1044 route.goodbye_target.epoch,
1045 ) {
1046 Ok(RouteRelease::Removed(module_goodbye)) => {
1047 warn!(
1048 opening_module_id,
1049 target_module_id,
1050 capability,
1051 "force-closing route because an attested capability deny edge now matches"
1052 );
1053 closed_routes.push(route);
1054 module_goodbyes.push(module_goodbye);
1055 }
1056 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1057 Err(err) => warn!(
1058 opening_module_id,
1059 target_module_id,
1060 capability,
1061 error = %err,
1062 "failed to force-close capability-denied route"
1063 ),
1064 }
1065 }
1066
1067 if closed_routes.is_empty() {
1068 continue;
1069 }
1070 send_route_control_pushes(
1071 &self.forwarding,
1072 closed_routes,
1073 ClientControlPush::RouteClosed {
1074 module_id: target_module_id,
1075 reason: RouteCloseReason::CapabilityDenied,
1076 drained: false,
1077 abandoned: 0,
1078 excluded_subscriptions: 0,
1079 terminal: Some(false),
1080 },
1081 );
1082 self.emit_route_goodbyes(module_goodbyes);
1083 }
1084 }
1085
1086 fn not_ready_reason(
1092 &self,
1093 registration: &crate::registry::ModuleRegistration,
1094 ) -> Option<NotReadyReason> {
1095 if !registration.ready {
1096 return Some(NotReadyReason {
1097 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1098 capability: None,
1099 });
1100 }
1101 self.first_unprovided_required_capability(registration)
1102 .map(|capability| NotReadyReason {
1103 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1104 capability: Some(capability),
1105 })
1106 }
1107
1108 fn first_unprovided_required_capability(
1126 &self,
1127 registration: &crate::registry::ModuleRegistration,
1128 ) -> Option<String> {
1129 let required = registration
1130 .manifest
1131 .capabilities
1132 .iter()
1133 .flat_map(|declarations| declarations.requires.iter())
1134 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1135 .map(|requirement| requirement.capability.as_str())
1136 .collect::<BTreeSet<_>>();
1137 if required.is_empty() {
1138 return None;
1139 }
1140 let module_id = registration.manifest.module_id.as_str();
1141 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1142 if required
1143 .iter()
1144 .any(|capability| verdict(capability).is_none())
1145 {
1146 self.refresh_capability_requirements();
1147 }
1148 required
1149 .into_iter()
1150 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1151 .map(str::to_string)
1152 }
1153
1154 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1155 self.capability_evaluator
1156 .statuses()
1157 .into_iter()
1158 .map(capability_requirement_status)
1159 .collect()
1160 }
1161
1162 fn deregister_connection(
1169 &self,
1170 connection_id: ConnectionId,
1171 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1172 self.registry.deregister_connection(connection_id)
1173 }
1174
1175 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1176 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1177 return None;
1178 }
1179 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1180 parse_client_control_request(&frame.body)
1181 else {
1182 return None;
1183 };
1184 Some(target_module_id(&target).to_string())
1185 }
1186
1187 pub(crate) fn route_open_capacity_refusal(
1188 &self,
1189 ctx: &RouteCtx,
1190 frame: &Frame,
1191 target_module_id: &str,
1192 in_flight: usize,
1193 limit: usize,
1194 ) -> Result<Frame, RouterError> {
1195 self.route_open_admission_refusal_frame(
1196 ctx,
1197 frame,
1198 target_module_id,
1199 "open_admission_full",
1200 (in_flight, limit),
1201 format!(
1202 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1203 ),
1204 )
1205 }
1206
1207 fn route_open_target_capacity_refusal(
1208 &self,
1209 ctx: &RouteCtx,
1210 frame: &Frame,
1211 target_module_id: &str,
1212 in_flight: usize,
1213 ) -> Result<Frame, RouterError> {
1214 self.route_open_admission_refusal_frame(
1215 ctx,
1216 frame,
1217 target_module_id,
1218 "target_binds_full",
1219 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1220 format!(
1221 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1222 ),
1223 )
1224 }
1225
1226 fn route_open_admission_refusal_frame(
1234 &self,
1235 ctx: &RouteCtx,
1236 frame: &Frame,
1237 target_module_id: &str,
1238 reason: &'static str,
1239 (in_flight, limit): (usize, usize),
1240 message: impl Into<String>,
1241 ) -> Result<Frame, RouterError> {
1242 let code = error_codes::TARGET_UNAVAILABLE;
1243 self.counters.increment_route_open_refused(code);
1244 info!(
1245 target: "control",
1246 code,
1247 reason,
1248 module_id = ?target_module_id,
1249 connection_id = ctx.connection_id.get(),
1250 in_flight,
1251 limit,
1252 "route.open refused"
1253 );
1254 control_error_frame(frame, code, message.into())
1255 }
1256
1257 #[cfg(test)]
1263 pub fn handle_control(
1264 &self,
1265 connection_id: ConnectionId,
1266 frame: Frame,
1267 ) -> Result<Vec<Frame>, RouterError> {
1268 match frame.header.ty {
1269 FrameType::Ping => Ok(vec![pong(&frame)?]),
1270 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1271 FrameType::Goodbye => self.handle_goodbye(connection_id),
1272 ty => Ok(vec![control_error_frame(
1273 &frame,
1274 "unsupported_control_frame",
1275 format!("unsupported channel-0 frame {ty:?}"),
1276 )?]),
1277 }
1278 }
1279
1280 pub async fn handle_control_frame(
1281 &self,
1282 ctx: &RouteCtx,
1283 frame: Frame,
1284 ) -> Result<Vec<Frame>, RouterError> {
1285 self.handle_control_frame_timed(ctx, frame, None).await
1286 }
1287
1288 pub(crate) async fn handle_control_frame_timed(
1289 &self,
1290 ctx: &RouteCtx,
1291 frame: Frame,
1292 dispatch_started_at: Option<StdInstant>,
1293 ) -> Result<Vec<Frame>, RouterError> {
1294 match frame.header.ty {
1295 FrameType::Ping => Ok(vec![pong(&frame)?]),
1296 FrameType::Hello => {
1297 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1298 }
1299 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1300 FrameType::Cancel => {
1301 if self
1302 .supervisor
1303 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1304 {
1305 Ok(Vec::new())
1306 } else {
1307 Ok(vec![control_error_frame(
1308 &frame,
1309 "unknown_subscription",
1310 "no supervisor spawn subscription has this correlation id",
1311 )?])
1312 }
1313 }
1314 FrameType::Request => {
1315 if self
1316 .forwarding
1317 .module_endpoint_for_connection(ctx.connection_id)
1318 .map_err(RouterError::Forwarding)?
1319 .is_some()
1320 {
1321 if !is_known_module_request_op(&frame.body) {
1322 return Ok(vec![control_error_frame(
1323 &frame,
1324 "unsupported_control_frame",
1325 "module-originated channel-0 REQUEST is not supported",
1326 )?]);
1327 }
1328 let request = match parse_module_control_request_from_module(&frame.body) {
1329 Ok(request) => request,
1330 Err((err, ControlRequestBodyError::UnknownOp)) => {
1331 return Ok(vec![control_error_frame(
1332 &frame,
1333 "unsupported_control_frame",
1334 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1335 )?])
1336 }
1337 Err((err, ControlRequestBodyError::InvalidBody)) => {
1338 return Ok(vec![control_error_frame(
1339 &frame,
1340 "invalid_control_body",
1341 format!("malformed module control body: {err}"),
1342 )?])
1343 }
1344 };
1345 let op = module_control_request_op(&request);
1346 let corr = frame.header.corr;
1347 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1348 let result =
1349 self.handle_module_control_request(ctx.connection_id, frame, request);
1350 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1351 return result;
1352 }
1353
1354 if is_known_module_request_op(&frame.body) {
1355 return Ok(vec![control_error_frame(
1356 &frame,
1357 "not_registered",
1358 "catalog.update requires an active module registration owned by this connection",
1359 )?]);
1360 }
1361
1362 let request = match parse_client_control_request(&frame.body) {
1363 Ok(request) => request,
1364 Err((err, ControlRequestBodyError::UnknownOp)) => {
1365 return Ok(vec![control_error_frame(
1366 &frame,
1367 "unknown_control_op",
1368 format!("unknown client control op: {err}"),
1369 )?])
1370 }
1371 Err((err, ControlRequestBodyError::InvalidBody)) => {
1372 return Ok(vec![control_error_frame(
1373 &frame,
1374 "invalid_control_body",
1375 format!("malformed client control body: {err}"),
1376 )?])
1377 }
1378 };
1379 let op = client_control_request_op(&request);
1380 let corr = frame.header.corr;
1381 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1382 #[cfg(test)]
1383 if let Some(delay) = self.control_dispatch_delay {
1384 tokio::time::sleep(delay).await;
1385 }
1386 let result = self
1387 .handle_client_control_request(ctx, frame, request)
1388 .await;
1389 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1390 result
1391 }
1392 FrameType::Push => {
1393 let Some(endpoint) = self
1394 .forwarding
1395 .module_endpoint_for_connection(ctx.connection_id)
1396 .map_err(RouterError::Forwarding)?
1397 else {
1398 return Ok(vec![control_error_frame(
1399 &frame,
1400 "unsupported_control_frame",
1401 "client-originated channel-0 PUSH is not supported",
1402 )?]);
1403 };
1404 self.handle_status_update(endpoint, frame)
1405 }
1406 FrameType::Response | FrameType::Error
1407 if self
1408 .forwarding
1409 .module_endpoint_for_connection(ctx.connection_id)
1410 .map_err(RouterError::Forwarding)?
1411 .is_some() =>
1412 {
1413 self.handle_module_relay_response(ctx.connection_id, frame)
1414 }
1415 ty => Ok(vec![control_error_frame(
1416 &frame,
1417 "unsupported_control_frame",
1418 format!("unsupported channel-0 frame {ty:?}"),
1419 )?]),
1420 }
1421 }
1422
1423 pub fn cleanup_connection(
1424 &self,
1425 connection_id: ConnectionId,
1426 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1427 let crash_closed = self
1428 .registry
1429 .get_module_by_connection(connection_id)?
1430 .and_then(|registration| {
1431 self.forwarding
1432 .module_endpoint_for_connection(connection_id)
1433 .ok()
1434 .flatten()
1435 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1436 .map(|routes| (registration.manifest.module_id, routes))
1437 });
1438 let crash_closed = crash_closed.map(|(module_id, routes)| {
1439 let terminal = match self.supervisor.get(&module_id) {
1440 None => false,
1441 Some(module) => match module.will_recover_after_connection_loss() {
1442 Ok(will_recover) => !will_recover,
1443 Err(err) => {
1444 warn!(
1445 %module_id,
1446 error = %err,
1447 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1448 );
1449 false
1450 }
1451 },
1452 };
1453 let reason = match self.forwarding.is_daemon_draining() {
1457 Ok(true) => RouteCloseReason::Restart,
1458 Ok(false) => RouteCloseReason::Crash,
1459 Err(err) => {
1460 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1461 RouteCloseReason::Crash
1462 }
1463 };
1464 (module_id, routes, reason, terminal)
1465 });
1466 let registrations = self.deregister_connection(connection_id);
1467 let cleanup = self.forwarding.cleanup_connection_counted(connection_id);
1468 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1473 let abandoned = cleanup
1474 .as_ref()
1475 .map_or(0, |cleanup| cleanup.abandoned_relays);
1476 send_route_control_pushes(
1477 &self.forwarding,
1478 routes,
1479 ClientControlPush::RouteClosed {
1480 module_id,
1481 reason,
1482 drained: false,
1483 abandoned,
1484 excluded_subscriptions: 0,
1485 terminal: Some(terminal),
1486 },
1487 );
1488 }
1489 if let Ok(cleanup) = cleanup {
1490 self.emit_route_goodbyes(cleanup.released);
1491 }
1492 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1496 crate::supervise::notify_registration_release();
1497 self.capability_evaluator.wake_deadline_loop();
1498 self.refresh_capability_requirements();
1499 }
1500 self.supervisor.remove_spawn_subscribers(connection_id);
1501 registrations
1502 }
1503
1504 pub(crate) fn handle_route_goodbye(
1505 &self,
1506 connection_id: ConnectionId,
1507 route_channel: u16,
1508 route_epoch: u32,
1509 ) -> Result<bool, RouterError> {
1510 debug!(
1511 connection_id = connection_id.get(),
1512 route_channel, route_epoch, "handling route GOODBYE"
1513 );
1514 let RouteRelease::Removed(released_route) = self
1515 .forwarding
1516 .release_client_route(connection_id, route_channel, route_epoch)
1517 .map_err(RouterError::Forwarding)?
1518 else {
1519 return Ok(false);
1520 };
1521 self.emit_route_goodbyes(vec![released_route]);
1522 Ok(true)
1523 }
1524
1525 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1526 for released in released_routes {
1527 let frame = match Frame::build_with_version(
1528 released.negotiated_ver,
1529 FrameType::Goodbye,
1530 control_flags(),
1531 released.channel,
1532 released.epoch,
1533 0,
1534 Vec::new(),
1535 ) {
1536 Ok(frame) => frame,
1537 Err(err) => {
1538 warn!(
1539 route_channel = released.channel,
1540 error = %err,
1541 "failed to build route GOODBYE frame"
1542 );
1543 continue;
1544 }
1545 };
1546 if !released.close_on_delivery_failure() {
1547 crate::forwarding::send_module_route_goodbye(
1548 &self.counters,
1549 &released.sink,
1550 frame,
1551 released.module_id.as_deref(),
1552 "client route released",
1553 );
1554 continue;
1555 }
1556 if let Err(err) = released.sink.try_send(frame) {
1557 warn!(
1558 target_connection_id = released.connection_id.get(),
1559 route_channel = released.channel,
1560 error = %err,
1561 "route GOODBYE was not delivered to client; closing target connection"
1562 );
1563 if self
1564 .forwarding
1565 .escalate_client_delivery_failure(
1566 released.connection_id,
1567 released.channel,
1568 released.epoch,
1569 CloseReason::new(
1570 "route_goodbye_delivery_failed",
1571 format!(
1572 "failed to enqueue route GOODBYE for channel {}: {err}",
1573 released.channel
1574 ),
1575 ),
1576 crate::forwarding::UndeliveredFrame {
1577 module_id: released.module_id.as_deref(),
1578 sink: &released.sink,
1579 },
1580 )
1581 .unwrap_or(false)
1582 {
1583 self.counters.increment_goodbye_relay_client_failed();
1584 }
1585 }
1586 }
1587 }
1588
1589 fn send_abandoned_route_bind_goodbye(
1601 &self,
1602 module_sink: &crate::FrameSink,
1603 negotiated_ver: u8,
1604 module_channel: u16,
1605 module_epoch: u32,
1606 ) {
1607 let frame = match Frame::build_with_version(
1608 negotiated_ver,
1609 FrameType::Goodbye,
1610 control_flags(),
1611 module_channel,
1612 module_epoch,
1613 0,
1614 Vec::new(),
1615 ) {
1616 Ok(frame) => frame,
1617 Err(err) => {
1618 warn!(
1619 route_channel = module_channel,
1620 error = %err,
1621 "failed to build GOODBYE for abandoned route.bind"
1622 );
1623 return;
1624 }
1625 };
1626 crate::forwarding::send_module_route_goodbye(
1627 &self.counters,
1628 module_sink,
1629 frame,
1630 None,
1631 "abandoned route.bind",
1632 );
1633 }
1634
1635 fn handle_hello(
1636 &self,
1637 connection_id: ConnectionId,
1638 sink: Option<crate::FrameSink>,
1639 frame: Frame,
1640 ) -> Result<Vec<Frame>, RouterError> {
1641 debug!(
1642 connection_id = connection_id.get(),
1643 corr = frame.header.corr,
1644 "handling HELLO"
1645 );
1646 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1647 Ok(value) => value,
1648 Err(err) => {
1649 return Ok(vec![control_error_frame(
1650 &frame,
1651 "invalid_hello",
1652 format!("malformed HELLO body: {err}"),
1653 )?])
1654 }
1655 };
1656 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1657 return Ok(vec![control_error_frame(
1658 &frame,
1659 "invalid_capability_grammar",
1660 err.to_string(),
1661 )?]);
1662 }
1663 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1664 return Ok(vec![control_error_frame(
1665 &frame,
1666 "invalid_manifest",
1667 err.to_string(),
1668 )?]);
1669 }
1670 if let Some(provenance) = hello_value
1671 .get("manifest")
1672 .and_then(|manifest| manifest.get("provenance"))
1673 {
1674 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1675 return Ok(vec![control_error_frame(
1676 &frame,
1677 "invalid_manifest",
1678 format!("malformed manifest provenance: {err}"),
1679 )?]);
1680 }
1681 }
1682 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1683 Ok(hello) => hello,
1684 Err(err) => {
1685 return Ok(vec![control_error_frame(
1686 &frame,
1687 "invalid_hello",
1688 format!("malformed HELLO body: {err}"),
1689 )?])
1690 }
1691 };
1692
1693 if hello.protocol_ver != hello.manifest.protocol_ver {
1694 return Ok(vec![control_error_frame(
1695 &frame,
1696 "invalid_manifest",
1697 format!(
1698 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1699 hello.protocol_ver, hello.manifest.protocol_ver
1700 ),
1701 )?]);
1702 }
1703
1704 if hello.manifest.module_id.trim().is_empty() {
1705 return Ok(vec![control_error_frame(
1706 &frame,
1707 "invalid_manifest",
1708 "manifest module_id must not be empty",
1709 )?]);
1710 }
1711
1712 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1713 Ok(negotiated_ver) => negotiated_ver,
1714 Err(message) => {
1715 return Ok(vec![control_error_frame(
1716 &frame,
1717 "version_unsupported",
1718 message,
1719 )?])
1720 }
1721 };
1722
1723 let swap_admission = self
1731 .supervisor
1732 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1733 if swap_admission == SwapHelloAdmission::Refused {
1734 warn!(
1735 module_id = %hello.manifest.module_id,
1736 connection_id = connection_id.get(),
1737 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1738 );
1739 return Ok(vec![control_error_frame(
1740 &frame,
1741 "swap_token_invalid",
1742 format!(
1743 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1744 hello.manifest.module_id
1745 ),
1746 )?]);
1747 }
1748 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1749
1750 if let Some(rejection) = (!swap_candidate)
1758 .then(|| {
1759 self.supervisor.reserved_hello_rejection(
1760 &hello.manifest.module_id,
1761 hello.launch_nonce.as_deref(),
1762 )
1763 })
1764 .flatten()
1765 {
1766 let message = match rejection {
1767 ReservedHelloRejection::Exact { module_id } => format!(
1768 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1769 ),
1770 ReservedHelloRejection::Prefix {
1771 prefix,
1772 owner_module_id,
1773 } => format!(
1774 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1775 hello.manifest.module_id
1776 ),
1777 };
1778 return Ok(vec![control_error_frame(
1779 &frame,
1780 "reserved_module",
1781 message,
1782 )?]);
1783 }
1784
1785 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1786 &hello.manifest.module_id,
1787 hello.manifest.capabilities.as_ref(),
1788 );
1789 if let Some(refusal) = reserved_capability_refusals.first() {
1790 let capability = refusal.capability.clone();
1791 let bound_module = refusal.claimants[0].clone();
1792 log_duplicate_claim_events(reserved_capability_refusals);
1793 return Ok(vec![control_error_frame(
1794 &frame,
1795 "reserved_capability",
1796 format!(
1797 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1798 capability, bound_module, hello.manifest.module_id
1799 ),
1800 )?]);
1801 }
1802
1803 if self
1806 .forwarding
1807 .connection_has_client_routes(connection_id)
1808 .map_err(RouterError::Forwarding)?
1809 {
1810 return Ok(vec![control_error_frame(
1811 &frame,
1812 "invalid_hello",
1813 "connection has open client routes and cannot also register as a module",
1814 )?]);
1815 }
1816
1817 let control_ops = effective_module_control_ops(hello.control_ops);
1818 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1821 if swap_candidate {
1822 return self.register_swap_candidate(
1823 connection_id,
1824 sink,
1825 &frame,
1826 hello.manifest,
1827 negotiated_ver,
1828 control_ops,
1829 hello_ack,
1830 );
1831 }
1832 let registration = match self.registry.register_with_control_ops(
1833 hello.manifest,
1834 negotiated_ver,
1835 connection_id,
1836 control_ops,
1837 ) {
1838 Ok(registration) => registration,
1839 Err(RegistryError::DuplicateModuleId { module_id }) => {
1840 return Ok(vec![control_error_frame(
1841 &frame,
1842 "duplicate_module_id",
1843 format!(
1844 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1845 ),
1846 )?])
1847 }
1848 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1849 return Ok(vec![control_error_frame(
1850 &frame,
1851 "invalid_module_id",
1852 err.to_string(),
1853 )?])
1854 }
1855 Err(err) => {
1856 return Ok(vec![control_error_frame(
1857 &frame,
1858 "registry_error",
1859 err.to_string(),
1860 )?])
1861 }
1862 };
1863
1864 let reply = if let Some(sink) = sink {
1865 let concurrency = manifest_concurrency(®istration.manifest);
1881 if let Err(err) = self.forwarding.register_module_connection_acked(
1882 connection_id,
1883 registration.manifest.module_id.clone(),
1884 negotiated_ver,
1885 concurrency,
1886 sink,
1887 hello_ack,
1888 ) {
1889 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1893 crate::supervise::notify_registration_release();
1894 }
1895 return Ok(vec![control_error_frame(
1896 &frame,
1897 forwarding_error_code(&err),
1898 err.to_string(),
1899 )?]);
1900 }
1901 Vec::new()
1902 } else {
1903 vec![hello_ack]
1906 };
1907
1908 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
1917 info!(
1918 module_id = %registration.manifest.module_id,
1919 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
1920 );
1921 }
1922
1923 self.apply_registration_capabilities(®istration);
1924
1925 info!(
1926 module_id = %registration.manifest.module_id,
1927 module_version = %registration.manifest.module_version,
1928 negotiated_ver,
1929 routable_provider = manifest_provides_routable_role(®istration.manifest),
1930 connection_id = connection_id.get(),
1931 "module registered"
1932 );
1933
1934 Ok(reply)
1935 }
1936
1937 #[allow(clippy::too_many_arguments)]
1947 fn register_swap_candidate(
1948 &self,
1949 connection_id: ConnectionId,
1950 sink: Option<crate::FrameSink>,
1951 frame: &Frame,
1952 manifest: ModuleManifest,
1953 negotiated_ver: u8,
1954 control_ops: Vec<String>,
1955 hello_ack: Frame,
1956 ) -> Result<Vec<Frame>, RouterError> {
1957 let module_id = manifest.module_id.clone();
1958 let registration = match self.registry.register_candidate_with_control_ops(
1959 manifest,
1960 negotiated_ver,
1961 connection_id,
1962 control_ops,
1963 ) {
1964 Ok(registration) => registration,
1965 Err(RegistryError::DuplicateModuleId { module_id }) => {
1966 return Ok(vec![control_error_frame(
1967 frame,
1968 "duplicate_module_id",
1969 format!(
1970 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
1971 ),
1972 )?])
1973 }
1974 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1975 return Ok(vec![control_error_frame(
1976 frame,
1977 "invalid_module_id",
1978 err.to_string(),
1979 )?])
1980 }
1981 Err(err) => {
1982 return Ok(vec![control_error_frame(
1983 frame,
1984 "registry_error",
1985 err.to_string(),
1986 )?])
1987 }
1988 };
1989 let reply = if let Some(sink) = sink {
1990 let concurrency = manifest_concurrency(®istration.manifest);
1994 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
1995 connection_id,
1996 module_id.clone(),
1997 negotiated_ver,
1998 concurrency,
1999 sink,
2000 hello_ack,
2001 ) {
2002 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2003 crate::supervise::notify_registration_release();
2004 }
2005 return Ok(vec![control_error_frame(
2006 frame,
2007 forwarding_error_code(&err),
2008 err.to_string(),
2009 )?]);
2010 }
2011 Vec::new()
2012 } else {
2013 vec![hello_ack]
2014 };
2015 self.supervisor.mark_swap_candidate_admitted(&module_id);
2016 info!(
2017 module_id = %module_id,
2018 module_version = %registration.manifest.module_version,
2019 negotiated_ver,
2020 ready = registration.ready,
2021 connection_id = connection_id.get(),
2022 "swap candidate registered; not routable until cutover"
2023 );
2024 Ok(reply)
2025 }
2026
2027 fn build_hello_ack(
2028 &self,
2029 frame: &Frame,
2030 negotiated_ver: u8,
2031 module_id: &str,
2032 ) -> Result<Frame, RouterError> {
2033 let ack = ModuleHelloAckBody {
2034 negotiated_ver,
2035 subc_ops: module_subc_ops(),
2036 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2037 storage: self
2038 .storage_config
2039 .as_ref()
2040 .map(|cfg| cfg.descriptor_for(module_id)),
2041 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2042 };
2043 let body = serde_json::to_vec(&ack).map_err(|err| {
2044 RouterError::backend(
2045 0,
2046 frame.header.corr,
2047 format!("failed to encode HELLO_ACK: {err}"),
2048 )
2049 })?;
2050
2051 Frame::build_with_version(
2052 negotiated_ver,
2053 FrameType::HelloAck,
2054 control_flags(),
2055 0,
2056 0,
2057 frame.header.corr,
2058 body,
2059 )
2060 .map_err(RouterError::FrameBuild)
2061 }
2062
2063 async fn handle_client_control_request(
2064 &self,
2065 ctx: &RouteCtx,
2066 frame: Frame,
2067 request: ClientControlRequest,
2068 ) -> Result<Vec<Frame>, RouterError> {
2069 match request {
2070 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2071 ClientControlRequest::CatalogList { module_id } => {
2072 self.handle_catalog_list(frame, module_id)
2073 }
2074 ClientControlRequest::RouteOpen {
2075 target,
2076 identity,
2077 consumer_identity,
2078 consumer_capabilities,
2079 admission_facts,
2080 } => {
2081 self.handle_route_open(
2082 ctx,
2083 frame,
2084 RouteOpenRequest {
2085 target,
2086 identity,
2087 consumer_identity,
2088 consumer_capabilities,
2089 admission_facts,
2090 },
2091 )
2092 .await
2093 }
2094 ClientControlRequest::RoutePoll {
2095 route_channel,
2096 route_epoch,
2097 kind,
2098 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2099 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2100 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2101 self.handle_supervisor_spawn_snapshot(frame)
2102 }
2103 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2104 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2105 }
2106 ClientControlRequest::SupervisorRestart {
2107 module_id,
2108 drain_timeout_ms,
2109 } => {
2110 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2111 .await
2112 }
2113 ClientControlRequest::SupervisorSwap {
2114 module_id,
2115 ready_timeout_ms,
2116 } => {
2117 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2118 .await
2119 }
2120 ClientControlRequest::SupervisorReload { module_id } => {
2121 self.handle_supervisor_reload(frame, module_id).await
2122 }
2123 ClientControlRequest::SupervisorRescan { preview } => {
2124 self.handle_supervisor_rescan(frame, preview).await
2125 }
2126 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2127 self.handle_supervisor_release_reserved(frame, module_id)
2128 .await
2129 }
2130 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2131 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2132 .await
2133 }
2134 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2135 self.handle_supervisor_health_probe(frame, module_id).await
2136 }
2137 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2138 ClientControlRequest::SupervisorRoutes { module_id } => {
2139 self.handle_supervisor_routes(frame, module_id)
2140 }
2141 ClientControlRequest::SupervisorProvenance { module_id } => {
2142 self.handle_supervisor_provenance(frame, module_id).await
2143 }
2144 ClientControlRequest::SupervisorStderrTail {
2145 module_id,
2146 max_lines,
2147 max_bytes,
2148 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2149 ClientControlRequest::SupervisorTerminals { module_id } => {
2150 self.handle_supervisor_terminals(frame, module_id).await
2151 }
2152 }
2153 }
2154
2155 fn handle_module_control_request(
2156 &self,
2157 connection_id: ConnectionId,
2158 frame: Frame,
2159 request: ModuleControlRequestFromModule,
2160 ) -> Result<Vec<Frame>, RouterError> {
2161 match request {
2162 ModuleControlRequestFromModule::CatalogUpdate {
2163 provides,
2164 capabilities,
2165 ready,
2166 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2167 ModuleControlRequestFromModule::LiveRoots {} => {
2168 let registered = self
2169 .registry
2170 .get_module_by_connection(connection_id)
2171 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2172 let Some(registration) = registered else {
2173 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2174 };
2175 let response = self
2176 .forwarding
2177 .live_roots(®istration.manifest.module_id)
2178 .map_err(RouterError::Forwarding)?;
2179 Ok(vec![control_response_body_frame(
2180 &frame,
2181 &response,
2182 "ModuleControlResponseToModule::LiveRoots",
2183 )?])
2184 }
2185 }
2186 }
2187
2188 fn handle_catalog_update(
2189 &self,
2190 connection_id: ConnectionId,
2191 frame: Frame,
2192 provides: Vec<ProviderRole>,
2193 capabilities: Option<CapabilityDeclarations>,
2194 ready: Option<bool>,
2195 ) -> Result<Vec<Frame>, RouterError> {
2196 self.refresh_capability_requirements();
2197 let Some(registration) = self
2198 .registry
2199 .get_module_by_connection(connection_id)
2200 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2201 else {
2202 return Ok(vec![control_error_frame(
2203 &frame,
2204 "not_registered",
2205 "catalog.update requires an active module registration owned by this connection",
2206 )?]);
2207 };
2208
2209 if let Some(message) =
2210 catalog_update_frozen_field_message(®istration.manifest, &provides)
2211 {
2212 return Ok(vec![control_error_frame(
2213 &frame,
2214 "catalog_update_frozen_field",
2215 message,
2216 )?]);
2217 }
2218
2219 let mut candidate = registration.manifest.clone();
2220 candidate.provides = provides.clone();
2221 candidate.capabilities = capabilities
2222 .clone()
2223 .or_else(|| registration.manifest.capabilities.clone());
2224 if let Err(err) = candidate.validate_capability_grammar() {
2225 return Ok(vec![control_error_frame(
2226 &frame,
2227 "invalid_capability_grammar",
2228 err.to_string(),
2229 )?]);
2230 }
2231
2232 let updated = self
2233 .registry
2234 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2235 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2236 if updated.is_none() {
2237 return Ok(vec![control_error_frame(
2238 &frame,
2239 "not_registered",
2240 "catalog.update requires an active module registration owned by this connection",
2241 )?]);
2242 }
2243 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2244 log_duplicate_claim_events(
2245 self.capability_evaluator
2246 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2247 );
2248 }
2249 if capability_census_trigger(
2250 registration.manifest.capabilities.as_ref(),
2251 updated
2252 .as_ref()
2253 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2254 ) {
2255 self.enforce_capability_denies();
2256 }
2257 self.refresh_capability_requirements();
2258
2259 let response = ModuleControlResponseToModule::CatalogUpdate {};
2260 control_response_body_frame(
2261 &frame,
2262 &response,
2263 "ModuleControlResponseToModule::CatalogUpdate",
2264 )
2265 .map(|frame| vec![frame])
2266 }
2267
2268 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2269 self.refresh_capability_requirements();
2270 let mut counters = self.counters.snapshot();
2276 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2277 self.forwarding.client_route_concentration(),
2278 counters.as_object_mut(),
2279 ) {
2280 obj.insert(
2281 "client_connections_with_routes".into(),
2282 connections_with_routes.into(),
2283 );
2284 obj.insert("max_routes_on_one_connection".into(), max.into());
2285 }
2286 if let (Some(open_breakers), Some(obj)) = (
2292 self.route_bind_breakers.open_snapshot(),
2293 counters.as_object_mut(),
2294 ) {
2295 obj.insert("route_bind_breakers_open".into(), open_breakers);
2296 }
2297 let response = ClientControlResponse::ServerDescribe {
2298 protocol_ver: PROTOCOL_VERSION,
2299 subc_ops: subc_ops(),
2300 capabilities: self.subc_capabilities.as_ref().to_vec(),
2301 connected_clients: self.connected_clients.count(),
2302 counters: Some(counters),
2303 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2304 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2305 capability_requirements: self.capability_requirement_statuses(),
2306 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2307 };
2308 Ok(vec![control_response_body_frame(
2309 &frame,
2310 &response,
2311 "ClientControlResponse::ServerDescribe",
2312 )?])
2313 }
2314
2315 fn handle_catalog_list(
2316 &self,
2317 frame: Frame,
2318 module_id: Option<String>,
2319 ) -> Result<Vec<Frame>, RouterError> {
2320 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2321 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2322 })?;
2323 let entries = modules
2324 .into_iter()
2325 .filter(|registration| {
2326 module_id
2327 .as_deref()
2328 .map(|wanted| registration.manifest.module_id == wanted)
2329 .unwrap_or(true)
2330 })
2331 .map(|registration| {
2332 let not_ready = self.not_ready_reason(®istration);
2333 let roles = registration.manifest.provides;
2334 CatalogEntry {
2335 module_id: registration.manifest.module_id,
2336 ready: not_ready.is_none(),
2337 not_ready,
2338 module_version: Some(registration.manifest.module_version),
2339 roles,
2340 control_ops: registration.control_ops,
2341 capabilities: registration.manifest.capabilities,
2342 self_signals: registration.manifest.self_signals,
2343 }
2344 })
2345 .collect();
2346 let response = ClientControlResponse::CatalogList {
2347 generation,
2348 modules: entries,
2349 subc_ops: subc_ops(),
2350 };
2351 Ok(vec![control_response_body_frame(
2352 &frame,
2353 &response,
2354 "ClientControlResponse::CatalogList",
2355 )?])
2356 }
2357
2358 fn route_open_principal(
2359 &self,
2360 frame: &Frame,
2361 consumer_identity: Option<ConsumerIdentity>,
2362 ) -> Result<Result<Principal, Frame>, RouterError> {
2363 let Some(consumer_identity) = consumer_identity else {
2364 return Ok(Ok(Principal::Direct));
2365 };
2366
2367 if self.supervisor.spawned_consumer_authorized(
2368 &consumer_identity.module_id,
2369 &consumer_identity.launch_nonce,
2370 ) {
2371 return Ok(Ok(Principal::Reserved {
2372 module_id: consumer_identity.module_id,
2373 }));
2374 }
2375
2376 Ok(Err(control_error_frame(
2377 frame,
2378 "bad_consumer_identity",
2379 format!(
2380 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2381 consumer_identity.module_id
2382 ),
2383 )?))
2384 }
2385
2386 fn route_open_refusal_frame(
2393 &self,
2394 ctx: &RouteCtx,
2395 frame: &Frame,
2396 module_id: &str,
2397 reason: &'static str,
2398 code: &'static str,
2399 message: impl Into<String>,
2400 ) -> Result<Frame, RouterError> {
2401 self.observe_route_open_refusal(ctx, module_id, reason, code);
2402 control_error_frame(frame, code, message.into())
2403 }
2404
2405 fn route_open_breaker_refusal_frame(
2423 &self,
2424 ctx: &RouteCtx,
2425 frame: &Frame,
2426 module_id: &str,
2427 consecutive_timeouts: u32,
2428 retry_in: Duration,
2429 probe_in_flight: bool,
2430 ) -> Result<Frame, RouterError> {
2431 self.counters
2432 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2433 debug!(
2434 target: "control",
2435 code = "module_timeout",
2436 module_id = ?module_id,
2437 connection_id = ctx.connection_id.get(),
2438 consecutive_timeouts,
2439 retry_in_ms = retry_in.as_millis() as u64,
2440 probe_in_flight,
2441 "route.open refused by open bind-relay breaker"
2442 );
2443 let detail = if probe_in_flight {
2444 "one probe bind is already in flight; retry once it settles".to_string()
2445 } else {
2446 format!("not relaying for another {retry_in:?}")
2447 };
2448 control_error_frame(
2449 frame,
2450 "module_timeout",
2451 format!(
2452 "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2453 relays; {detail}"
2454 ),
2455 )
2456 }
2457
2458 fn observe_route_open_refusal(
2471 &self,
2472 ctx: &RouteCtx,
2473 module_id: &str,
2474 reason: &'static str,
2475 code: &'static str,
2476 ) {
2477 self.counters.increment_route_open_refused(code);
2478 info!(
2479 target: "control",
2480 code,
2481 reason,
2482 module_id = ?module_id,
2483 connection_id = ctx.connection_id.get(),
2484 "route.open refused"
2485 );
2486 }
2487
2488 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2533 self.counters.increment_route_open_accepted(principal);
2534 info!(
2535 target: "control",
2536 principal,
2537 module_id,
2538 connection_id = ctx.connection_id.get(),
2539 "route.open accepted"
2540 );
2541 }
2542
2543 fn supervised_absent_route_open_refusal_frame(
2544 &self,
2545 ctx: &RouteCtx,
2546 frame: &Frame,
2547 module_id: &str,
2548 code: &'static str,
2549 status: &crate::supervise::ModuleStatus,
2550 ) -> Result<Frame, RouterError> {
2551 self.counters.increment_route_open_refused(code);
2552 info!(
2553 target: "control",
2554 code,
2555 reason = "supervised_not_registered",
2556 module_id = ?module_id,
2557 connection_id = ctx.connection_id.get(),
2558 state = %status.state,
2559 enabled = status.enabled,
2560 live = status.live,
2561 "route.open refused"
2562 );
2563 control_error_frame(
2564 frame,
2565 code,
2566 format!(
2567 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2568 status.state, status.enabled, status.live
2569 ),
2570 )
2571 }
2572
2573 async fn handle_route_open(
2574 &self,
2575 ctx: &RouteCtx,
2576 frame: Frame,
2577 request: RouteOpenRequest,
2578 ) -> Result<Vec<Frame>, RouterError> {
2579 let RouteOpenRequest {
2580 target,
2581 mut identity,
2582 consumer_identity,
2583 consumer_capabilities,
2584 admission_facts,
2585 } = request;
2586 let target_module_id = target_module_id(&target).to_string();
2587 debug!(
2588 connection_id = ctx.connection_id.get(),
2589 corr = frame.header.corr,
2590 module_id = %target_module_id,
2591 "handling route.open"
2592 );
2593
2594 let Some(registration) = self
2613 .registry
2614 .get_module(&target_module_id)
2615 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2616 else {
2617 if let Some((status, warming)) =
2618 self.supervisor_status(&target_module_id, frame.header.corr)?
2619 {
2620 if status.protocol == ModuleProtocol::None {
2627 return Ok(vec![self.route_open_refusal_frame(
2628 ctx,
2629 &frame,
2630 &target_module_id,
2631 "protocol_none",
2632 error_codes::MODULE_NO_PROTOCOL,
2633 format!(
2634 "module_id '{target_module_id}' is declared protocol: none; \
2635 it speaks no subc wire and serves no routes"
2636 ),
2637 )?]);
2638 }
2639 let code = if warming {
2640 "module_warming"
2641 } else {
2642 "target_unavailable"
2643 };
2644 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2645 ctx,
2646 &frame,
2647 &target_module_id,
2648 code,
2649 &status,
2650 )?]);
2651 }
2652 if let Some(removed_ago_ms) =
2653 self.supervisor.removal_tombstone_age_ms(&target_module_id)
2654 {
2655 return Ok(vec![self.route_open_refusal_frame(
2656 ctx,
2657 &frame,
2658 &target_module_id,
2659 "removed",
2660 error_codes::MODULE_REMOVED,
2661 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
2662 )?]);
2663 }
2664 return Ok(vec![self.route_open_refusal_frame(
2665 ctx,
2666 &frame,
2667 &target_module_id,
2668 "not_registered",
2669 error_codes::UNKNOWN_MODULE,
2670 format!("module_id '{target_module_id}' is not registered"),
2671 )?]);
2672 };
2673
2674 if !registration.ready {
2678 self.counters
2679 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
2680 info!(
2681 target: "control",
2682 code = error_codes::MODULE_WARMING,
2683 module_id = ?target_module_id,
2684 connection_id = ctx.connection_id.get(),
2685 reason = "declared_not_ready",
2686 "route.open refused"
2687 );
2688 return Ok(vec![control_error_body_frame(
2689 &frame,
2690 ErrorBody {
2691 code: error_codes::MODULE_WARMING.to_string(),
2692 message: format!(
2693 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
2694 ),
2695 detail: Some(serde_json::json!({
2696 "reason": "declared_not_ready"
2697 })),
2698 },
2699 )?]);
2700 }
2701
2702 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
2721 self.counters
2722 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
2723 info!(
2724 target: "control",
2725 code = error_codes::MODULE_WARMING,
2726 module_id = ?target_module_id,
2727 connection_id = ctx.connection_id.get(),
2728 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2729 capability = %capability,
2730 "route.open refused"
2731 );
2732 return Ok(vec![control_error_body_frame(
2733 &frame,
2734 ErrorBody {
2735 code: error_codes::MODULE_WARMING.to_string(),
2736 message: format!(
2737 "module_id '{target_module_id}' requires capability '{capability}', \
2738 which no registered module provides; retry"
2739 ),
2740 detail: Some(serde_json::json!({
2741 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
2742 "capability": capability,
2743 })),
2744 },
2745 )?]);
2746 }
2747
2748 if !target_has_required_role(&target, ®istration.manifest.provides) {
2749 return Ok(vec![self.route_open_refusal_frame(
2750 ctx,
2751 &frame,
2752 &target_module_id,
2753 "role_not_provided",
2754 "target_unavailable",
2755 format!("module_id '{target_module_id}' does not provide the requested target"),
2756 )?]);
2757 }
2758
2759 if registration.state != ChannelState::Active {
2760 return Ok(vec![self.route_open_refusal_frame(
2761 ctx,
2762 &frame,
2763 &target_module_id,
2764 "registration_not_active",
2765 "target_unavailable",
2766 format!("module_id '{target_module_id}' is not active"),
2767 )?]);
2768 }
2769
2770 if self
2771 .forwarding
2772 .module_is_draining(&target_module_id)
2773 .map_err(RouterError::Forwarding)?
2774 {
2775 return Ok(vec![self.route_open_refusal_frame(
2776 ctx,
2777 &frame,
2778 &target_module_id,
2779 "reloading",
2780 "module_reloading",
2781 format!("module_id '{target_module_id}' is reloading"),
2782 )?]);
2783 }
2784
2785 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
2786 process_liveness.process_live(&target_module_id) == Some(false)
2787 }) {
2788 if process_liveness.process_replacing(&target_module_id) {
2795 return Ok(vec![self.route_open_refusal_frame(
2796 ctx,
2797 &frame,
2798 &target_module_id,
2799 "reloading",
2800 "module_reloading",
2801 format!("module_id '{target_module_id}' is reloading"),
2802 )?]);
2803 }
2804 return Ok(vec![self.route_open_refusal_frame(
2805 ctx,
2806 &frame,
2807 &target_module_id,
2808 "supervisor_not_live",
2809 "target_unavailable",
2810 format!("module_id '{target_module_id}' is not live"),
2811 )?]);
2812 }
2813
2814 if !self
2815 .forwarding
2816 .has_live_module_connection(&target_module_id)
2817 .map_err(RouterError::Forwarding)?
2818 {
2819 return Ok(vec![self.route_open_refusal_frame(
2820 ctx,
2821 &frame,
2822 &target_module_id,
2823 "no_forwarding_connection",
2824 "target_unavailable",
2825 format!("module_id '{target_module_id}' has no live forwarding connection"),
2826 )?]);
2827 }
2828
2829 if let Some(error) =
2830 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
2831 {
2832 self.observe_route_open_refusal(
2833 ctx,
2834 &target_module_id,
2835 "op_not_allowed",
2836 "op_not_allowed",
2837 );
2838 return Ok(vec![error]);
2839 }
2840
2841 let principal = match self.route_open_principal(&frame, consumer_identity)? {
2842 Ok(principal) => principal,
2843 Err(error) => {
2844 self.observe_route_open_refusal(
2845 ctx,
2846 &target_module_id,
2847 "bad_consumer_identity",
2848 "bad_consumer_identity",
2849 );
2850 return Ok(vec![error]);
2851 }
2852 };
2853
2854 if let Principal::Reserved {
2858 module_id: opening_module_id,
2859 } = &principal
2860 {
2861 if let Some(opening_registration) = self
2862 .registry
2863 .get_module(opening_module_id)
2864 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2865 {
2866 if let Some(capability) =
2867 denied_capability(&opening_registration.manifest, ®istration.manifest)
2868 {
2869 warn!(
2870 opening_module_id,
2871 target_module_id,
2872 capability,
2873 "refusing route.open because an attested capability deny edge matches"
2874 );
2875 return Ok(vec![self.route_open_refusal_frame(
2876 ctx,
2877 &frame,
2878 &target_module_id,
2879 "capability_deny_edge",
2880 "capability_forbidden",
2881 format!(
2882 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
2883 ),
2884 )?]);
2885 }
2886 }
2887 }
2888
2889 if admission_facts.is_some() {
2890 let carrier_matches = matches!(
2891 &principal,
2892 Principal::Reserved { module_id }
2893 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
2894 );
2895 if !carrier_matches {
2896 return Ok(vec![self.route_open_refusal_frame(
2897 ctx,
2898 &frame,
2899 &target_module_id,
2900 "admission_facts_carrier_not_permitted",
2901 "admission_facts_not_permitted",
2902 "admission facts may only be carried by the configured reserved module",
2903 )?]);
2904 }
2905
2906 let target_allowed = self
2907 .admission_facts_targets
2908 .as_ref()
2909 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
2910 if !target_allowed {
2911 return Ok(vec![self.route_open_refusal_frame(
2912 ctx,
2913 &frame,
2914 &target_module_id,
2915 "admission_facts_target_not_listed",
2916 "admission_facts_target_not_allowed",
2917 format!(
2918 "admission facts are not permitted for target module_id '{target_module_id}'"
2919 ),
2920 )?]);
2921 }
2922
2923 }
2927
2928 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
2950 Ok(project_root) => project_root,
2951 Err(err) => {
2952 return Ok(vec![control_error_frame(
2953 &frame,
2954 "invalid_project_root",
2955 err.to_string(),
2956 )?])
2957 }
2958 };
2959 identity.project_root = project_root.as_path().to_path_buf();
2960
2961 let _concurrency_guard = match self
2972 .route_bind_concurrency
2973 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
2974 {
2975 Ok(guard) => guard,
2976 Err(in_flight) => {
2977 return Ok(vec![self.route_open_target_capacity_refusal(
2978 ctx,
2979 &frame,
2980 &target_module_id,
2981 in_flight,
2982 )?]);
2983 }
2984 };
2985
2986 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
2989 RouteBindAdmission::Admitted { guard, probe } => {
2990 if probe {
2991 info!(
2992 module_id = %target_module_id,
2993 connection_id = ctx.connection_id.get(),
2994 "route.bind breaker half-open: admitting one probe"
2995 );
2996 }
2997 guard
2998 }
2999 RouteBindAdmission::Refused {
3000 consecutive_timeouts,
3001 retry_in,
3002 probe_in_flight,
3003 } => {
3004 return Ok(vec![self.route_open_breaker_refusal_frame(
3005 ctx,
3006 &frame,
3007 &target_module_id,
3008 consecutive_timeouts,
3009 retry_in,
3010 probe_in_flight,
3011 )?]);
3012 }
3013 };
3014
3015 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3020 let relay_deadline = Instant::now() + route_bind_relay_timeout;
3021 let pending = match self
3022 .forwarding
3023 .begin_route_bind_relay_for(
3024 ctx.connection_id,
3025 ctx.egress.clone(),
3026 response_version(&frame),
3027 frame.header.corr,
3028 &target_module_id,
3029 principal.clone(),
3030 Some(project_root),
3031 relay_deadline,
3032 )
3033 .await
3034 {
3035 Ok(pending) => pending,
3036 Err(err) => {
3037 return Ok(vec![self.route_open_refusal_frame(
3038 ctx,
3039 &frame,
3040 &target_module_id,
3041 "relay_reservation_failed",
3042 forwarding_error_code(&err),
3043 err.to_string(),
3044 )?])
3045 }
3046 };
3047 let crate::forwarding::PendingRouteBindRelay {
3048 endpoint,
3049 module_sink,
3050 negotiated_ver,
3051 client_channel,
3052 client_epoch,
3053 module_channel,
3054 module_epoch,
3055 corr: relay_corr,
3056 receiver,
3057 } = pending;
3058 let mut reservation =
3059 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3060
3061 debug!(
3062 connection_id = ctx.connection_id.get(),
3063 client_channel,
3064 client_epoch,
3065 module_channel,
3066 module_epoch,
3067 "reserved route handle pair"
3068 );
3069 let principal_label = match &principal {
3072 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3073 Principal::Direct => "direct".to_string(),
3074 other => format!("{other:?}"),
3075 };
3076 let relay = ModuleControlRequest::RouteBind {
3077 route_channel: module_channel,
3078 epoch: module_epoch,
3079 target,
3080 identity,
3081 principal: Some(principal),
3082 consumer_capabilities,
3083 admission_facts,
3084 };
3085 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3086 RouterError::backend(
3087 0,
3088 frame.header.corr,
3089 format!("failed to encode route.bind request: {err}"),
3090 )
3091 })?;
3092 let relay_frame = Frame::build_with_version(
3093 negotiated_ver,
3094 FrameType::Request,
3095 control_flags(),
3096 0,
3097 0,
3098 relay_corr,
3099 relay_body,
3100 )
3101 .map_err(RouterError::FrameBuild)?;
3102
3103 if let Err(err) = module_sink.send(relay_frame).await {
3104 reservation.release_and_disarm();
3105 return Ok(vec![self.route_open_refusal_frame(
3106 ctx,
3107 &frame,
3108 &target_module_id,
3109 "relay_send_failed",
3110 "target_unavailable",
3111 err.to_string(),
3112 )?]);
3113 }
3114
3115 if !self
3116 .forwarding
3117 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3118 .map_err(RouterError::Forwarding)?
3119 {
3120 self.send_abandoned_route_bind_goodbye(
3121 &module_sink,
3122 negotiated_ver,
3123 module_channel,
3124 module_epoch,
3125 );
3126 }
3127
3128 match timeout_at(relay_deadline, receiver).await {
3129 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3130 reservation.disarm();
3131 if breaker.record_accepted() {
3132 info!(
3133 module_id = %target_module_id,
3134 "route.bind breaker closed: the probe was accepted"
3135 );
3136 }
3137 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3138 Ok(Vec::new())
3139 }
3140 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3141 reservation.release_and_disarm();
3142 breaker.record_inconclusive();
3146 self.counters
3147 .increment_route_open_refused("module_rejected");
3148 info!(
3149 target: "control",
3150 code = "module_rejected",
3151 module_code = ?body.code,
3152 module_id = ?target_module_id,
3153 connection_id = ctx.connection_id.get(),
3154 "route.open refused"
3155 );
3156 Ok(vec![control_error_body_frame(&frame, body)?])
3157 }
3158 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3159 reservation.release_and_disarm();
3160 breaker.record_inconclusive();
3161 tracing::warn!(
3168 module_id = %target_module_id,
3169 "route.bind relay abandoned: {message}"
3170 );
3171 Ok(vec![self.route_open_refusal_frame(
3172 ctx,
3173 &frame,
3174 &target_module_id,
3175 "relay_abandoned",
3176 "target_unavailable",
3177 message,
3178 )?])
3179 }
3180 Ok(Err(_)) => {
3181 reservation.release_and_disarm();
3182 breaker.record_inconclusive();
3183 Ok(vec![self.route_open_refusal_frame(
3184 ctx,
3185 &frame,
3186 &target_module_id,
3187 "relay_waiter_canceled",
3188 "target_unavailable",
3189 "route.bind relay waiter was canceled before the module responded",
3190 )?])
3191 }
3192 Err(_) => {
3193 reservation.release_and_disarm();
3194 if let Some(opened) = breaker.record_timeout(
3198 self.route_bind_breaker_threshold,
3199 self.route_bind_breaker_cooldown,
3200 ) {
3201 warn!(
3202 module_id = %target_module_id,
3203 consecutive_timeouts = opened.consecutive_timeouts,
3204 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3205 reopened_after_probe = opened.reopened_after_probe,
3206 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3207 );
3208 }
3209 tracing::warn!(
3217 module_id = %target_module_id,
3218 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3219 "route.bind relay timed out: module did not ack within budget"
3220 );
3221 Ok(vec![self.route_open_refusal_frame(
3222 ctx,
3223 &frame,
3224 &target_module_id,
3225 "relay_timed_out",
3226 "module_timeout",
3227 format!(
3228 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3229 route_bind_relay_timeout
3230 ),
3231 )?])
3232 }
3233 }
3234 }
3235
3236 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3237 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3238 snapshot: self.supervisor.spawn_snapshot(),
3239 };
3240 Ok(vec![control_response_body_frame(
3241 &frame,
3242 &response,
3243 "ClientControlResponse::SupervisorSpawnSnapshot",
3244 )?])
3245 }
3246
3247 fn handle_supervisor_spawn_subscribe(
3248 &self,
3249 ctx: &RouteCtx,
3250 frame: Frame,
3251 since: Option<SpawnCursor>,
3252 ) -> Result<Vec<Frame>, RouterError> {
3253 match self.supervisor.subscribe_spawns(
3254 ctx.connection_id,
3255 frame.header.corr,
3256 response_version(&frame),
3257 since,
3258 ctx.egress.clone(),
3259 ) {
3260 Ok(()) => Ok(Vec::new()),
3261 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3262 Ok(vec![control_error_body_frame(
3263 &frame,
3264 ErrorBody {
3265 code: "spawn_cursor_incarnation_mismatch".to_string(),
3266 message: "spawn cursor belongs to a different daemon incarnation"
3267 .to_string(),
3268 detail: Some(serde_json::json!({
3269 "current_daemon_incarnation": current
3270 })),
3271 },
3272 )?])
3273 }
3274 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3275 &frame,
3276 ErrorBody {
3277 code: "spawn_cursor_too_old".to_string(),
3278 message: "spawn cursor predates the retained event ring".to_string(),
3279 detail: Some(serde_json::json!({
3280 "oldest_retained_cursor": oldest
3281 })),
3282 },
3283 )?]),
3284 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3285 0,
3286 frame.header.corr,
3287 format!("failed to open supervisor spawn subscription: {error}"),
3288 )),
3289 }
3290 }
3291
3292 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3293 let generation = self
3294 .registry
3295 .generation()
3296 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3297 let mut modules = Vec::new();
3298 for module in self.supervisor.list() {
3299 let status = module.status_for_control("list").map_err(|err| {
3300 RouterError::backend(
3301 0,
3302 frame.header.corr,
3303 format!("failed to read supervisor status: {err}"),
3304 )
3305 })?;
3306 let (configured, _) = module.configuration().map_err(|err| {
3307 RouterError::backend(
3308 0,
3309 frame.header.corr,
3310 format!("failed to read module configuration: {err}"),
3311 )
3312 })?;
3313 let image = module.running_image_agreement().await;
3315 let resources = Some(module.child_resource_usage());
3318 let pending_reload = Some(reload_verdict(
3319 &configured.program,
3320 status.spawned_from.as_deref(),
3321 image,
3322 ));
3323 modules.push(SupervisorEntry {
3324 module_id: status.module_id,
3325 state: status.state.to_string(),
3326 enabled: status.enabled,
3327 live: status.live,
3328 protocol: status.protocol,
3329 health: status.health.status,
3330 pending_reload,
3331 last_probe_ms: status.health.last_probe_ms,
3332 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3333 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3334 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3335 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3336 restart_count: Some(status.restart_count),
3337 max_restarts: Some(status.max_restarts),
3338 lifetime_restarts: Some(status.lifetime_restarts),
3339 spawn_generation: Some(status.spawn_generation),
3340 restart_window_secs: Some(status.restart_window.as_secs()),
3341 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3342 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3343 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3344 resources,
3345 });
3346 }
3347 let response = ClientControlResponse::SupervisorList {
3348 generation,
3349 modules,
3350 };
3351 Ok(vec![control_response_body_frame(
3352 &frame,
3353 &response,
3354 "ClientControlResponse::SupervisorList",
3355 )?])
3356 }
3357
3358 fn handle_supervisor_stderr_tail(
3359 &self,
3360 frame: Frame,
3361 module_id: String,
3362 max_lines: Option<u32>,
3363 max_bytes: Option<u32>,
3364 ) -> Result<Vec<Frame>, RouterError> {
3365 let Some(module) = self.supervisor.get(&module_id) else {
3366 return Ok(vec![control_error_frame(
3367 &frame,
3368 "unknown_module",
3369 format!("module_id '{module_id}' is not supervised"),
3370 )?]);
3371 };
3372
3373 let snapshot = module.stderr_tail(
3374 max_lines.map(|value| value as usize),
3375 max_bytes.map(|value| value as usize),
3376 );
3377
3378 let response = ClientControlResponse::SupervisorStderrTail {
3379 module_id,
3380 tail: StderrTail {
3381 capture: match snapshot.capture {
3382 CaptureState::Captured => StderrCaptureState::Captured,
3383 CaptureState::Incomplete { reason } => {
3384 StderrCaptureState::Incomplete { reason }
3385 }
3386 CaptureState::NotCaptured { reason } => {
3387 StderrCaptureState::NotCaptured { reason }
3388 }
3389 },
3390 entries: snapshot
3391 .entries
3392 .into_iter()
3393 .map(|entry| match entry {
3394 TailEntry::Line { text, truncated } => {
3395 StderrTailEntry::Line { text, truncated }
3396 }
3397 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3398 })
3399 .collect(),
3400 dropped_lines: snapshot.dropped_lines,
3401 },
3402 };
3403 Ok(vec![control_response_body_frame(
3404 &frame,
3405 &response,
3406 "ClientControlResponse::SupervisorStderrTail",
3407 )?])
3408 }
3409
3410 async fn handle_supervisor_terminals(
3411 &self,
3412 frame: Frame,
3413 module_id: String,
3414 ) -> Result<Vec<Frame>, RouterError> {
3415 let Some(module) = self.supervisor.get(&module_id) else {
3416 return Ok(vec![control_error_frame(
3417 &frame,
3418 "unknown_module",
3419 format!("module_id '{module_id}' is not supervised"),
3420 )?]);
3421 };
3422
3423 let terminals = module
3426 .read_durable_terminal_history()
3427 .await
3428 .map_err(|error| {
3429 RouterError::backend(
3430 0,
3431 frame.header.corr,
3432 format!("failed to read terminal history: {error}"),
3433 )
3434 })?;
3435 let response = ClientControlResponse::SupervisorTerminals {
3436 module_id,
3437 terminals,
3438 };
3439 Ok(vec![control_response_body_frame(
3440 &frame,
3441 &response,
3442 "ClientControlResponse::SupervisorTerminals",
3443 )?])
3444 }
3445
3446 fn handle_supervisor_routes(
3447 &self,
3448 frame: Frame,
3449 module_id: Option<String>,
3450 ) -> Result<Vec<Frame>, RouterError> {
3451 let modules = self
3452 .forwarding
3453 .route_census(module_id.as_deref())
3454 .map_err(RouterError::Forwarding)?
3455 .into_iter()
3456 .map(|(module_id, routes)| SupervisorRouteModule {
3457 module_id,
3458 routes: routes
3459 .into_iter()
3460 .map(|route| SupervisorRoute {
3461 consumer: match route.principal {
3462 Principal::Reserved { module_id } => {
3463 SupervisorRouteConsumer::Reserved { module_id }
3464 }
3465 Principal::Direct | Principal::Unverified => {
3466 SupervisorRouteConsumer::Direct {
3467 connection_id: route.goodbye_target.connection_id.get(),
3468 }
3469 }
3470 },
3471 age_ms: Instant::now()
3472 .saturating_duration_since(route.bound_at)
3473 .as_millis()
3474 .try_into()
3475 .unwrap_or(u64::MAX),
3476 draining: route.draining,
3477 drain_reason: route.drain_reason,
3478 })
3479 .collect(),
3480 })
3481 .collect();
3482 let response = ClientControlResponse::SupervisorRoutes { modules };
3483 Ok(vec![control_response_body_frame(
3484 &frame,
3485 &response,
3486 "ClientControlResponse::SupervisorRoutes",
3487 )?])
3488 }
3489
3490 async fn handle_supervisor_provenance(
3491 &self,
3492 frame: Frame,
3493 module_id: Option<String>,
3494 ) -> Result<Vec<Frame>, RouterError> {
3495 let mut selected = if let Some(module_id) = module_id {
3496 let Some(module) = self.supervisor.get(&module_id) else {
3497 return Ok(vec![control_error_frame(
3498 &frame,
3499 "unknown_module",
3500 format!("module_id '{module_id}' is not supervised"),
3501 )?]);
3502 };
3503 vec![module]
3504 } else {
3505 self.supervisor.list()
3506 };
3507
3508 let mut modules = Vec::with_capacity(selected.len());
3509 for module in selected.drain(..) {
3510 let status = module.status().map_err(|err| {
3511 RouterError::backend(
3512 0,
3513 frame.header.corr,
3514 format!("failed to read supervisor status: {err}"),
3515 )
3516 })?;
3517 let module_declared = self
3518 .registry
3519 .get_module(&status.module_id)
3520 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3521 .and_then(|registration| registration.manifest.provenance)
3522 .map(|build| ModuleDeclaredProvenance::Reported { build })
3523 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3524 #[cfg(test)]
3525 let running_image = match &self.provenance_probe_override {
3526 Some(result) => result.clone(),
3527 None => module.running_image_agreement().await,
3528 };
3529 #[cfg(not(test))]
3530 let running_image = module.running_image_agreement().await;
3531 modules.push(SupervisorModuleProvenance {
3532 module_id: status.module_id,
3533 module_declared,
3534 daemon_observed: SupervisorObservedProcess {
3535 pid: status.pid,
3536 spawned_at_ms: status.spawned_at_ms,
3537 spawned_from: status.spawned_from,
3538 running_image,
3539 },
3540 });
3541 }
3542 let daemon = SupervisorDaemonProvenance {
3543 daemon_build: self.daemon_provenance.build.clone(),
3544 daemon_observed: DaemonObservedProcess {
3545 pid: self.daemon_provenance.pid,
3546 started_at_ms: self
3547 .daemon_provenance
3548 .start_clock
3549 .map(|clock| clock.started_at_ms())
3550 .or(self.daemon_provenance.started_at_ms),
3551 running_image: self
3552 .daemon_provenance
3553 .probe
3554 .observe(
3555 self.daemon_provenance.pid,
3556 self.daemon_provenance.executable_path.as_deref(),
3557 self.daemon_provenance.executable_identity,
3558 self.daemon_provenance.process_start_time,
3559 )
3560 .await,
3561 },
3562 };
3563 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3564 Ok(vec![control_response_body_frame(
3565 &frame,
3566 &response,
3567 "ClientControlResponse::SupervisorProvenance",
3568 )?])
3569 }
3570
3571 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3572 self.refresh_capability_requirements();
3573 let generation = self
3574 .registry
3575 .generation()
3576 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3577 let modules = self
3578 .supervisor
3579 .list()
3580 .into_iter()
3581 .map(|module| {
3582 let status = module.status_for_control("health").map_err(|err| {
3583 RouterError::backend(
3584 0,
3585 frame.header.corr,
3586 format!("failed to read supervisor health: {err}"),
3587 )
3588 })?;
3589 let module_id = status.module_id;
3590 let capability_detail = self
3591 .capability_evaluator
3592 .required_problem_detail(&module_id);
3593 Ok(SupervisorHealthEntry {
3594 module_id,
3595 status: status.health.status,
3596 detail: append_capability_problem_detail(
3597 status.health.detail,
3598 capability_detail,
3599 ),
3600 metrics: status.health.metrics,
3601 consecutive_failures: status.health.consecutive_failures,
3602 late_answer_count: status.health.late_answer_count,
3603 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
3604 last_action: status.health.last_action,
3605 last_action_ms: status.health.last_action_ms,
3606 last_probe_ms: status.health.last_probe_ms,
3607 })
3608 })
3609 .collect::<Result<Vec<_>, RouterError>>()?;
3610 let response = ClientControlResponse::SupervisorHealth {
3611 generation,
3612 modules,
3613 };
3614 Ok(vec![control_response_body_frame(
3615 &frame,
3616 &response,
3617 "ClientControlResponse::SupervisorHealth",
3618 )?])
3619 }
3620
3621 async fn handle_supervisor_restart(
3622 &self,
3623 frame: Frame,
3624 module_id: String,
3625 drain_timeout_ms: Option<u64>,
3626 ) -> Result<Vec<Frame>, RouterError> {
3627 let operation_lock = self.supervisor.operation_lock();
3628 let _operation_guard = operation_lock.lock().await;
3629 let Some(module) = self.supervisor.get(&module_id) else {
3630 return Ok(vec![control_error_frame(
3631 &frame,
3632 "unknown_module",
3633 format!("module_id '{module_id}' is not supervised"),
3634 )?]);
3635 };
3636
3637 if let Err(err) = module.restart(drain_timeout_ms).await {
3638 let (code, message) = match err {
3639 crate::supervise::SuperviseError::Disabled { .. } => {
3640 ("module_disabled", err.to_string())
3641 }
3642 crate::supervise::SuperviseError::SwapInProgress { .. } => {
3643 ("swap_in_progress", err.to_string())
3644 }
3645 _ => (
3646 "target_unavailable",
3647 format!("failed to restart module_id '{module_id}': {err}"),
3648 ),
3649 };
3650 return Ok(vec![control_error_frame(&frame, code, message)?]);
3651 }
3652
3653 let response = ClientControlResponse::SupervisorAck {
3654 module_id,
3655 applied: true,
3656 };
3657 Ok(vec![control_response_body_frame(
3658 &frame,
3659 &response,
3660 "ClientControlResponse::SupervisorAck",
3661 )?])
3662 }
3663
3664 async fn handle_supervisor_swap(
3668 &self,
3669 frame: Frame,
3670 module_id: String,
3671 ready_timeout_ms: Option<u64>,
3672 ) -> Result<Vec<Frame>, RouterError> {
3673 let module = {
3680 let operation_lock = self.supervisor.operation_lock();
3681 let _operation_guard = operation_lock.lock().await;
3682 self.supervisor.get(&module_id)
3683 };
3684 let Some(module) = module else {
3685 return Ok(vec![control_error_frame(
3686 &frame,
3687 "unknown_module",
3688 format!("module_id '{module_id}' is not supervised"),
3689 )?]);
3690 };
3691
3692 if let Err(err) = module
3693 .swap(ready_timeout_ms.map(Duration::from_millis))
3694 .await
3695 {
3696 use crate::supervise::SuperviseError;
3697 let message = err.to_string();
3698 let error = match err {
3699 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
3700 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
3701 code: "swap_refused".to_string(),
3702 message,
3703 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
3704 },
3705 SuperviseError::SwapFailed {
3706 arm,
3707 candidate_exit,
3708 ..
3709 } => ErrorBody {
3710 code: "swap_failed".to_string(),
3711 message,
3712 detail: Some(serde_json::json!({
3713 "arm": arm.as_str(),
3714 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
3715 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
3716 })),
3717 },
3718 _ => ErrorBody::new(
3719 "target_unavailable",
3720 format!("failed to swap module_id '{module_id}': {message}"),
3721 ),
3722 };
3723 return Ok(vec![control_error_body_frame(&frame, error)?]);
3724 }
3725
3726 let response = ClientControlResponse::SupervisorAck {
3727 module_id,
3728 applied: true,
3729 };
3730 Ok(vec![control_response_body_frame(
3731 &frame,
3732 &response,
3733 "ClientControlResponse::SupervisorAck",
3734 )?])
3735 }
3736
3737 async fn handle_supervisor_reload(
3738 &self,
3739 frame: Frame,
3740 module_id: String,
3741 ) -> Result<Vec<Frame>, RouterError> {
3742 let operation_lock = self.supervisor.operation_lock();
3743 let _operation_guard = operation_lock.lock().await;
3744 let Some(module) = self.supervisor.get(&module_id) else {
3745 return Ok(vec![control_error_frame(
3746 &frame,
3747 "unknown_module",
3748 format!("module_id '{module_id}' is not supervised"),
3749 )?]);
3750 };
3751
3752 if let Err(err) = module.reload().await {
3753 let (code, message) = match err {
3754 crate::supervise::SuperviseError::Disabled { .. } => {
3755 ("module_disabled", err.to_string())
3756 }
3757 crate::supervise::SuperviseError::SwapInProgress { .. } => {
3758 ("swap_in_progress", err.to_string())
3759 }
3760 _ => (
3761 "reload_failed",
3762 format!("failed to reload module_id '{module_id}': {err}"),
3763 ),
3764 };
3765 return Ok(vec![control_error_frame(&frame, code, message)?]);
3766 }
3767
3768 let response = ClientControlResponse::SupervisorAck {
3769 module_id,
3770 applied: true,
3771 };
3772 Ok(vec![control_response_body_frame(
3773 &frame,
3774 &response,
3775 "ClientControlResponse::SupervisorAck",
3776 )?])
3777 }
3778
3779 async fn handle_supervisor_rescan(
3780 &self,
3781 frame: Frame,
3782 preview: bool,
3783 ) -> Result<Vec<Frame>, RouterError> {
3784 let Some(context) = self.rescan.clone() else {
3785 return Ok(vec![control_error_frame(
3786 &frame,
3787 "rescan_unavailable",
3788 "the daemon was not started with a reloadable config path".to_string(),
3789 )?]);
3790 };
3791
3792 let operation_lock = self.supervisor.operation_lock();
3793 let _operation_guard = operation_lock.lock().await;
3794 let loaded = match crate::daemon_config::load(&context.config_path) {
3795 Ok(config) => config,
3796 Err(err) => {
3797 return Ok(vec![control_error_frame(
3798 &frame,
3799 "invalid_daemon_config",
3800 format!("supervisor rescan rejected daemon config: {err}"),
3801 )?])
3802 }
3803 };
3804 let Some(config) = loaded else {
3812 return Ok(vec![control_error_frame(
3813 &frame,
3814 "invalid_daemon_config",
3815 format!(
3816 "daemon config not found at {}; refusing to rescan (an absent config would \
3817 retire every supervised module)",
3818 context.config_path.display()
3819 ),
3820 )?]);
3821 };
3822 let (
3823 configured_port,
3824 storage_config,
3825 admission_facts_carrier_module_id,
3826 admission_facts_targets,
3827 modules,
3828 reserved_capabilities,
3829 ) = (
3830 config.port,
3831 config.storage,
3832 config.admission_facts_carrier_module_id,
3833 config.admission_facts_targets,
3834 config.modules,
3835 config.reserved_capabilities,
3836 );
3837
3838 let mut restart_required = Vec::new();
3847 for section in RestartRequiredSection::ALL {
3848 let changed = match section {
3849 RestartRequiredSection::Port => configured_port != context.configured_port,
3850 RestartRequiredSection::Storage => storage_config != context.storage_config,
3851 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
3852 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
3853 }
3854 RestartRequiredSection::AdmissionFactsTargets => {
3855 admission_facts_targets != context.admission_facts_targets
3856 }
3857 };
3858 if changed {
3859 restart_required.push(section.label().to_string());
3860 }
3861 }
3862 if !restart_required.is_empty() {
3863 warn!(
3864 config_path = %context.config_path.display(),
3865 sections = %restart_required.join(", "),
3866 "daemon config changed outside the modules section; restart the daemon to apply those changes"
3867 );
3868 }
3869
3870 for configured in &modules {
3871 if let Err(err) = validate_spec(&configured.module_spec()) {
3872 return Ok(vec![control_error_frame(
3873 &frame,
3874 "invalid_daemon_config",
3875 format!("supervisor rescan rejected daemon config: {err}"),
3876 )?]);
3877 }
3878 }
3879
3880 let configured_capabilities = modules
3881 .iter()
3882 .map(|module| (module.module_id.clone(), module.enabled))
3883 .collect::<Vec<_>>();
3884 let preview_capability_warnings = if preview {
3885 let (_, registrations) = self.runtime_capability_snapshot()?;
3886 let current_modules = self
3887 .supervisor
3888 .list()
3889 .into_iter()
3890 .map(|module| module.module_id().to_string())
3891 .collect::<BTreeSet<_>>();
3892 let resulting_modules = configured_capabilities.clone();
3893 let removed = current_modules
3894 .into_iter()
3895 .filter(|module_id| {
3896 !resulting_modules
3897 .iter()
3898 .any(|(configured_id, _)| configured_id == module_id)
3899 })
3900 .collect::<Vec<_>>();
3901 self.capability_evaluator.preview_removal_warnings(
3902 resulting_modules,
3903 &removed,
3904 ®istrations,
3905 )
3906 } else {
3907 Vec::new()
3908 };
3909 let result = match self
3910 .reconcile_supervised_modules(&context.supervisor, modules, preview)
3911 .await
3912 {
3913 Ok(result) => result,
3914 Err(message) => {
3915 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
3916 }
3917 };
3918 if !preview {
3919 self.capability_evaluator
3920 .configure(configured_capabilities, reserved_capabilities);
3921 self.capability_evaluator.wake_deadline_loop();
3922 self.refresh_capability_requirements();
3923 }
3924 let mut result = result;
3925 result.restart_required = restart_required;
3926 result.capability_warnings = preview_capability_warnings;
3927 let response = ClientControlResponse::SupervisorRescan { result };
3928 Ok(vec![control_response_body_frame(
3929 &frame,
3930 &response,
3931 "ClientControlResponse::SupervisorRescan",
3932 )?])
3933 }
3934
3935 async fn handle_supervisor_release_reserved(
3936 &self,
3937 frame: Frame,
3938 module_id: String,
3939 ) -> Result<Vec<Frame>, RouterError> {
3940 let Some(context) = self.rescan.clone() else {
3941 return Ok(vec![control_error_frame(
3942 &frame,
3943 "release_unavailable",
3944 "reserved-id release requires a daemon started with a reloadable config path",
3945 )?]);
3946 };
3947 let operation_lock = self.supervisor.operation_lock();
3948 let _operation_guard = operation_lock.lock().await;
3949 let loaded = match crate::daemon_config::load(&context.config_path) {
3950 Ok(Some(config)) => config,
3951 Ok(None) => {
3952 return Ok(vec![control_error_frame(
3953 &frame,
3954 "invalid_daemon_config",
3955 format!(
3956 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
3957 context.config_path.display()
3958 ),
3959 )?])
3960 }
3961 Err(err) => {
3962 return Ok(vec![control_error_frame(
3963 &frame,
3964 "invalid_daemon_config",
3965 format!("unable to verify reserved-id release against daemon config: {err}"),
3966 )?])
3967 }
3968 };
3969 if loaded
3970 .modules
3971 .iter()
3972 .any(|configured| configured.module_id == module_id)
3973 {
3974 return Ok(vec![control_error_frame(
3975 &frame,
3976 "reserved_module_configured",
3977 format!(
3978 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
3979 ),
3980 )?]);
3981 }
3982 if !self.supervisor.release_retained_reserved_gate(&module_id) {
3983 return Ok(vec![control_error_frame(
3984 &frame,
3985 "reserved_gate_not_retained",
3986 format!(
3987 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
3988 ),
3989 )?]);
3990 }
3991
3992 let response = ClientControlResponse::SupervisorAck {
3993 module_id,
3994 applied: true,
3995 };
3996 Ok(vec![control_response_body_frame(
3997 &frame,
3998 &response,
3999 "ClientControlResponse::SupervisorAck",
4000 )?])
4001 }
4002
4003 async fn reconcile_supervised_modules(
4012 &self,
4013 supervisor: &Supervisor,
4014 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4015 preview: bool,
4016 ) -> Result<SupervisorRescanResult, String> {
4017 let mut current = BTreeMap::new();
4018 for module in self.supervisor.list() {
4019 let (spec, health) = module.configuration().map_err(|err| {
4020 format!(
4021 "failed to read configuration for module_id '{}': {err}",
4022 module.module_id()
4023 )
4024 })?;
4025 let enabled = module
4026 .status()
4027 .map_err(|err| {
4028 format!(
4029 "failed to read status for module_id '{}': {err}",
4030 module.module_id()
4031 )
4032 })?
4033 .enabled;
4034 current.insert(
4035 module.module_id().to_string(),
4036 (module, spec, health, enabled),
4037 );
4038 }
4039 let configured = configured_modules
4040 .into_iter()
4041 .map(|module| (module.module_id.clone(), module))
4042 .collect::<BTreeMap<_, _>>();
4043
4044 let added = configured
4045 .keys()
4046 .filter(|module_id| !current.contains_key(*module_id))
4047 .cloned()
4048 .collect::<Vec<_>>();
4049 let removed = current
4050 .keys()
4051 .filter(|module_id| !configured.contains_key(*module_id))
4052 .cloned()
4053 .collect::<Vec<_>>();
4054 let mut changed_pending_reload = Vec::new();
4055 let mut configuration_changes = BTreeSet::new();
4056 let mut enabled_changes = BTreeSet::new();
4057 let mut unchanged = 0_u32;
4058
4059 for (module_id, configured_module) in &configured {
4060 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4061 else {
4062 continue;
4063 };
4064 let configuration_changed = *current_spec != configured_module.module_spec()
4065 || *current_health != configured_module.health;
4066 let enabled_changed = *current_enabled != configured_module.enabled;
4067 if configuration_changed {
4068 configuration_changes.insert(module_id.clone());
4069 changed_pending_reload.push(module_id.clone());
4070 }
4071 if enabled_changed {
4072 enabled_changes.insert(module_id.clone());
4073 }
4074 if !configuration_changed && !enabled_changed {
4075 unchanged = unchanged.saturating_add(1);
4076 }
4077 }
4078
4079 if preview {
4085 return Ok(SupervisorRescanResult {
4086 added,
4087 removed,
4088 changed_pending_reload,
4089 enabled_changes: enabled_changes.iter().cloned().collect(),
4090 unchanged,
4091 preview: true,
4092 restart_required: Vec::new(),
4096 capability_warnings: Vec::new(),
4097 });
4098 }
4099
4100 for module_id in &removed {
4101 let module = ¤t
4102 .get(module_id)
4103 .expect("removed module came from current supervisor state")
4104 .0;
4105 module.retire().await.map_err(|err| {
4106 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4107 })?;
4108 self.supervisor.record_rescan_removal(module_id);
4142 self.supervisor.retire(module_id);
4143 }
4144
4145 for module_id in configured.keys() {
4146 let Some((module, _, _, _)) = current.get(module_id) else {
4147 continue;
4148 };
4149 let configured_module = configured
4150 .get(module_id)
4151 .expect("configured module id came from configured map");
4152 if configuration_changes.contains(module_id) {
4153 module
4154 .update_configuration(
4155 configured_module.module_spec(),
4156 configured_module.health,
4157 configured_module.drain_timeout_ms,
4158 )
4159 .await
4160 .map_err(|err| {
4161 format!(
4162 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4163 )
4164 })?;
4165 }
4166 if enabled_changes.contains(module_id) {
4167 module
4168 .set_enabled(configured_module.enabled)
4169 .await
4170 .map_err(|err| {
4171 format!(
4172 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4173 configured_module.enabled
4174 )
4175 })?;
4176 }
4177 }
4178
4179 for module_id in &added {
4180 let configured_module = configured
4181 .get(module_id)
4182 .expect("added module id came from configured map");
4183 supervisor
4184 .supervise_configured_with_health(
4185 configured_module.module_spec(),
4186 configured_module.enabled,
4187 configured_module.health,
4188 configured_module.drain_timeout_ms,
4189 configured_module.restart,
4190 )
4191 .map_err(|err| {
4192 format!("failed to add module_id '{module_id}' during rescan: {err}")
4193 })?;
4194 }
4195
4196 Ok(SupervisorRescanResult {
4197 added,
4198 removed,
4199 changed_pending_reload,
4200 enabled_changes: enabled_changes.iter().cloned().collect(),
4201 unchanged,
4202 preview: false,
4203 restart_required: Vec::new(),
4206 capability_warnings: Vec::new(),
4207 })
4208 }
4209
4210 async fn handle_supervisor_set_enabled(
4211 &self,
4212 frame: Frame,
4213 module_id: String,
4214 enabled: bool,
4215 ) -> Result<Vec<Frame>, RouterError> {
4216 let operation_lock = self.supervisor.operation_lock();
4217 let _operation_guard = operation_lock.lock().await;
4218 let Some(module) = self.supervisor.get(&module_id) else {
4219 return Ok(vec![control_error_frame(
4220 &frame,
4221 "unknown_module",
4222 format!("module_id '{module_id}' is not supervised"),
4223 )?]);
4224 };
4225
4226 let applied = match module.set_enabled(enabled).await {
4227 Ok(applied) => applied,
4228 Err(err) => {
4229 return Ok(vec![control_error_frame(
4230 &frame,
4231 "target_unavailable",
4232 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4233 )?])
4234 }
4235 };
4236
4237 self.capability_evaluator.wake_deadline_loop();
4238 self.refresh_capability_requirements();
4239 let response = ClientControlResponse::SupervisorAck { module_id, applied };
4240 Ok(vec![control_response_body_frame(
4241 &frame,
4242 &response,
4243 "ClientControlResponse::SupervisorAck",
4244 )?])
4245 }
4246
4247 async fn handle_supervisor_health_probe(
4248 &self,
4249 frame: Frame,
4250 module_id: String,
4251 ) -> Result<Vec<Frame>, RouterError> {
4252 self.refresh_capability_requirements();
4253 let Some(registration) = self
4254 .registry
4255 .get_module(&module_id)
4256 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4257 else {
4258 return Ok(vec![control_error_frame(
4259 &frame,
4260 "unknown_module",
4261 format!("module_id '{module_id}' is not registered"),
4262 )?]);
4263 };
4264
4265 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4278 {
4279 return Ok(vec![control_error_frame(
4280 &frame,
4281 "health_not_advertised",
4282 format!("module_id '{module_id}' did not advertise health.check"),
4283 )?]);
4284 }
4285
4286 let deadline = Instant::now() + self.health_probe_timeout;
4287 let pending = match self.forwarding.begin_module_control_rpc_for(
4288 &module_id,
4289 MODULE_CONTROL_OP_HEALTH_CHECK,
4290 deadline,
4291 ) {
4292 Ok(pending) => pending,
4293 Err(err) => {
4294 return Ok(vec![control_error_frame(
4295 &frame,
4296 forwarding_error_code(&err),
4297 err.to_string(),
4298 )?])
4299 }
4300 };
4301
4302 let PendingModuleControlRpc {
4303 endpoint,
4304 module_sink,
4305 negotiated_ver,
4306 corr: probe_corr,
4307 receiver,
4308 } = pending;
4309 let mut guard =
4310 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4311 let probe_body =
4312 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4313 RouterError::backend(
4314 0,
4315 frame.header.corr,
4316 format!("failed to encode health.check request: {err}"),
4317 )
4318 })?;
4319 let probe_frame = Frame::build_with_version(
4320 negotiated_ver,
4321 FrameType::Request,
4322 control_flags(),
4323 0,
4324 0,
4325 probe_corr,
4326 probe_body,
4327 )
4328 .map_err(RouterError::FrameBuild)?;
4329
4330 if let Err(err) = module_sink.send(probe_frame).await {
4331 return Ok(vec![control_error_frame(
4332 &frame,
4333 "target_unavailable",
4334 err.to_string(),
4335 )?]);
4336 }
4337
4338 match timeout_at(deadline, receiver).await {
4339 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4340 guard.disarm();
4341 let Some(report) = response.health_report() else {
4342 return Ok(vec![control_error_frame(
4343 &frame,
4344 "invalid_control_body",
4345 "health.check RPC returned a non-health response",
4346 )?]);
4347 };
4348 let HealthReport {
4353 status,
4354 detail,
4355 metrics,
4356 } = report;
4357 let capability_detail = self
4358 .capability_evaluator
4359 .required_problem_detail(&module_id);
4360 let response = ClientControlResponse::SupervisorHealthProbe {
4361 module_id,
4362 status,
4363 detail: append_capability_problem_detail(detail, capability_detail),
4364 metrics,
4365 };
4366 Ok(vec![control_response_body_frame(
4367 &frame,
4368 &response,
4369 "ClientControlResponse::SupervisorHealthProbe",
4370 )?])
4371 }
4372 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4373 guard.disarm();
4374 Ok(vec![control_error_body_frame(&frame, body)?])
4375 }
4376 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4377 guard.disarm();
4378 Ok(vec![control_error_frame(
4379 &frame,
4380 "target_unavailable",
4381 message,
4382 )?])
4383 }
4384 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4385 guard.disarm();
4386 Ok(vec![control_error_frame(
4387 &frame,
4388 "invalid_control_body",
4389 message,
4390 )?])
4391 }
4392 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4393 guard.disarm();
4394 Ok(vec![control_error_frame(
4395 &frame,
4396 "invalid_control_body",
4397 format!("expected module-control op '{expected}', got '{actual}'"),
4398 )?])
4399 }
4400 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4401 guard.disarm();
4402 Ok(vec![control_error_frame(
4403 &frame,
4404 "module_timeout",
4405 format!(
4406 "module_id '{module_id}' answered health.check after {:?}",
4407 self.health_probe_timeout
4408 ),
4409 )?])
4410 }
4411 Ok(Err(_)) => Ok(vec![control_error_frame(
4412 &frame,
4413 "target_unavailable",
4414 "health.check waiter was canceled before the module responded",
4415 )?]),
4416 Err(_) => Ok(vec![control_error_frame(
4417 &frame,
4418 "module_timeout",
4419 format!(
4420 "module_id '{module_id}' did not answer health.check within {:?}",
4421 self.health_probe_timeout
4422 ),
4423 )?]),
4424 }
4425 }
4426
4427 fn supervisor_status(
4428 &self,
4429 module_id: &str,
4430 corr: u64,
4431 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4432 self.supervisor
4433 .get(module_id)
4434 .map(|module| {
4435 let warming = module.is_warming_for_control("status").map_err(|err| {
4436 RouterError::backend(
4437 0,
4438 corr,
4439 format!(
4440 "failed to read supervisor warming state for module_id '{module_id}': {err}"
4441 ),
4442 )
4443 })?;
4444 module.status_for_control("status").map_err(|err| {
4445 RouterError::backend(
4446 0,
4447 corr,
4448 format!(
4449 "failed to read supervisor status for module_id '{module_id}': {err}"
4450 ),
4451 )
4452 }).map(|status| (status, warming))
4453 })
4454 .transpose()
4455 }
4456
4457 fn guard_module_control_op(
4458 &self,
4459 frame: &Frame,
4460 module_id: &str,
4461 op: &str,
4462 ) -> Result<Option<Frame>, RouterError> {
4463 if self.module_grants_op(module_id, op, frame.header.corr)? {
4464 return Ok(None);
4465 }
4466
4467 Ok(Some(control_error_frame(
4468 frame,
4469 "op_not_allowed",
4470 format!("module_id '{module_id}' did not grant control op '{op}'"),
4471 )?))
4472 }
4473
4474 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4475 let Some(registration) = self
4476 .registry
4477 .get_module(module_id)
4478 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4479 else {
4480 return Ok(false);
4481 };
4482 Ok(module_registration_grants_op(®istration.control_ops, op))
4483 }
4484
4485 fn handle_status_update(
4486 &self,
4487 endpoint: ModuleEndpointId,
4488 frame: Frame,
4489 ) -> Result<Vec<Frame>, RouterError> {
4490 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4491 Ok(update) => update,
4492 Err(err) => {
4493 if is_known_module_push_op(&frame.body) {
4498 return Ok(vec![control_error_frame(
4499 &frame,
4500 "invalid_control_body",
4501 format!("malformed module control push body: {err}"),
4502 )?]);
4503 }
4504 return Ok(Vec::new());
4505 }
4506 };
4507
4508 match update {
4509 ModuleControlPush::RouteStatus {
4510 route_channel,
4511 route_epoch,
4512 status,
4513 } => {
4514 self.forwarding
4515 .cache_status(endpoint, route_channel, route_epoch, status)
4516 .map_err(RouterError::Forwarding)?;
4517 }
4518 }
4519 Ok(Vec::new())
4520 }
4521
4522 fn handle_route_poll(
4523 &self,
4524 ctx: &RouteCtx,
4525 frame: Frame,
4526 route_channel: u16,
4527 route_epoch: u32,
4528 kind: PollKind,
4529 ) -> Result<Vec<Frame>, RouterError> {
4530 let snapshot = self
4531 .forwarding
4532 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4533 .map_err(RouterError::Forwarding)?;
4534 let response = match (kind, snapshot) {
4535 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4536 ClientControlResponse::RoutePoll {
4537 route_channel,
4538 route_epoch,
4539 status,
4540 live: None,
4541 }
4542 }
4543 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4544 route_channel,
4545 route_epoch,
4546 status: None,
4547 live: None,
4548 },
4549 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
4550 let live = self
4576 .process_liveness
4577 .as_ref()
4578 .and_then(|source| source.process_live(&module_id))
4579 .unwrap_or(true);
4580 ClientControlResponse::RoutePoll {
4581 route_channel,
4582 route_epoch,
4583 status: None,
4584 live: Some(live),
4585 }
4586 }
4587 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4588 route_channel,
4589 route_epoch,
4590 status: None,
4591 live: Some(false),
4592 },
4593 };
4594
4595 Ok(vec![control_response_body_frame(
4596 &frame,
4597 &response,
4598 "ClientControlResponse::RoutePoll",
4599 )?])
4600 }
4601
4602 pub(crate) fn observe_module_control_completion(
4603 &self,
4604 completion: ModuleControlRpcCompletion,
4605 ) -> bool {
4606 match completion {
4607 ModuleControlRpcCompletion::Unknown => false,
4608 ModuleControlRpcCompletion::Settled => true,
4609 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
4610 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
4611 info!(
4612 module_id = %module_id,
4613 latency_ms,
4614 "late health.check answer proves the module is alive"
4615 );
4616 match self
4617 .supervisor
4618 .record_late_health_answer(&module_id, latency_ms)
4619 {
4620 Ok(true) => {}
4621 Ok(false) => debug!(
4622 module_id = %module_id,
4623 latency_ms,
4624 "late health.check answer has no active supervisor snapshot"
4625 ),
4626 Err(err) => warn!(
4627 module_id = %module_id,
4628 latency_ms,
4629 error = %err,
4630 "failed to record late health.check answer"
4631 ),
4632 }
4633 true
4634 }
4635 }
4636 }
4637
4638 fn refuse_to_end_module_connection_for_a_client(
4659 &self,
4660 module_connection_id: ConnectionId,
4661 corr: u64,
4662 err: ForwardingError,
4663 ) -> Result<(), RouterError> {
4664 if let ForwardingError::ConnectionClosing { connection_id } = err {
4665 if connection_id != module_connection_id {
4666 warn!(
4667 module_connection_id = module_connection_id.get(),
4668 client_connection_id = connection_id.get(),
4669 corr,
4670 "dropping a route.bind response for a closing client; the module connection keeps serving"
4671 );
4672 return Ok(());
4673 }
4674 }
4675 Err(RouterError::Forwarding(err))
4676 }
4677
4678 fn handle_module_relay_response(
4679 &self,
4680 connection_id: ConnectionId,
4681 frame: Frame,
4682 ) -> Result<Vec<Frame>, RouterError> {
4683 let mut secondary_error = None;
4684 let outcome = match frame.header.ty {
4685 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
4686 Ok(probe) if probe.op == "route.bind" => {
4687 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
4688 Ok(ModuleControlResponse::RouteBindAck {}) => {
4689 RouteBindRelayOutcome::Accepted
4690 }
4691 Ok(other) => {
4692 let message =
4693 format!("route.bind response carried unexpected body: {other:?}");
4694 secondary_error = Some(control_error_frame(
4695 &frame,
4696 "invalid_control_body",
4697 message.clone(),
4698 )?);
4699 RouteBindRelayOutcome::ModuleGone(message)
4700 }
4701 Err(err) => {
4702 let message = format!("malformed route.bind response body: {err}");
4703 secondary_error = Some(control_error_frame(
4704 &frame,
4705 "invalid_control_body",
4706 message.clone(),
4707 )?);
4708 RouteBindRelayOutcome::ModuleGone(message)
4709 }
4710 }
4711 }
4712 Ok(probe) => {
4713 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
4714 {
4715 Ok(response) => ModuleControlRpcOutcome::Response(response),
4716 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4717 "malformed {} response body: {err}",
4718 probe.op
4719 )),
4720 };
4721 let completion = self
4722 .forwarding
4723 .complete_module_control_rpc(
4724 connection_id,
4725 frame.header.corr,
4726 Some(&probe.op),
4727 outcome,
4728 )
4729 .map_err(RouterError::Forwarding)?;
4730 if !self.observe_module_control_completion(completion) {
4731 debug!(
4732 connection_id = connection_id.get(),
4733 corr = frame.header.corr,
4734 op = %probe.op,
4735 "dropping late or unknown module-control RPC response"
4736 );
4737 }
4738 return Ok(Vec::new());
4739 }
4740 Err(err) => {
4741 if let Some(expected_op) = self
4742 .forwarding
4743 .pending_module_control_op(connection_id, frame.header.corr)
4744 .map_err(RouterError::Forwarding)?
4745 {
4746 let completion = self
4747 .forwarding
4748 .complete_module_control_rpc(
4749 connection_id,
4750 frame.header.corr,
4751 None,
4752 ModuleControlRpcOutcome::MalformedResponse(format!(
4753 "malformed {expected_op} response body: {err}"
4754 )),
4755 )
4756 .map_err(RouterError::Forwarding)?;
4757 if !self.observe_module_control_completion(completion) {
4758 debug!(
4759 connection_id = connection_id.get(),
4760 corr = frame.header.corr,
4761 "dropping late malformed module-control RPC response"
4762 );
4763 }
4764 return Ok(Vec::new());
4765 }
4766 let message = format!("malformed route.bind response body: {err}");
4767 secondary_error = Some(control_error_frame(
4768 &frame,
4769 "invalid_control_body",
4770 message.clone(),
4771 )?);
4772 RouteBindRelayOutcome::ModuleGone(message)
4773 }
4774 },
4775 FrameType::Error => {
4776 if self
4777 .forwarding
4778 .pending_module_control_op(connection_id, frame.header.corr)
4779 .map_err(RouterError::Forwarding)?
4780 .is_some()
4781 {
4782 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
4783 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
4784 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
4785 "malformed module-control ERROR body: {err}"
4786 )),
4787 };
4788 let completion = self
4789 .forwarding
4790 .complete_module_control_rpc(
4791 connection_id,
4792 frame.header.corr,
4793 None,
4794 outcome,
4795 )
4796 .map_err(RouterError::Forwarding)?;
4797 if !self.observe_module_control_completion(completion) {
4798 debug!(
4799 connection_id = connection_id.get(),
4800 corr = frame.header.corr,
4801 "dropping late or unknown module-control RPC error"
4802 );
4803 }
4804 return Ok(Vec::new());
4805 }
4806 match serde_json::from_slice::<ErrorBody>(&frame.body) {
4807 Ok(body) => RouteBindRelayOutcome::Rejected(body),
4808 Err(err) => {
4809 let message = format!("malformed route.bind ERROR body: {err}");
4810 secondary_error = Some(control_error_frame(
4811 &frame,
4812 "invalid_control_body",
4813 message.clone(),
4814 )?);
4815 RouteBindRelayOutcome::ModuleGone(message)
4816 }
4817 }
4818 }
4819 ty => {
4820 return Ok(vec![control_error_frame(
4821 &frame,
4822 "unsupported_control_frame",
4823 format!("unsupported module channel-0 frame {ty:?}"),
4824 )?])
4825 }
4826 };
4827
4828 let settled =
4829 self.forwarding
4830 .complete_pending_relay(connection_id, frame.header.corr, outcome);
4831 let completion = match settled {
4832 Ok(completion) => completion,
4833 Err(err) => {
4834 self.refuse_to_end_module_connection_for_a_client(
4835 connection_id,
4836 frame.header.corr,
4837 err,
4838 )?;
4839 return Ok(secondary_error.into_iter().collect());
4840 }
4841 };
4842 if let Some(target) = completion.abandoned.as_ref() {
4843 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
4844 }
4845 if !completion.settled {
4846 debug!(
4847 connection_id = connection_id.get(),
4848 corr = frame.header.corr,
4849 frame_type = ?frame.header.ty,
4850 "dropping late or unknown route.bind relay response"
4851 );
4852 }
4853 Ok(secondary_error.into_iter().collect())
4854 }
4855
4856 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
4857 debug!(connection_id = connection_id.get(), "handling GOODBYE");
4858 let registrations = self
4859 .deregister_connection(connection_id)
4860 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
4861 let released_routes = self
4862 .forwarding
4863 .cleanup_connection(connection_id)
4864 .map_err(RouterError::Forwarding)?;
4865 self.emit_route_goodbyes(released_routes);
4866 if !registrations.is_empty() {
4868 crate::supervise::notify_registration_release();
4869 }
4870 Ok(Vec::new())
4871 }
4872}
4873
4874impl Default for ControlHandler {
4875 fn default() -> Self {
4876 Self::new(Arc::new(Registry::default()))
4877 }
4878}
4879
4880impl crate::supervise::SwapPromotionObserver for ControlHandler {
4881 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
4882 self.apply_registration_capabilities(registration);
4883 }
4884}
4885
4886fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
4887 CapabilityRequirementStatus {
4888 consumer: status.consumer,
4889 capability: status.capability,
4890 need: match status.need {
4891 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
4892 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
4893 },
4894 verdict: status.verdict.as_str().to_string(),
4895 episode_seq: status.episode_seq,
4896 config_satisfiable: status.config_satisfiable,
4897 runtime_available: status.runtime_available,
4898 detail: status.detail,
4899 }
4900}
4901
4902fn append_capability_problem_detail(
4903 detail: Option<String>,
4904 capability_detail: Option<String>,
4905) -> Option<String> {
4906 match (detail, capability_detail) {
4907 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
4908 (Some(detail), None) => Some(detail),
4909 (None, Some(capability_detail)) => Some(capability_detail),
4910 (None, None) => None,
4911 }
4912}
4913
4914fn subc_ops() -> Vec<String> {
4915 SUBC_CONTROL_OPS
4916 .iter()
4917 .map(|op| (*op).to_string())
4918 .collect()
4919}
4920
4921fn module_subc_ops() -> Vec<String> {
4922 SUBC_CONTROL_OPS
4923 .iter()
4924 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
4925 .map(|op| (*op).to_string())
4926 .collect()
4927}
4928
4929#[cfg(test)]
4930fn module_baseline_control_ops() -> Vec<String> {
4931 MODULE_BASELINE_CONTROL_OPS
4932 .iter()
4933 .map(|op| (*op).to_string())
4934 .collect()
4935}
4936
4937fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
4938 let mut seen = HashSet::new();
4939 let mut effective = Vec::new();
4940 for op in MODULE_BASELINE_CONTROL_OPS {
4941 if seen.insert((*op).to_string()) {
4942 effective.push((*op).to_string());
4943 }
4944 }
4945 for op in declared.unwrap_or_default() {
4946 if seen.insert(op.clone()) {
4947 effective.push(op);
4948 }
4949 }
4950 effective
4951}
4952
4953fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
4954 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
4955}
4956
4957fn target_module_id(target: &RouteTarget) -> &str {
4958 match target {
4959 RouteTarget::ToolProvider { module_id }
4960 | RouteTarget::ManagementSurface { module_id }
4961 | RouteTarget::InternalService { module_id, .. } => module_id,
4962 }
4963}
4964
4965fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
4966 roles.iter().any(|role| match (target, role) {
4967 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
4968 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
4969 (
4970 RouteTarget::InternalService { service_id, .. },
4971 ProviderRole::InternalService {
4972 service_id: provided,
4973 ..
4974 },
4975 ) => service_id == provided,
4976 _ => false,
4977 })
4978}
4979
4980fn is_routable_role(role: &ProviderRole) -> bool {
4981 matches!(
4982 role,
4983 ProviderRole::ToolProvider { .. }
4984 | ProviderRole::ManagementSurface { .. }
4985 | ProviderRole::InternalService { .. }
4986 )
4987}
4988
4989#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4990enum ControlRequestBodyError {
4991 UnknownOp,
4992 InvalidBody,
4993}
4994
4995#[derive(Debug, Deserialize)]
4996struct ControlOpProbe {
4997 op: String,
4998}
4999
5000const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5003
5004fn is_known_module_push_op(body: &[u8]) -> bool {
5005 serde_json::from_slice::<ControlOpProbe>(body)
5006 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5007 .unwrap_or(false)
5008}
5009
5010fn is_known_module_request_op(body: &[u8]) -> bool {
5011 serde_json::from_slice::<ControlOpProbe>(body)
5012 .map(|probe| MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()))
5013 .unwrap_or(false)
5014}
5015
5016fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5017 debug!(
5018 op = %op,
5019 connection_id = connection_id.get(),
5020 corr,
5021 "control dispatch"
5022 );
5023}
5024
5025fn log_slow_control_dispatch(
5026 dispatch_started_at: Option<StdInstant>,
5027 op: &'static str,
5028 connection_id: ConnectionId,
5029 corr: u64,
5030) {
5031 let Some(dispatch_started_at) = dispatch_started_at else {
5032 return;
5033 };
5034 let elapsed = dispatch_started_at.elapsed();
5035 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5036 warn!(
5037 op = %op,
5038 connection_id = connection_id.get(),
5039 corr,
5040 elapsed_ms = elapsed.as_millis() as u64,
5041 "slow control dispatch"
5042 );
5043 }
5044}
5045
5046fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5047 match request {
5048 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5049 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5050 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5051 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5052 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5053 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5054 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5055 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5056 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5057 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5058 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5059 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5060 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5061 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5062 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5063 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5064 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5065 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5066 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5067 }
5068}
5069
5070fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5071 match request {
5072 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5073 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5074 }
5075}
5076
5077fn parse_client_control_request(
5078 body: &[u8],
5079) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5080 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5081 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5082 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5083 ControlRequestBodyError::InvalidBody
5084 }
5085 Ok(_) => ControlRequestBodyError::UnknownOp,
5086 Err(_) => ControlRequestBodyError::InvalidBody,
5087 };
5088 (err, classification)
5089 })
5090}
5091
5092fn parse_module_control_request_from_module(
5093 body: &[u8],
5094) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5095 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5096 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5097 Ok(probe) if MODULE_TO_SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5098 ControlRequestBodyError::InvalidBody
5099 }
5100 Ok(_) => ControlRequestBodyError::UnknownOp,
5101 Err(_) => ControlRequestBodyError::InvalidBody,
5102 };
5103 (err, classification)
5104 })
5105}
5106
5107#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5108enum ProviderRoleKind {
5109 ToolProvider,
5110 PipelineStage,
5111 ManagementSurface,
5112 InternalService,
5113}
5114
5115fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5116 match role {
5117 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5118 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5119 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5120 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5121 }
5122}
5123
5124fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5125 roles.iter().map(provider_role_kind).collect()
5126}
5127
5128fn capability_census_trigger(
5132 old: Option<&CapabilityDeclarations>,
5133 new: Option<&CapabilityDeclarations>,
5134) -> bool {
5135 let old_provides = old
5136 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5137 .unwrap_or_default();
5138 let old_denies = old
5139 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5140 .unwrap_or_default();
5141 let new = new.cloned().unwrap_or(CapabilityDeclarations {
5142 provides: Vec::new(),
5143 requires: Vec::new(),
5144 must_never_reach: Vec::new(),
5145 });
5146
5147 new.provides
5148 .iter()
5149 .any(|capability| !old_provides.contains(capability))
5150 || new
5151 .must_never_reach
5152 .iter()
5153 .any(|capability| !old_denies.contains(capability))
5154}
5155
5156fn denied_capability<'a>(
5159 opening_manifest: &'a ModuleManifest,
5160 target_manifest: &ModuleManifest,
5161) -> Option<&'a str> {
5162 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5163 let target_capabilities = target_manifest.capabilities.as_ref()?;
5164 opening_capabilities
5165 .must_never_reach
5166 .iter()
5167 .find(|denied| {
5168 target_capabilities
5169 .provides
5170 .iter()
5171 .any(|provided| provided == *denied)
5172 })
5173 .map(String::as_str)
5174}
5175
5176fn catalog_update_frozen_field_message(
5177 registered: &ModuleManifest,
5178 provides: &[ProviderRole],
5179) -> Option<String> {
5180 let old_has_provides = !registered.provides.is_empty();
5181 let new_has_provides = !provides.is_empty();
5182 if old_has_provides != new_has_provides {
5183 return Some(format!(
5184 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5185 registered.module_id
5186 ));
5187 }
5188
5189 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
5190 return Some(format!(
5191 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5192 registered.module_id
5193 ));
5194 }
5195
5196 let registered_concurrency = manifest_concurrency(registered);
5197 let mut candidate = registered.clone();
5198 candidate.provides = provides.to_vec();
5199 let candidate_concurrency = manifest_concurrency(&candidate);
5200 if candidate_concurrency != registered_concurrency {
5201 return Some(format!(
5202 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5203 registered.module_id, registered_concurrency, candidate_concurrency
5204 ));
5205 }
5206
5207 None
5210}
5211
5212fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5213 manifest.provides.iter().any(is_routable_role)
5214}
5215
5216fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5222 manifest
5223 .provides
5224 .iter()
5225 .find_map(|provider| match provider {
5226 ProviderRole::ToolProvider { concurrency, .. }
5227 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5228 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5229 })
5230 .unwrap_or(Concurrency::ModuleManaged)
5231}
5232
5233fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5240 let has_management_surface = manifest
5241 .provides
5242 .iter()
5243 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5244 if !has_management_surface {
5245 return false;
5246 }
5247 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5248 return false;
5249 };
5250 let Some(provides) = raw
5251 .get("manifest")
5252 .and_then(|manifest| manifest.get("provides"))
5253 .and_then(serde_json::Value::as_array)
5254 else {
5255 return false;
5256 };
5257 provides.iter().any(|role| {
5262 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5263 && role.get("concurrency").is_none()
5264 })
5265}
5266
5267fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5268 if peer_version != PROTOCOL_VERSION {
5269 return Err(format!(
5270 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5271 ));
5272 }
5273 Ok(PROTOCOL_VERSION)
5274}
5275
5276fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5277 Frame::build_with_version(
5278 response_version(frame),
5279 FrameType::Pong,
5280 frame.header.flags,
5281 0,
5282 0,
5283 frame.header.corr,
5284 Vec::new(),
5285 )
5286 .map_err(RouterError::FrameBuild)
5287}
5288
5289fn control_error_frame(
5290 frame: &Frame,
5291 code: &'static str,
5292 message: impl Into<String>,
5293) -> Result<Frame, RouterError> {
5294 control_error_body_frame(
5295 frame,
5296 ErrorBody {
5297 code: code.to_string(),
5298 message: message.into(),
5299 detail: None,
5300 },
5301 )
5302}
5303
5304fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5305 let body = serde_json::to_vec(&error).map_err(|err| {
5306 RouterError::backend(
5307 0,
5308 frame.header.corr,
5309 format!("failed to encode control ERROR: {err}"),
5310 )
5311 })?;
5312
5313 Frame::build_with_version(
5314 response_version(frame),
5315 FrameType::Error,
5316 control_flags(),
5317 0,
5318 0,
5319 frame.header.corr,
5320 body,
5321 )
5322 .map_err(RouterError::FrameBuild)
5323}
5324
5325fn control_response_body_frame<T: Serialize>(
5326 frame: &Frame,
5327 reply: &T,
5328 label: &'static str,
5329) -> Result<Frame, RouterError> {
5330 let body = serde_json::to_vec(reply).map_err(|err| {
5331 RouterError::backend(
5332 0,
5333 frame.header.corr,
5334 format!("failed to encode {label}: {err}"),
5335 )
5336 })?;
5337
5338 Frame::build_with_version(
5339 response_version(frame),
5340 FrameType::Response,
5341 control_flags(),
5342 0,
5343 0,
5344 frame.header.corr,
5345 body,
5346 )
5347 .map_err(RouterError::FrameBuild)
5348}
5349
5350fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5380 match err {
5381 ForwardingError::NoModuleConnection => "target_unavailable",
5382 ForwardingError::ModuleReloading { .. } => "module_reloading",
5383 ForwardingError::ClientRouteChannelExhausted { .. }
5384 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5385 ForwardingError::StaleModuleEndpoint
5386 | ForwardingError::UnknownReservation { .. }
5387 | ForwardingError::ConnectionClosing { .. }
5388 | ForwardingError::ClientEgressClosed { .. }
5389 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5390 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5393 ForwardingError::RelayCorrelationExhausted
5394 | ForwardingError::RouteOpenBuild(_)
5395 | ForwardingError::Poisoned => "forwarding_error",
5396 }
5397}
5398
5399fn response_version(frame: &Frame) -> u8 {
5400 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5401 frame.header.ver
5402 } else {
5403 PROTOCOL_VERSION
5404 }
5405}
5406
5407fn control_flags() -> Flags {
5408 Flags::new(false, Priority::Passive, false)
5409}
5410
5411fn send_goodbye_target_best_effort(
5416 counters: &DaemonCounters,
5417 target: &GoodbyeTarget,
5418 context: &'static str,
5419) {
5420 let Ok(frame) = Frame::build_with_version(
5421 target.negotiated_ver,
5422 FrameType::Goodbye,
5423 control_flags(),
5424 target.channel,
5425 target.epoch,
5426 0,
5427 Vec::new(),
5428 ) else {
5429 return;
5430 };
5431 crate::forwarding::send_module_route_goodbye(
5432 counters,
5433 &target.sink,
5434 frame,
5435 target.module_id.as_deref(),
5436 context,
5437 );
5438}
5439
5440pub(crate) fn send_route_control_pushes(
5441 forwarding: &ForwardingTable,
5442 routes: Vec<EndpointRoute>,
5443 push: ClientControlPush,
5444) {
5445 let body = match serde_json::to_vec(&push) {
5446 Ok(body) => body,
5447 Err(err) => {
5448 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5449 return;
5450 }
5451 };
5452 let mut targets = Vec::new();
5453 for route in routes {
5454 let target = route.goodbye_target;
5455 if let Some(existing) = targets
5456 .iter()
5457 .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5458 {
5459 debug_assert_eq!(
5460 existing.negotiated_ver, target.negotiated_ver,
5461 "one connection cannot negotiate multiple frame versions"
5462 );
5463 continue;
5464 }
5465 targets.push(target);
5466 }
5467 for target in targets {
5468 let frame = match Frame::build_with_version(
5469 target.negotiated_ver,
5470 FrameType::Push,
5471 control_flags(),
5472 0,
5473 0,
5474 0,
5475 body.clone(),
5476 ) {
5477 Ok(frame) => frame,
5478 Err(err) => {
5479 warn!(
5480 route_channel = target.channel,
5481 error = %err,
5482 "failed to build route lifecycle control PUSH frame"
5483 );
5484 continue;
5485 }
5486 };
5487 if let Err(err) = target.sink.try_send(frame) {
5488 if target.close_on_delivery_failure() {
5489 warn!(
5490 target_connection_id = target.connection_id.get(),
5491 route_channel = target.channel,
5492 error = %err,
5493 "route lifecycle control PUSH was not delivered to client; closing target connection"
5494 );
5495 let _ = forwarding.escalate_client_delivery_failure(
5496 target.connection_id,
5497 target.channel,
5498 target.epoch,
5499 CloseReason::new(
5500 "route_lifecycle_push_delivery_failed",
5501 format!(
5502 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
5503 target.channel
5504 ),
5505 ),
5506 crate::forwarding::UndeliveredFrame {
5507 module_id: target.module_id.as_deref(),
5508 sink: &target.sink,
5509 },
5510 );
5511 }
5512 }
5513 }
5514}
5515
5516#[cfg(test)]
5517mod tests {
5518 use std::{
5519 collections::BTreeMap,
5520 fmt,
5521 path::PathBuf,
5522 sync::{Arc, Mutex},
5523 time::Duration,
5524 };
5525 use subc_test_support::TestTempDir;
5526
5527 use serde_json::{json, Value};
5528 use subc_protocol::{
5529 manifest::{
5530 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
5531 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
5532 },
5533 session::HealthStatus,
5534 FrameType,
5535 };
5536
5537 use super::*;
5538 use crate::{
5539 forwarding::{DataRoute, DataRouteState},
5540 registry::ChannelState,
5541 router::FrameSink,
5542 stderr_tail::DEFAULT_MAX_LINE_BYTES,
5543 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
5544 RouteCtx, Router,
5545 };
5546 use tokio::{
5547 sync::mpsc,
5548 time::{sleep, Instant},
5549 };
5550 use tracing::{
5551 field::{Field, Visit},
5552 Event, Subscriber,
5553 };
5554 use tracing_subscriber::{layer::Context, prelude::*, Layer};
5555
5556 fn fake_aft_stub_path() -> PathBuf {
5575 let mut path = std::env::current_exe().expect("current_exe available in tests");
5576 path.pop(); path.pop(); path.push(if cfg!(windows) {
5579 "fake-aft-stub.exe"
5580 } else {
5581 "fake-aft-stub"
5582 });
5583 assert!(
5584 path.exists(),
5585 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
5586 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
5587 path.display()
5588 );
5589 path
5590 }
5591
5592 fn client_retries(code: &str) -> bool {
5597 subc_protocol::error_codes::is_retryable_route_open(code)
5598 }
5599
5600 #[test]
5611 fn retryability_of_forwarding_codes_matches_the_failure() {
5612 let transient = [
5615 ForwardingError::NoModuleConnection,
5616 ForwardingError::ModuleReloading {
5617 module_id: "m".into(),
5618 },
5619 ForwardingError::StaleModuleEndpoint,
5620 ForwardingError::UnknownReservation {
5621 client_channel: 1,
5622 module_channel: 1,
5623 },
5624 ForwardingError::ConnectionClosing {
5625 connection_id: ConnectionId::new(1),
5626 },
5627 ForwardingError::ClientEgressClosed {
5628 connection_id: ConnectionId::new(1),
5629 },
5630 ForwardingError::ModuleEgressUnavailable {
5631 connection_id: ConnectionId::new(1),
5632 },
5633 ];
5634 for err in transient {
5635 let code = forwarding_error_code(&err);
5636 assert!(
5637 client_retries(code),
5638 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
5639 );
5640 }
5641
5642 let permanent = [
5647 ForwardingError::ClientRouteChannelExhausted {
5648 connection_id: ConnectionId::new(1),
5649 },
5650 ForwardingError::ModuleRouteChannelExhausted {
5651 endpoint: ModuleEndpointId {
5652 connection_id: ConnectionId::new(1),
5653 generation: 1,
5654 },
5655 },
5656 ForwardingError::RelayCorrelationExhausted,
5657 ForwardingError::RouteOpenBuild("x".into()),
5658 ForwardingError::Poisoned,
5659 ];
5660 for err in permanent {
5661 let code = forwarding_error_code(&err);
5662 assert!(
5663 !client_retries(code),
5664 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
5665 );
5666 }
5667 }
5668
5669 #[tokio::test]
5697 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
5698 let handler = ControlHandler::default();
5699 let frame =
5700 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
5701
5702 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
5706 assert_eq!(
5707 stamped,
5708 Principal::Direct,
5709 "a caller that proved nothing must not be stamped as a supervised module"
5710 );
5711
5712 let forged = handler
5717 .route_open_principal(
5718 &frame,
5719 Some(ConsumerIdentity {
5720 module_id: "aft".to_string(),
5721 launch_nonce: "not-a-real-nonce".to_string(),
5722 }),
5723 )
5724 .unwrap();
5725 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
5726 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
5727 }
5728
5729 #[test]
5746 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
5747 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
5748 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
5749 let ClientControlRequest::RouteOpen {
5750 consumer_identity, ..
5751 } = parsed
5752 else {
5753 panic!("route.open body must parse as RouteOpen");
5754 };
5755 assert_eq!(
5756 consumer_identity,
5757 Some(ConsumerIdentity {
5758 module_id: "aft".to_string(),
5759 launch_nonce: "n".to_string(),
5760 }),
5761 "the wire field name must reach the value route_open_principal reads"
5762 );
5763 }
5764
5765 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
5766 ModuleManifest::builder(module_id, "0.1.0")
5767 .protocol_ver(protocol_ver)
5768 .provides(vec![ProviderRole::ToolProvider {
5769 tools: vec![Tool {
5770 name: "read".to_string(),
5771 description: None,
5772 execution_mode: ExecutionMode::Pure,
5773 schema: json!({"type": "object"}),
5774 }],
5775 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
5776 concurrency: Concurrency::ModuleManaged,
5777 emits_push: true,
5778 sub_supervises: true,
5779 }])
5780 .build()
5781 }
5782
5783 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
5784 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
5785 }
5786
5787 fn hello_frame_with_control_ops(
5788 module_id: &str,
5789 protocol_ver: u8,
5790 corr: u64,
5791 control_ops: Option<Vec<String>>,
5792 ) -> Frame {
5793 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
5794 }
5795
5796 fn hello_frame_with_nonce(
5797 module_id: &str,
5798 protocol_ver: u8,
5799 corr: u64,
5800 launch_nonce: Option<&str>,
5801 ) -> Frame {
5802 hello_frame_full(
5803 module_id,
5804 protocol_ver,
5805 corr,
5806 None,
5807 launch_nonce.map(ToOwned::to_owned),
5808 )
5809 }
5810
5811 fn hello_frame_full(
5812 module_id: &str,
5813 protocol_ver: u8,
5814 corr: u64,
5815 control_ops: Option<Vec<String>>,
5816 launch_nonce: Option<String>,
5817 ) -> Frame {
5818 let body = serde_json::to_vec(&ModuleHelloBody {
5819 manifest: manifest(module_id, protocol_ver),
5820 protocol_ver,
5821 control_ops,
5822 launch_nonce,
5823 })
5824 .unwrap();
5825 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5826 }
5827
5828 fn non_routable_hello_frame_with_control_ops(
5829 module_id: &str,
5830 corr: u64,
5831 control_ops: Option<Vec<String>>,
5832 ) -> Frame {
5833 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
5834 manifest.provides.clear();
5835 let body = serde_json::to_vec(&ModuleHelloBody {
5836 manifest,
5837 protocol_ver: PROTOCOL_VERSION,
5838 control_ops,
5839 launch_nonce: None,
5840 })
5841 .unwrap();
5842 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
5843 }
5844
5845 fn capability_grammar_hello_frame(
5846 capabilities: Value,
5847 runtime_computed: Option<Value>,
5848 corr: u64,
5849 ) -> Frame {
5850 let mut body = serde_json::to_value(ModuleHelloBody {
5851 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
5852 protocol_ver: PROTOCOL_VERSION,
5853 control_ops: None,
5854 launch_nonce: None,
5855 })
5856 .expect("HELLO body serializes");
5857 body["manifest"]["capabilities"] = capabilities;
5858 if let Some(runtime_computed) = runtime_computed {
5859 body["runtime_computed"] = runtime_computed;
5860 }
5861 Frame::build(
5862 FrameType::Hello,
5863 control_flags(),
5864 0,
5865 0,
5866 corr,
5867 serde_json::to_vec(&body).expect("HELLO body reserializes"),
5868 )
5869 .expect("HELLO frame builds")
5870 }
5871
5872 fn channel_request(channel: u16, corr: u64) -> Frame {
5873 Frame::build(
5874 FrameType::Request,
5875 Flags::new(true, Priority::Interactive, false),
5876 channel,
5877 0,
5878 corr,
5879 b"opaque".to_vec(),
5880 )
5881 .unwrap()
5882 }
5883
5884 fn route_ctx(
5885 connection_id: ConnectionId,
5886 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
5887 let (tx, rx) = mpsc::channel(8);
5888 (
5889 RouteCtx {
5890 connection_id,
5891 egress: FrameSink::new(tx),
5892 },
5893 rx,
5894 )
5895 }
5896
5897 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
5898 serde_json::from_slice(&frame.body).unwrap()
5899 }
5900
5901 async fn hello_via_sink(
5906 handler: &ControlHandler,
5907 ctx: &RouteCtx,
5908 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
5909 hello: Frame,
5910 ) -> Frame {
5911 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
5912 assert!(
5913 replies.is_empty(),
5914 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
5915 );
5916 let ack = rx
5917 .try_recv()
5918 .expect("HELLO_ACK is queued on the module sink")
5919 .frame;
5920 assert_eq!(ack.header.ty, FrameType::HelloAck);
5921 ack
5922 }
5923
5924 fn parse_error(frame: &Frame) -> Value {
5925 serde_json::from_slice(&frame.body).unwrap()
5926 }
5927
5928 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
5929 serde_json::from_slice(&frame.body).unwrap()
5930 }
5931
5932 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
5933 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
5934 route_channel,
5935 route_epoch: 0,
5936 kind,
5937 })
5938 .unwrap();
5939 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5940 }
5941
5942 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
5943 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
5944 module_id: module_id.to_string(),
5945 })
5946 .unwrap();
5947 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5948 }
5949
5950 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
5951 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
5952 }
5953
5954 fn route_open_frame_with_consumer_capabilities(
5955 corr: u64,
5956 module_id: &str,
5957 project_root: TestTempDir,
5958 consumer_capabilities: Option<Vec<String>>,
5959 ) -> Frame {
5960 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5961 target: RouteTarget::ToolProvider {
5962 module_id: module_id.to_string(),
5963 },
5964 identity: BindIdentity::new(
5965 project_root.path().to_path_buf(),
5966 "unit".to_string(),
5967 "session".to_string(),
5968 ),
5969 consumer_identity: None,
5970 consumer_capabilities,
5971 admission_facts: None,
5972 })
5973 .unwrap();
5974 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5975 }
5976
5977 fn route_open_frame_with_admission_facts(
5978 corr: u64,
5979 module_id: &str,
5980 project_root: TestTempDir,
5981 consumer_identity: Option<subc_control::ConsumerIdentity>,
5982 facts: Option<Value>,
5983 ) -> Frame {
5984 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
5985 target: RouteTarget::ToolProvider {
5986 module_id: module_id.to_string(),
5987 },
5988 identity: BindIdentity::new(
5989 project_root.path().to_path_buf(),
5990 "unit".to_string(),
5991 format!("session-{corr}"),
5992 ),
5993 consumer_identity,
5994 consumer_capabilities: None,
5995 admission_facts: facts,
5996 })
5997 .unwrap();
5998 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
5999 }
6000
6001 #[derive(Clone, Default)]
6002 struct EventCapture {
6003 events: Arc<Mutex<Vec<CapturedEvent>>>,
6004 }
6005
6006 #[derive(Clone, Debug)]
6007 struct CapturedEvent {
6008 target: String,
6009 fields: BTreeMap<String, String>,
6010 }
6011
6012 impl EventCapture {
6013 fn events(&self) -> Vec<CapturedEvent> {
6014 self.events.lock().unwrap().clone()
6015 }
6016 }
6017
6018 impl<S> Layer<S> for EventCapture
6019 where
6020 S: Subscriber,
6021 {
6022 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6023 let mut visitor = EventFieldVisitor::default();
6024 event.record(&mut visitor);
6025 self.events.lock().unwrap().push(CapturedEvent {
6026 target: event.metadata().target().to_string(),
6027 fields: visitor.fields,
6028 });
6029 }
6030 }
6031
6032 #[derive(Default)]
6033 struct EventFieldVisitor {
6034 fields: BTreeMap<String, String>,
6035 }
6036
6037 impl Visit for EventFieldVisitor {
6038 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6039 self.fields
6040 .insert(field.name().to_string(), format!("{value:?}"));
6041 }
6042 }
6043
6044 fn health_response(corr: u64, status: HealthStatus) -> Frame {
6045 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6046 status,
6047 detail: Some("warming".to_string()),
6048 metrics: Some(json!({"queue_depth": 3})),
6049 })
6050 .unwrap();
6051 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6052 }
6053
6054 fn route_bind_ack(corr: u64) -> Frame {
6055 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6056 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6057 }
6058
6059 fn unique_project_root(label: &str) -> TestTempDir {
6060 TestTempDir::new(label)
6061 }
6062
6063 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6064 match parse_route_poll(frame) {
6065 ClientControlResponse::RoutePoll {
6066 status: None,
6067 live: Some(live),
6068 ..
6069 } => assert_eq!(live, expected_live),
6070 other => panic!("unexpected route.poll response: {other:?}"),
6071 }
6072 }
6073
6074 fn bind_liveness_route(
6075 registry: &Registry,
6076 forwarding: &ForwardingTable,
6077 module_id: &str,
6078 ) -> (RouteCtx, u16, u32) {
6079 let module_connection = ConnectionId::new(101);
6080 let client_connection = ConnectionId::new(202);
6081 let registration = registry
6082 .register_with_control_ops(
6083 manifest(module_id, PROTOCOL_VERSION),
6084 PROTOCOL_VERSION,
6085 module_connection,
6086 module_baseline_control_ops(),
6087 )
6088 .unwrap();
6089 let (module_tx, _module_rx) = mpsc::channel(8);
6090 let endpoint = forwarding
6091 .register_module_connection(
6092 module_connection,
6093 module_id.to_string(),
6094 PROTOCOL_VERSION,
6095 manifest_concurrency(®istration.manifest),
6096 FrameSink::new(module_tx),
6097 )
6098 .unwrap();
6099 let (client_ctx, _client_rx) = route_ctx(client_connection);
6100 let pending = forwarding
6101 .begin_route_bind_relay_for_test(
6102 client_connection,
6103 client_ctx.egress.clone(),
6104 1,
6105 module_id,
6106 )
6107 .unwrap();
6108 assert_eq!(pending.endpoint, endpoint);
6109 let route_channel = pending.client_channel;
6110 let route_epoch = pending.client_epoch;
6111 forwarding
6112 .complete_pending_relay(
6113 module_connection,
6114 pending.corr,
6115 RouteBindRelayOutcome::Accepted,
6116 )
6117 .unwrap();
6118 (client_ctx, route_channel, route_epoch)
6119 }
6120
6121 struct FakeProcessLiveness {
6122 live: Option<bool>,
6123 }
6124
6125 impl ModuleProcessLiveness for FakeProcessLiveness {
6126 fn process_live(&self, _module_id: &str) -> Option<bool> {
6127 self.live
6128 }
6129 }
6130
6131 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6132 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6133 {
6134 let registry = Arc::new(Registry::default());
6135 let supervisor_handle = SupervisorHandle::new();
6136 let supervisor = Supervisor::new(
6137 Arc::clone(®istry),
6138 RestartPolicy::new(1, Duration::from_millis(10)),
6139 )
6140 .with_handle(supervisor_handle.clone());
6141 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6142 let module = supervisor
6143 .spawn(ModuleSpec {
6144 module_id: "stderr-tail-wire".to_string(),
6145 program: fake_aft_stub_path(),
6146 args: Vec::new(),
6147 env: vec![
6148 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6149 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6150 ],
6151 reserved: false,
6152 reserved_prefixes: Vec::new(),
6153 protocol: ModuleProtocol::Subc,
6154 overlap: Default::default(),
6155 })
6156 .unwrap();
6157
6158 let deadline = Instant::now() + Duration::from_secs(5);
6159 loop {
6160 let tail = module.stderr_tail(None, None);
6161 if tail
6162 .entries
6163 .iter()
6164 .any(|entry| matches!(entry, TailEntry::ProcessStart))
6165 && tail.entries.iter().any(|entry| {
6166 matches!(
6167 entry,
6168 TailEntry::Line {
6169 truncated: true,
6170 ..
6171 }
6172 )
6173 })
6174 {
6175 break;
6176 }
6177 assert!(
6178 Instant::now() < deadline,
6179 "module did not produce a truncated line and restart boundary: {tail:?}"
6180 );
6181 sleep(Duration::from_millis(10)).await;
6182 }
6183
6184 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6185 let request = ClientControlRequest::SupervisorStderrTail {
6186 module_id: "stderr-tail-wire".to_string(),
6187 max_lines: None,
6188 max_bytes: None,
6189 };
6190 let frame = Frame::build(
6191 FrameType::Request,
6192 control_flags(),
6193 0,
6194 0,
6195 1,
6196 serde_json::to_vec(&request).unwrap(),
6197 )
6198 .unwrap();
6199 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6200 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6201 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6202 serde_json::from_slice(&responses[0].body).unwrap()
6203 else {
6204 panic!("expected supervisor.stderr_tail response");
6205 };
6206
6207 assert!(
6208 tail.entries
6209 .iter()
6210 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6211 "the control response lost the restart boundary"
6212 );
6213 let Some(StderrTailEntry::Line { text, truncated }) = tail.entries.iter().find(|entry| {
6214 matches!(
6215 entry,
6216 StderrTailEntry::Line {
6217 truncated: true,
6218 ..
6219 }
6220 )
6221 }) else {
6222 panic!("the control response lost the truncated line");
6223 };
6224 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6225 assert!(*truncated);
6226 }
6227
6228 #[tokio::test(flavor = "current_thread")]
6233 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6234 let dir = TestTempDir::new("terminals-off-worker");
6235 let journal_path = dir.join("terminals.jsonl");
6236 let registry = Arc::new(Registry::default());
6237 let supervisor_handle = SupervisorHandle::new();
6238 let supervisor =
6239 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6240 .with_handle(supervisor_handle.clone())
6241 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6242 let module = supervisor
6243 .spawn(ModuleSpec {
6244 module_id: "terminal-off-worker".to_string(),
6245 program: fake_aft_stub_path(),
6246 args: Vec::new(),
6247 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6248 reserved: false,
6249 reserved_prefixes: Vec::new(),
6250 protocol: ModuleProtocol::Subc,
6251 overlap: Default::default(),
6252 })
6253 .unwrap();
6254 let deadline = Instant::now() + Duration::from_secs(5);
6255 while module.terminal_history().entries.len() != 2 {
6256 assert!(Instant::now() < deadline, "module did not record two exits");
6257 sleep(Duration::from_millis(10)).await;
6258 }
6259
6260 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6261 let handler =
6262 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
6263 let frame = Frame::build(
6264 FrameType::Request,
6265 control_flags(),
6266 0,
6267 0,
6268 1,
6269 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6270 module_id: "terminal-off-worker".to_string(),
6271 })
6272 .unwrap(),
6273 )
6274 .unwrap();
6275 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6276 let spawned_at = std::time::Instant::now();
6277 let read = tokio::spawn({
6278 let handler = Arc::clone(&handler);
6279 async move { handler.handle_control_frame(&ctx, frame).await }
6280 });
6281 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6284 .await
6285 .unwrap()
6286 .expect("the history read reached its pause");
6287 let elapsed = spawned_at.elapsed();
6288 assert!(
6289 elapsed < Duration::from_secs(2) && !read.is_finished(),
6290 "this task could not run while the history read was paused \
6291 (resumed after {elapsed:?}, read finished: {})",
6292 read.is_finished()
6293 );
6294
6295 drop(release);
6296 let responses = read.await.unwrap().unwrap();
6297 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6298 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6299 panic!("expected supervisor.terminals response");
6300 };
6301 assert_eq!(terminals.entries.len(), 2);
6302 assert_eq!(terminals.journal_skipped_lines, 0);
6303 assert_eq!(terminals.journal_read_errors, 0);
6304 }
6305
6306 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6307 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6308 let registry = Arc::new(Registry::default());
6309 let supervisor_handle = SupervisorHandle::new();
6310 let supervisor =
6311 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6312 .with_handle(supervisor_handle.clone());
6313 let module = supervisor
6314 .spawn(ModuleSpec {
6315 module_id: "terminal-golden".to_string(),
6316 program: fake_aft_stub_path(),
6317 args: Vec::new(),
6318 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6319 reserved: false,
6320 reserved_prefixes: Vec::new(),
6321 protocol: ModuleProtocol::Subc,
6322 overlap: Default::default(),
6323 })
6324 .unwrap();
6325
6326 let deadline = Instant::now() + Duration::from_secs(5);
6327 while module.terminal_history().entries.len() != 2 {
6328 assert!(
6329 Instant::now() < deadline,
6330 "module did not retain two terminal exits: {:?}",
6331 module.terminal_history()
6332 );
6333 sleep(Duration::from_millis(10)).await;
6334 }
6335
6336 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6337 let request = ClientControlRequest::SupervisorTerminals {
6338 module_id: "terminal-golden".to_string(),
6339 };
6340 let frame = Frame::build(
6341 FrameType::Request,
6342 control_flags(),
6343 0,
6344 0,
6345 1,
6346 serde_json::to_vec(&request).unwrap(),
6347 )
6348 .unwrap();
6349 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6350 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6351 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6352 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6353 panic!("expected supervisor.terminals response");
6354 };
6355 assert_eq!(terminals.entries.len(), 2);
6356 assert_eq!(terminals.dropped, 0);
6357
6358 let mut rendered = serde_json::to_value(response).unwrap();
6359 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6362 for (index, entry) in rendered["entries"]
6363 .as_array_mut()
6364 .expect("terminal response entries array")
6365 .iter_mut()
6366 .enumerate()
6367 {
6368 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6369 }
6370
6371 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6372 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6373 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6374 if std::env::var_os("UPDATE_GOLDEN").is_some() {
6375 std::fs::write(&golden_path, &serialized).unwrap();
6376 }
6377 let expected: Value =
6378 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6379 assert_eq!(rendered, expected);
6380 }
6381
6382 #[test]
6383 fn hello_registers_manifest_and_returns_ack() {
6384 let registry = Arc::new(Registry::default());
6385 let handler = ControlHandler::new(Arc::clone(®istry));
6386 let conn = ConnectionId::new(1);
6387
6388 let responses = handler
6389 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6390 .unwrap();
6391
6392 assert_eq!(responses.len(), 1);
6393 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6394 assert_eq!(responses[0].header.channel, 0);
6395 assert_eq!(responses[0].header.corr, 7);
6396 let ack = parse_ack(&responses[0]);
6397 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6398 assert!(ack
6399 .subc_capabilities
6400 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6401 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6402 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6403 assert!(ack
6404 .subc_ops
6405 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6406 assert!(ack
6407 .subc_ops
6408 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6409
6410 let registration = registry.get_module("aft").unwrap().unwrap();
6411 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6412 assert_eq!(registration.state, ChannelState::Active);
6413 assert_eq!(registration.connection_id, conn);
6414 assert_eq!(registration.control_ops, module_baseline_control_ops());
6415 }
6416
6417 #[test]
6418 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
6419 let invalid_identifiers = [
6420 ("case_change", "credentials-Provider/v1"),
6421 ("leading_zero", "credentials-provider/v01"),
6422 ("trailing_hyphen", "credentials-provider-/v1"),
6423 ("consecutive_hyphens", "credentials--provider/v1"),
6424 ("uppercase", "Credentials-provider/v1"),
6425 ("missing_v", "credentials-provider/1"),
6426 ("whitespace", "credentials provider/v1"),
6427 ("zero_version", "credentials-provider/v0"),
6428 ("out_of_range_version", "credentials-provider/v4294967296"),
6429 (
6430 "overlength_name",
6431 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
6432 ),
6433 ];
6434 let mut cases = invalid_identifiers
6435 .into_iter()
6436 .map(|(name, identifier)| {
6437 (
6438 format!("identifier_{name}"),
6439 "capabilities.provides[0]".to_string(),
6440 identifier.to_string(),
6441 json!({ "provides": [identifier] }),
6442 None,
6443 )
6444 })
6445 .collect::<Vec<_>>();
6446 cases.extend([
6447 (
6448 "unknown_need".to_string(),
6449 "capabilities.requires[0].need".to_string(),
6450 "deferred".to_string(),
6451 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
6452 None,
6453 ),
6454 (
6455 "duplicate_provides".to_string(),
6456 "capabilities.provides[1]".to_string(),
6457 "credentials-provider/v1".to_string(),
6458 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
6459 None,
6460 ),
6461 (
6462 "duplicate_must_never_reach".to_string(),
6463 "capabilities.must_never_reach[1]".to_string(),
6464 "credentials-provider/v1".to_string(),
6465 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
6466 None,
6467 ),
6468 (
6469 "duplicate_requires_same_need".to_string(),
6470 "capabilities.requires[1]".to_string(),
6471 "credentials-provider/v1".to_string(),
6472 json!({ "requires": [
6473 { "capability": "credentials-provider/v1", "need": "required" },
6474 { "capability": "credentials-provider/v1", "need": "required" }
6475 ] }),
6476 None,
6477 ),
6478 (
6479 "duplicate_requires_conflicting_need".to_string(),
6480 "capabilities.requires[1]".to_string(),
6481 "credentials-provider/v1".to_string(),
6482 json!({ "requires": [
6483 { "capability": "credentials-provider/v1", "need": "required" },
6484 { "capability": "credentials-provider/v1", "need": "optional" }
6485 ] }),
6486 None,
6487 ),
6488 (
6489 "capabilities_root_pointer".to_string(),
6490 "runtime_computed[0]".to_string(),
6491 "/capabilities".to_string(),
6492 json!({}),
6493 Some(json!(["/capabilities"])),
6494 ),
6495 (
6496 "capabilities_descendant_pointer".to_string(),
6497 "runtime_computed[0]".to_string(),
6498 "/capabilities/provides".to_string(),
6499 json!({}),
6500 Some(json!(["/capabilities/provides"])),
6501 ),
6502 (
6503 "malformed_pointer_without_leading_slash".to_string(),
6504 "runtime_computed[0]".to_string(),
6505 "capabilities".to_string(),
6506 json!({}),
6507 Some(json!(["capabilities"])),
6508 ),
6509 (
6510 "malformed_pointer_escape".to_string(),
6511 "runtime_computed[0]".to_string(),
6512 "/roles/~2/tools".to_string(),
6513 json!({}),
6514 Some(json!(["/roles/~2/tools"])),
6515 ),
6516 (
6517 "unknown_capabilities_field".to_string(),
6518 "capabilities.future".to_string(),
6519 "<array>".to_string(),
6520 json!({ "future": [] }),
6521 None,
6522 ),
6523 ]);
6524
6525 for (index, (name, field, value, capabilities, runtime_computed)) in
6526 cases.into_iter().enumerate()
6527 {
6528 let registry = Arc::new(Registry::default());
6529 let handler = ControlHandler::new(Arc::clone(®istry));
6530 let response = handler
6531 .handle_control(
6532 ConnectionId::new((index + 1) as u64),
6533 capability_grammar_hello_frame(
6534 capabilities,
6535 runtime_computed,
6536 index as u64 + 1,
6537 ),
6538 )
6539 .expect("invalid HELLO returns a refusal");
6540
6541 assert_eq!(response.len(), 1, "{name} must emit one refusal");
6542 let error = parse_error(&response[0]);
6543 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
6544 let message = error["message"]
6545 .as_str()
6546 .expect("error message is a string");
6547 assert!(
6548 message.contains(&field),
6549 "{name}: field missing from {message}"
6550 );
6551 assert!(
6552 message.contains(&value),
6553 "{name}: value missing from {message}"
6554 );
6555 assert_eq!(
6556 registry
6557 .active_registration_count()
6558 .expect("registry reads"),
6559 0,
6560 "{name}: refused HELLO must not create a catalog entry"
6561 );
6562 }
6563 }
6564
6565 #[test]
6566 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
6567 let registry = Arc::new(Registry::default());
6568 let handler = ControlHandler::new(Arc::clone(®istry));
6569 let capabilities = json!({
6570 "provides": ["credentials-provider/v1"],
6571 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
6572 "must_never_reach": ["federation-transport/v1"]
6573 });
6574 let response = handler
6575 .handle_control(
6576 ConnectionId::new(99),
6577 capability_grammar_hello_frame(
6578 capabilities.clone(),
6579 Some(json!(["/roles/0/tools"])),
6580 99,
6581 ),
6582 )
6583 .expect("valid HELLO registers");
6584 assert_eq!(response[0].header.ty, FrameType::HelloAck);
6585
6586 let request = Frame::build(
6587 FrameType::Request,
6588 control_flags(),
6589 0,
6590 0,
6591 100,
6592 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6593 .expect("catalog request serializes"),
6594 )
6595 .expect("catalog request frame builds");
6596 let response = handler
6597 .handle_catalog_list(request, None)
6598 .expect("catalog list succeeds");
6599 let ClientControlResponse::CatalogList { modules, .. } =
6600 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
6601 else {
6602 panic!("catalog request must return catalog.list");
6603 };
6604 assert_eq!(modules.len(), 1);
6605 assert_eq!(
6606 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
6607 capabilities
6608 );
6609 }
6610
6611 #[test]
6612 fn catalog_list_mirrors_management_operation_description() {
6613 let registry = Arc::new(Registry::default());
6614 let handler = ControlHandler::new(Arc::clone(®istry));
6615 let description = "List managed records and return their identifiers and metadata.";
6616 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
6617 manifest.provides = vec![ProviderRole::ManagementSurface {
6618 operations: vec![ManagementOperation {
6619 name: "records.list".to_string(),
6620 kind: ManagementOperationKind::Query,
6621 description: Some(description.to_string()),
6622 }],
6623 config_schema: json!({"type": "object"}),
6624 observability: vec![ObservabilitySurface {
6625 name: "records.stats".to_string(),
6626 kind: ObservabilityKind::Snapshot,
6627 }],
6628 identity_scope: vec![IdentityScope::Project],
6629 concurrency: Concurrency::ModuleManaged,
6630 }];
6631 registry
6632 .register_with_control_ops(
6633 manifest,
6634 PROTOCOL_VERSION,
6635 ConnectionId::new(99),
6636 Vec::new(),
6637 )
6638 .expect("described management manifest registers");
6639
6640 let request = Frame::build(
6641 FrameType::Request,
6642 control_flags(),
6643 0,
6644 0,
6645 100,
6646 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6647 .expect("catalog request serializes"),
6648 )
6649 .expect("catalog request frame builds");
6650 let response = handler
6651 .handle_catalog_list(request, None)
6652 .expect("catalog list succeeds");
6653 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6654 assert_eq!(
6655 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
6656 "catalog.list must preserve the declared operation description verbatim"
6657 );
6658 }
6659
6660 #[test]
6661 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
6662 let registry = Arc::new(Registry::default());
6663 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
6664 [("vault".to_string(), true), ("squatter".to_string(), true)],
6665 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
6666 );
6667 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
6668 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6669 provides: vec!["credentials-provider/v1".to_string()],
6670 requires: Vec::new(),
6671 must_never_reach: Vec::new(),
6672 });
6673 let frame = Frame::build(
6674 FrameType::Hello,
6675 control_flags(),
6676 0,
6677 0,
6678 77,
6679 serde_json::to_vec(&ModuleHelloBody {
6680 manifest: squatter,
6681 protocol_ver: PROTOCOL_VERSION,
6682 control_ops: None,
6683 launch_nonce: None,
6684 })
6685 .expect("HELLO serializes"),
6686 )
6687 .expect("HELLO frame builds");
6688 let response = handler
6689 .handle_control(ConnectionId::new(77), frame)
6690 .expect("reserved claim receives a typed refusal");
6691 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
6692 assert_eq!(
6693 registry
6694 .active_registration_count()
6695 .expect("registry reads"),
6696 0,
6697 "a reserved capability refusal must not leave a catalog entry"
6698 );
6699 }
6700
6701 #[test]
6702 fn server_describe_surfaces_required_capability_verdict_fields() {
6703 let registry = Arc::new(Registry::default());
6704 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
6705 [
6706 ("consumer".to_string(), true),
6707 ("provider".to_string(), false),
6708 ],
6709 BTreeMap::new(),
6710 );
6711 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
6712 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
6713 provides: Vec::new(),
6714 requires: vec![subc_protocol::manifest::CapabilityRequirement {
6715 capability: "credentials-provider/v1".to_string(),
6716 need: subc_protocol::manifest::CapabilityNeed::Required,
6717 }],
6718 must_never_reach: Vec::new(),
6719 });
6720 let hello = Frame::build(
6721 FrameType::Hello,
6722 control_flags(),
6723 0,
6724 0,
6725 78,
6726 serde_json::to_vec(&ModuleHelloBody {
6727 manifest: consumer,
6728 protocol_ver: PROTOCOL_VERSION,
6729 control_ops: None,
6730 launch_nonce: None,
6731 })
6732 .expect("HELLO serializes"),
6733 )
6734 .expect("HELLO frame builds");
6735 handler
6736 .handle_control(ConnectionId::new(78), hello)
6737 .expect("consumer registers");
6738 let describe = Frame::build(
6739 FrameType::Request,
6740 control_flags(),
6741 0,
6742 0,
6743 79,
6744 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
6745 .expect("request serializes"),
6746 )
6747 .expect("describe frame builds");
6748 let response = handler
6749 .handle_server_describe(describe)
6750 .expect("server.describe succeeds");
6751 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
6752 let requirement = &rendered["capability_requirements"][0];
6753 assert_eq!(requirement["consumer"], "consumer");
6754 assert_eq!(requirement["verdict"], "never_provided");
6755 assert_eq!(requirement["episode_seq"], 1);
6756 assert_eq!(requirement["config_satisfiable"], false);
6757 assert_eq!(requirement["runtime_available"], false);
6758 assert!(requirement["detail"]
6759 .as_str()
6760 .expect("detail string")
6761 .contains("credentials-provider/v1"));
6762 }
6763
6764 #[test]
6765 fn catalog_list_omits_capabilities_for_legacy_manifest() {
6766 let registry = Arc::new(Registry::default());
6767 let handler = ControlHandler::new(Arc::clone(®istry));
6768 let hello = handler
6769 .handle_control(
6770 ConnectionId::new(101),
6771 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
6772 )
6773 .expect("legacy HELLO registers");
6774 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
6775
6776 let request = Frame::build(
6777 FrameType::Request,
6778 control_flags(),
6779 0,
6780 0,
6781 102,
6782 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
6783 .expect("catalog request serializes"),
6784 )
6785 .expect("catalog request frame builds");
6786 let response = handler
6787 .handle_catalog_list(request, None)
6788 .expect("catalog list succeeds");
6789 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
6790 assert!(
6791 body["modules"][0].get("capabilities").is_none(),
6792 "legacy manifest must retain an absent capabilities field on catalog.list"
6793 );
6794 }
6795
6796 #[test]
6797 fn hello_ack_omits_storage_when_no_storage_config() {
6798 let registry = Arc::new(Registry::default());
6799 let handler = ControlHandler::new(Arc::clone(®istry));
6800 let responses = handler
6801 .handle_control(
6802 ConnectionId::new(1),
6803 hello_frame("aft", PROTOCOL_VERSION, 7),
6804 )
6805 .unwrap();
6806 let ack = parse_ack(&responses[0]);
6807 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
6808 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
6809 }
6810
6811 #[tokio::test]
6812 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
6813 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
6814 let registry = Arc::new(Registry::default());
6815 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
6816 let responses = handler
6817 .handle_control(
6818 ConnectionId::new(1),
6819 hello_frame("aft", PROTOCOL_VERSION, 7),
6820 )
6821 .unwrap();
6822 let ack = parse_ack(&responses[0]);
6823 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
6824
6825 let described = handler
6826 .handle_control_frame(
6827 &route_ctx(ConnectionId::new(2)).0,
6828 Frame::build(
6829 FrameType::Request,
6830 control_flags(),
6831 0,
6832 0,
6833 9,
6834 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
6835 )
6836 .unwrap(),
6837 )
6838 .await
6839 .unwrap();
6840 let ClientControlResponse::ServerDescribe { machine_id, .. } =
6841 serde_json::from_slice(&described[0].body).unwrap()
6842 else {
6843 panic!("server.describe answered with another shape");
6844 };
6845 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
6846 }
6847
6848 #[test]
6849 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
6850 let registry = Arc::new(Registry::default());
6853 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
6854 crate::daemon_config::StorageConfig::Sqlite {
6855 data_home: std::path::PathBuf::from("/data"),
6856 },
6857 ));
6858
6859 let responses = handler
6860 .handle_control(
6861 ConnectionId::new(1),
6862 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
6863 )
6864 .unwrap();
6865 let ack = parse_ack(&responses[0]);
6866 assert_eq!(
6867 ack.storage,
6868 Some(serde_json::json!({
6869 "module_id": "alfonso-routing",
6870 "storage_namespace": "default",
6871 "isolation": { "kind": "module" },
6872 "backend": {
6873 "backend": "sqlite",
6874 "path": "/data/cortexkit/alfonso-routing/store.db"
6875 }
6876 })),
6877 "the delivered descriptor is the module's own sqlite store path"
6878 );
6879 }
6880
6881 #[test]
6882 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
6883 let registry = Arc::new(Registry::default());
6884 let handler = ControlHandler::new(Arc::clone(®istry));
6885 let conn = ConnectionId::new(1);
6886 let responses = handler
6887 .handle_control(
6888 conn,
6889 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6890 )
6891 .unwrap();
6892 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6893 let registration = registry.get_module("aft").unwrap().unwrap();
6894 assert_eq!(registration.control_ops, module_baseline_control_ops());
6895
6896 let frame =
6897 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
6898 assert!(handler
6899 .guard_module_control_op(&frame, "aft", "route.bind")
6900 .unwrap()
6901 .is_none());
6902 let error = handler
6903 .guard_module_control_op(&frame, "aft", "test.synthetic")
6904 .unwrap()
6905 .expect("synthetic ungranted op should be rejected");
6906 assert_eq!(error.header.ty, FrameType::Error);
6907 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
6908 }
6909
6910 #[test]
6911 fn hello_control_ops_some_adds_optional_grants() {
6912 let registry = Arc::new(Registry::default());
6913 let handler = ControlHandler::new(Arc::clone(®istry));
6914 handler
6915 .handle_control(
6916 ConnectionId::new(1),
6917 hello_frame_with_control_ops(
6918 "aft",
6919 PROTOCOL_VERSION,
6920 7,
6921 Some(vec![
6922 "future.synthetic".to_string(),
6923 "route.bind".to_string(),
6924 ]),
6925 ),
6926 )
6927 .unwrap();
6928 let registration = registry.get_module("aft").unwrap().unwrap();
6929 assert_eq!(
6930 registration.control_ops,
6931 vec![
6932 "route.bind".to_string(),
6933 "route.status".to_string(),
6934 "future.synthetic".to_string(),
6935 ]
6936 );
6937 let frame =
6938 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
6939 assert!(handler
6940 .guard_module_control_op(&frame, "aft", "future.synthetic")
6941 .unwrap()
6942 .is_none());
6943 }
6944
6945 #[tokio::test]
6946 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
6947 let registry = Arc::new(Registry::default());
6948 let forwarding = Arc::new(ForwardingTable::default());
6949 let handler =
6950 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
6951 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
6952 hello_via_sink(
6953 &handler,
6954 &module_ctx,
6955 &mut module_rx,
6956 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
6957 )
6958 .await;
6959
6960 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
6961 let responses = handler
6962 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
6963 .await
6964 .unwrap();
6965 assert_eq!(responses.len(), 1);
6966 assert_eq!(responses[0].header.ty, FrameType::Error);
6967 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
6968 assert!(module_rx.try_recv().is_err());
6969 }
6970
6971 #[tokio::test]
6972 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
6973 let registry = Arc::new(Registry::default());
6974 let forwarding = Arc::new(ForwardingTable::default());
6975 let handler =
6976 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
6977 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
6978 hello_via_sink(
6979 &handler,
6980 &module_ctx,
6981 &mut module_rx,
6982 hello_frame_with_control_ops(
6983 "aft",
6984 PROTOCOL_VERSION,
6985 7,
6986 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
6987 ),
6988 )
6989 .await;
6990
6991 let project_root = unique_project_root("demux");
6992 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
6993 let route_handler = handler.clone();
6994 let route_task = tokio::spawn(async move {
6995 route_handler
6996 .handle_control_frame(
6997 &route_client_ctx,
6998 route_open_frame(100, "aft", project_root),
6999 )
7000 .await
7001 .unwrap()
7002 });
7003 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7004 .await
7005 .unwrap()
7006 .unwrap();
7007 assert!(matches!(
7008 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
7009 ModuleControlRequest::RouteBind { .. }
7010 ));
7011
7012 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
7013 let health_handler = handler.clone();
7014 let health_task = tokio::spawn(async move {
7015 health_handler
7016 .handle_control_frame(
7017 &health_client_ctx,
7018 supervisor_health_probe_frame(101, "aft"),
7019 )
7020 .await
7021 .unwrap()
7022 });
7023 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7024 .await
7025 .unwrap()
7026 .unwrap();
7027 assert_eq!(
7028 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
7029 ModuleControlRequest::HealthCheck {}
7030 );
7031
7032 handler
7033 .handle_control_frame(
7034 &module_ctx,
7035 health_response(health_frame.header.corr, HealthStatus::Degraded),
7036 )
7037 .await
7038 .unwrap();
7039 let health_response = health_task.await.unwrap();
7040 assert_eq!(health_response.len(), 1);
7041 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
7042 ClientControlResponse::SupervisorHealthProbe {
7043 module_id,
7044 status,
7045 detail,
7046 metrics,
7047 } => {
7048 assert_eq!(module_id, "aft");
7049 assert_eq!(status, HealthStatus::Degraded);
7050 assert_eq!(detail.as_deref(), Some("warming"));
7051 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
7052 }
7053 other => panic!("unexpected health response: {other:?}"),
7054 }
7055
7056 handler
7057 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7058 .await
7059 .unwrap();
7060 let route_response = route_task.await.unwrap();
7061 assert!(route_response.is_empty());
7062 let published = route_client_rx.recv().await.unwrap();
7063 assert!(matches!(
7064 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7065 ClientControlResponse::RouteOpen { .. }
7066 ));
7067 }
7068
7069 async fn relay_route_open(
7074 handler: &ControlHandler,
7075 client_connection: ConnectionId,
7076 client_egress: &FrameSink,
7077 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7078 corr: u64,
7079 module_id: &str,
7080 project_root_label: &str,
7081 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7082 let ctx = RouteCtx {
7083 connection_id: client_connection,
7084 egress: client_egress.clone(),
7085 };
7086 let handler = handler.clone();
7087 let project_root = unique_project_root(project_root_label);
7088 let module_id = module_id.to_string();
7089 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7090 let task = tokio::spawn(async move {
7091 let _guard = tracing::dispatcher::set_default(&dispatch);
7092 handler
7093 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
7094 .await
7095 .unwrap()
7096 });
7097 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
7098 .await
7099 .expect("module receives the relayed route.bind")
7100 .expect("module egress is open");
7101 (task, bind.frame)
7102 }
7103
7104 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
7105 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7106 ModuleControlRequest::RouteBind {
7107 route_channel,
7108 epoch,
7109 ..
7110 } => (route_channel, epoch),
7111 other => panic!("expected a route.bind request, got {other:?}"),
7112 }
7113 }
7114
7115 fn published_route(frame: &Frame) -> (u16, u32) {
7116 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7117 ClientControlResponse::RouteOpen {
7118 route_channel,
7119 route_epoch,
7120 } => (route_channel, route_epoch),
7121 other => panic!("expected a route.open response, got {other:?}"),
7122 }
7123 }
7124
7125 #[tokio::test]
7145 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7146 let registry = Arc::new(Registry::default());
7147 let forwarding = Arc::new(ForwardingTable::default());
7148 let handler =
7149 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7150
7151 let module_connection = ConnectionId::new(30);
7152 let (module_ctx, mut module_rx) = route_ctx(module_connection);
7153 hello_via_sink(
7154 &handler,
7155 &module_ctx,
7156 &mut module_rx,
7157 hello_frame("aft", PROTOCOL_VERSION, 7),
7158 )
7159 .await;
7160
7161 let dying_client = ConnectionId::new(31);
7162 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7163
7164 let (first_task, first_bind) = relay_route_open(
7167 &handler,
7168 dying_client,
7169 &dying_ctx.egress,
7170 &mut module_rx,
7171 100,
7172 "aft",
7173 "closing-first",
7174 )
7175 .await;
7176 handler
7177 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7178 .await
7179 .unwrap();
7180 assert!(first_task.await.unwrap().is_empty());
7181 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7182
7183 let (second_task, second_bind) = relay_route_open(
7185 &handler,
7186 dying_client,
7187 &dying_ctx.egress,
7188 &mut module_rx,
7189 101,
7190 "aft",
7191 "closing-second",
7192 )
7193 .await;
7194 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7195
7196 assert!(forwarding
7199 .escalate_client_delivery_failure(
7200 dying_client,
7201 first_channel,
7202 first_epoch,
7203 CloseReason::new(
7204 "module_to_client_delivery_failed",
7205 "client egress refused a module frame",
7206 ),
7207 crate::forwarding::UndeliveredFrame {
7208 module_id: None,
7209 sink: &dying_ctx.egress,
7210 },
7211 )
7212 .unwrap());
7213 assert!(!dying_ctx.egress.is_closed());
7214
7215 let ack = handler
7217 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7218 .await;
7219 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7220 if module_loop_error.is_some() {
7221 handler.cleanup_connection(module_connection).unwrap();
7225 }
7226 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7230 .await
7231 .ok()
7232 .flatten();
7233
7234 assert!(
7236 registry
7237 .get_module_by_connection(module_connection)
7238 .unwrap()
7239 .is_some(),
7240 "one client's closing connection ended the shared module connection: \
7241 {module_loop_error:?}"
7242 );
7243 let cotenant = ConnectionId::new(32);
7245 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7246 let (cotenant_task, cotenant_bind) = relay_route_open(
7247 &handler,
7248 cotenant,
7249 &cotenant_ctx.egress,
7250 &mut module_rx,
7251 102,
7252 "aft",
7253 "closing-cotenant",
7254 )
7255 .await;
7256 handler
7257 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7258 .await
7259 .unwrap();
7260 assert!(cotenant_task.await.unwrap().is_empty());
7261 let (cotenant_channel, cotenant_epoch) =
7262 published_route(&cotenant_rx.recv().await.unwrap());
7263 assert!(matches!(
7264 forwarding
7265 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7266 .unwrap(),
7267 DataRoute::Client(DataRouteState::Bound(_))
7268 ));
7269
7270 let goodbye = post_ack_module_frame
7273 .expect("module receives a GOODBYE for the abandoned route channel");
7274 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7275 assert_eq!(goodbye.header.channel, abandoned_channel);
7276 assert_eq!(goodbye.header.epoch, abandoned_epoch);
7277
7278 assert!(dying_rx.try_recv().is_err());
7282 let second_response = second_task.await.unwrap();
7283 assert_eq!(second_response.len(), 1);
7284 assert_eq!(
7285 parse_error(&second_response[0])["code"],
7286 "target_unavailable"
7287 );
7288 }
7289
7290 #[test]
7297 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7298 let handler = ControlHandler::default();
7299 let module_connection = ConnectionId::new(30);
7300 let client_connection = ConnectionId::new(31);
7301
7302 handler
7303 .refuse_to_end_module_connection_for_a_client(
7304 module_connection,
7305 77,
7306 ForwardingError::ConnectionClosing {
7307 connection_id: client_connection,
7308 },
7309 )
7310 .expect("a closing client must never end the module connection");
7311
7312 assert!(matches!(
7313 handler.refuse_to_end_module_connection_for_a_client(
7314 module_connection,
7315 78,
7316 ForwardingError::ConnectionClosing {
7317 connection_id: module_connection,
7318 },
7319 ),
7320 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7321 connection_id
7322 })) if connection_id == module_connection
7323 ));
7324 assert!(matches!(
7325 handler.refuse_to_end_module_connection_for_a_client(
7326 module_connection,
7327 79,
7328 ForwardingError::Poisoned,
7329 ),
7330 Err(RouterError::Forwarding(ForwardingError::Poisoned))
7331 ));
7332 assert!(matches!(
7333 handler.refuse_to_end_module_connection_for_a_client(
7334 module_connection,
7335 80,
7336 ForwardingError::StaleModuleEndpoint,
7337 ),
7338 Err(RouterError::Forwarding(
7339 ForwardingError::StaleModuleEndpoint
7340 ))
7341 ));
7342 }
7343
7344 #[tokio::test]
7357 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7358 let registry = Arc::new(Registry::default());
7359 let forwarding = Arc::new(ForwardingTable::default());
7360 let supervisor = SupervisorHandle::new();
7361 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7362 let handler =
7363 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7364 .with_supervisor(supervisor);
7365
7366 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7367 hello_via_sink(
7368 &handler,
7369 &target_ctx,
7370 &mut target_rx,
7371 hello_frame("target", PROTOCOL_VERSION, 1),
7372 )
7373 .await;
7374
7375 let wrong_nonce = handler
7379 .handle_control_frame(
7380 &route_ctx(ConnectionId::new(91)).0,
7381 route_open_frame_with_admission_facts(
7382 20,
7383 "target",
7384 unique_project_root("admission-facts"),
7385 Some(subc_control::ConsumerIdentity {
7386 module_id: "fed".to_string(),
7387 launch_nonce: "not-the-real-nonce".to_string(),
7388 }),
7389 None,
7390 ),
7391 )
7392 .await
7393 .unwrap();
7394 assert_eq!(
7395 parse_error(&wrong_nonce[0])["code"],
7396 "bad_consumer_identity",
7397 "a mismatched launch nonce must be refused, not stamped Reserved"
7398 );
7399
7400 let never_spawned = handler
7404 .handle_control_frame(
7405 &route_ctx(ConnectionId::new(92)).0,
7406 route_open_frame_with_admission_facts(
7407 21,
7408 "target",
7409 unique_project_root("admission-facts"),
7410 Some(subc_control::ConsumerIdentity {
7411 module_id: "never-spawned".to_string(),
7412 launch_nonce: "any-nonce".to_string(),
7413 }),
7414 None,
7415 ),
7416 )
7417 .await
7418 .unwrap();
7419 assert_eq!(
7420 parse_error(&never_spawned[0])["code"],
7421 "bad_consumer_identity",
7422 "an unspawned module_id must be refused rather than accepted for lack of a record"
7423 );
7424 }
7425
7426 #[tokio::test]
7449 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
7450 let registry = Arc::new(Registry::default());
7451 let forwarding = Arc::new(ForwardingTable::default());
7452 let supervisor = SupervisorHandle::new();
7453 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7454 let handler =
7455 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7456 .with_supervisor(supervisor);
7457
7458 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
7459 hello_via_sink(
7460 &handler,
7461 &target_ctx,
7462 &mut target_rx,
7463 hello_frame("target", PROTOCOL_VERSION, 1),
7464 )
7465 .await;
7466
7467 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
7468 let route_handler = handler.clone();
7469 let route_task = tokio::spawn(async move {
7470 route_handler
7471 .handle_control_frame(
7472 &client_ctx,
7473 route_open_frame_with_admission_facts(
7474 30,
7475 "target",
7476 unique_project_root("admission-facts"),
7477 Some(subc_control::ConsumerIdentity {
7478 module_id: "fed".to_string(),
7479 launch_nonce: "fed-nonce".to_string(),
7480 }),
7481 None,
7482 ),
7483 )
7484 .await
7485 .unwrap()
7486 });
7487
7488 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7495 .await
7496 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
7497 .expect("module control channel closed before route.bind");
7498 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7499 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
7500 panic!("expected route.bind")
7501 };
7502 assert_eq!(
7503 principal,
7504 Some(Principal::Reserved {
7505 module_id: "fed".to_string()
7506 }),
7507 "a correctly attested consumer must be stamped Reserved for its own id"
7508 );
7509
7510 handler
7511 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
7512 .await
7513 .unwrap();
7514 assert!(route_task.await.unwrap().is_empty());
7515 assert!(
7516 matches!(
7517 serde_json::from_slice::<ClientControlResponse>(
7518 &client_rx.recv().await.unwrap().body
7519 )
7520 .unwrap(),
7521 ClientControlResponse::RouteOpen { .. }
7522 ),
7523 "the route must actually open, not merely avoid an error"
7524 );
7525 }
7526
7527 #[tokio::test(start_paused = true)]
7528 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
7529 let registry = Arc::new(Registry::default());
7530 let forwarding = Arc::new(ForwardingTable::default());
7531 let supervisor = SupervisorHandle::new();
7532 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7533 let handler =
7534 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7535 .with_supervisor(supervisor);
7536
7537 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
7538 hello_via_sink(
7539 &handler,
7540 &target_ctx,
7541 &mut target_rx,
7542 hello_frame("target", PROTOCOL_VERSION, 1),
7543 )
7544 .await;
7545
7546 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
7547 let direct_handler = handler.clone();
7548 let direct_open = tokio::spawn(async move {
7549 direct_handler
7550 .handle_control_frame(
7551 &direct_ctx,
7552 route_open_frame(2, "target", unique_project_root("route-census-direct")),
7553 )
7554 .await
7555 .unwrap()
7556 });
7557 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7558 .await
7559 .expect("no direct route.bind within 5s")
7560 .expect("target control channel closed before direct route.bind");
7561 handler
7562 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
7563 .await
7564 .unwrap();
7565 assert!(direct_open.await.unwrap().is_empty());
7566 let _ = direct_rx.recv().await.unwrap();
7567
7568 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
7569 let reserved_handler = handler.clone();
7570 let reserved_open = tokio::spawn(async move {
7571 reserved_handler
7572 .handle_control_frame(
7573 &reserved_ctx,
7574 route_open_frame_with_admission_facts(
7575 3,
7576 "target",
7577 unique_project_root("admission-facts"),
7578 Some(ConsumerIdentity {
7579 module_id: "fed".to_string(),
7580 launch_nonce: "fed-nonce".to_string(),
7581 }),
7582 None,
7583 ),
7584 )
7585 .await
7586 .unwrap()
7587 });
7588 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7589 .await
7590 .expect("no reserved route.bind within 5s")
7591 .expect("target control channel closed before reserved route.bind");
7592 handler
7593 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
7594 .await
7595 .unwrap();
7596 assert!(reserved_open.await.unwrap().is_empty());
7597 let _ = reserved_rx.recv().await.unwrap();
7598
7599 forwarding
7600 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
7601 .unwrap();
7602 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
7603 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
7604 module_id: Some("target".to_string()),
7605 })
7606 .unwrap();
7607 let census_frame =
7608 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
7609 let response = handler
7610 .handle_control_frame(&census_ctx, census_frame)
7611 .await
7612 .unwrap()
7613 .pop()
7614 .unwrap();
7615 let actual: Value = serde_json::from_slice(&response.body).unwrap();
7616 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
7617 assert!(matches!(
7618 decoded,
7619 ClientControlResponse::SupervisorRoutes { .. }
7620 ));
7621 let routes = actual["modules"][0]["routes"].as_array().unwrap();
7622 assert_eq!(routes.len(), 2);
7623 assert!(routes.iter().all(|route| route["draining"] == true));
7624 assert!(
7627 routes.iter().all(|route| route["drain_reason"] == "reload"),
7628 "draining routes must name the drain's reason: {routes:?}"
7629 );
7630 assert!(routes.iter().any(|route| {
7631 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
7632 }));
7633 assert!(routes.iter().any(|route| {
7634 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
7635 }));
7636
7637 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7638 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
7639 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7640 std::fs::write(
7641 &golden_path,
7642 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
7643 )
7644 .unwrap();
7645 }
7646 let expected: Value =
7647 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
7648 assert_eq!(actual, expected);
7649 }
7650
7651 async fn query_live_roots(
7652 handler: &ControlHandler,
7653 module_ctx: &RouteCtx,
7654 ) -> ModuleControlResponseToModule {
7655 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
7656 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
7657 let response = handler
7658 .handle_control_frame(module_ctx, frame)
7659 .await
7660 .unwrap()
7661 .pop()
7662 .unwrap();
7663 serde_json::from_slice(&response.body).unwrap()
7664 }
7665
7666 #[tokio::test(start_paused = true)]
7667 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
7668 let registry = Arc::new(Registry::default());
7669 let forwarding = Arc::new(ForwardingTable::default());
7670 let handler = ControlHandler::with_forwarding(registry, forwarding);
7671 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
7672 hello_via_sink(
7673 &handler,
7674 &target_ctx,
7675 &mut target_rx,
7676 hello_frame("target", PROTOCOL_VERSION, 1),
7677 )
7678 .await;
7679 let root = unique_project_root("live-roots-known");
7680 let path = ProjectRootId::from_path_allowing_missing(root.path())
7681 .unwrap()
7682 .as_path()
7683 .to_path_buf();
7684 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
7685 let open_handler = handler.clone();
7686 let opened = tokio::spawn(async move {
7687 open_handler
7688 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
7689 .await
7690 .unwrap()
7691 });
7692 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7693 .await
7694 .unwrap()
7695 .unwrap();
7696 handler
7697 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
7698 .await
7699 .unwrap();
7700 assert!(opened.await.unwrap().is_empty());
7701 let _ = client_rx.recv().await.unwrap();
7702
7703 let root = unique_project_root("live-roots-pending");
7704 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
7705 .unwrap()
7706 .as_path()
7707 .to_path_buf();
7708 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
7709 let open_handler = handler.clone();
7710 let pending = tokio::spawn(async move {
7711 open_handler
7712 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
7713 .await
7714 .unwrap()
7715 });
7716 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7717 .await
7718 .unwrap()
7719 .unwrap();
7720 let actual = query_live_roots(&handler, &target_ctx).await;
7721 let ModuleControlResponseToModule::LiveRoots {
7722 roots,
7723 unknown_root_bindings,
7724 total_bindings,
7725 } = actual
7726 else {
7727 panic!("expected live roots")
7728 };
7729 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
7730 assert_eq!(unknown_root_bindings, 0);
7731 assert_eq!(
7732 roots.len(),
7733 2,
7734 "root-known arm must retain each canonical root"
7735 );
7736 assert_eq!(
7737 total_bindings,
7738 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7739 );
7740 let counts = roots
7741 .iter()
7742 .map(|root| (root.project_root.clone(), root.bound, root.pending))
7743 .collect::<Vec<_>>();
7744 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
7745 expected.sort_by(|a, b| a.0.cmp(&b.0));
7746 assert_eq!(
7747 counts, expected,
7748 "roots must sort by path and count pending separately"
7749 );
7750 handler
7751 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
7752 .await
7753 .unwrap();
7754 assert!(pending.await.unwrap().is_empty());
7755 }
7756
7757 #[tokio::test(start_paused = true)]
7758 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
7759 let forwarding = Arc::new(ForwardingTable::default());
7760 let handler =
7761 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7762 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
7763 hello_via_sink(
7764 &handler,
7765 &target_ctx,
7766 &mut target_rx,
7767 hello_frame("target", PROTOCOL_VERSION, 1),
7768 )
7769 .await;
7770 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
7771 let pending = forwarding
7772 .begin_route_bind_relay_for_test(
7773 client_ctx.connection_id,
7774 client_ctx.egress.clone(),
7775 2,
7776 "target",
7777 )
7778 .unwrap();
7779 forwarding
7780 .complete_pending_relay(
7781 target_ctx.connection_id,
7782 pending.corr,
7783 RouteBindRelayOutcome::Accepted,
7784 )
7785 .unwrap();
7786 let actual = query_live_roots(&handler, &target_ctx).await;
7787 let ModuleControlResponseToModule::LiveRoots {
7788 roots,
7789 unknown_root_bindings,
7790 total_bindings,
7791 } = actual
7792 else {
7793 panic!("expected live roots")
7794 };
7795 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
7796 assert_eq!(
7797 unknown_root_bindings, 1,
7798 "unknown-root arm must not read as no bindings"
7799 );
7800 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
7801 assert_eq!(
7802 total_bindings,
7803 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7804 );
7805 }
7806
7807 #[tokio::test(start_paused = true)]
7815 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
7816 let forwarding = Arc::new(ForwardingTable::default());
7817 let handler =
7818 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
7819 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
7820 let replies = handler
7821 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
7822 .await
7823 .unwrap();
7824 let queued_by_hello = module_rx.len();
7825
7826 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
7827 let open_handler = handler.clone();
7828 let open = tokio::spawn(async move {
7829 open_handler
7830 .handle_control_frame(
7831 &client_ctx,
7832 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
7833 )
7834 .await
7835 .unwrap()
7836 });
7837 let mut spins = 0;
7839 while module_rx.len() == queued_by_hello {
7840 spins += 1;
7841 assert!(spins < 10_000, "route.open never queued a route.bind");
7842 tokio::task::yield_now().await;
7843 }
7844
7845 for reply in replies {
7847 module_ctx.egress.send(reply).await.unwrap();
7848 }
7849
7850 let first = module_rx.recv().await.unwrap().frame;
7851 assert_eq!(
7852 first.header.ty,
7853 FrameType::HelloAck,
7854 "the first frame a registering module reads must be its HELLO_ACK"
7855 );
7856 assert_eq!(first.header.corr, 7);
7857 let second = module_rx.recv().await.unwrap().frame;
7858 assert_eq!(second.header.ty, FrameType::Request);
7859 assert!(
7860 matches!(
7861 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
7862 ModuleControlRequest::RouteBind { .. }
7863 ),
7864 "the route.bind follows the ack"
7865 );
7866 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
7867
7868 handler
7869 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
7870 .await
7871 .unwrap();
7872 assert!(open.await.unwrap().is_empty());
7873 let _ = client_rx.recv().await.unwrap();
7874 }
7875
7876 #[tokio::test(start_paused = true)]
7877 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
7878 let handler = ControlHandler::with_forwarding(
7879 Arc::new(Registry::default()),
7880 Arc::new(ForwardingTable::default()),
7881 );
7882 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
7883 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
7884 hello_via_sink(
7885 &handler,
7886 &first_ctx,
7887 &mut first_rx,
7888 hello_frame("first", PROTOCOL_VERSION, 1),
7889 )
7890 .await;
7891 hello_via_sink(
7892 &handler,
7893 &second_ctx,
7894 &mut second_rx,
7895 hello_frame("second", PROTOCOL_VERSION, 2),
7896 )
7897 .await;
7898 let root = unique_project_root("second-only");
7899 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
7900 let cloned = handler.clone();
7901 let open = tokio::spawn(async move {
7902 cloned
7903 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
7904 .await
7905 .unwrap()
7906 });
7907 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
7908 .await
7909 .unwrap()
7910 .unwrap();
7911 let first = query_live_roots(&handler, &first_ctx).await;
7912 let second = query_live_roots(&handler, &second_ctx).await;
7913 assert!(
7914 matches!(
7915 first,
7916 ModuleControlResponseToModule::LiveRoots {
7917 total_bindings: 0,
7918 ..
7919 }
7920 ),
7921 "cross-module scope must not expose another module's roots"
7922 );
7923 assert!(
7924 matches!(
7925 second,
7926 ModuleControlResponseToModule::LiveRoots {
7927 total_bindings: 1,
7928 ..
7929 }
7930 ),
7931 "second module must see its pending route"
7932 );
7933 handler
7934 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
7935 .await
7936 .unwrap();
7937 assert!(open.await.unwrap().is_empty());
7938 }
7939
7940 #[tokio::test(start_paused = true)]
7941 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
7942 let handler = ControlHandler::with_forwarding(
7943 Arc::new(Registry::default()),
7944 Arc::new(ForwardingTable::default()),
7945 );
7946 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
7947 hello_via_sink(
7948 &handler,
7949 &target_ctx,
7950 &mut target_rx,
7951 hello_frame("target", PROTOCOL_VERSION, 1),
7952 )
7953 .await;
7954 let actual = query_live_roots(&handler, &target_ctx).await;
7955 let ModuleControlResponseToModule::LiveRoots {
7956 roots,
7957 unknown_root_bindings,
7958 total_bindings,
7959 } = actual
7960 else {
7961 panic!("expected live roots")
7962 };
7963 assert!(roots.is_empty());
7964 assert_eq!(unknown_root_bindings, 0);
7965 assert_eq!(total_bindings, 0);
7966 assert_eq!(
7967 total_bindings,
7968 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
7969 );
7970 }
7971
7972 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
7979 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7980 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
7981 let text = std::fs::read_to_string(&path)
7982 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
7983 let vectors: Vec<(String, Value)> = text
7984 .lines()
7985 .filter(|line| !line.trim().is_empty())
7986 .map(|line| {
7987 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
7988 let id = entry["corpus_id"]
7989 .as_str()
7990 .expect("every vector carries a corpus_id")
7991 .to_string();
7992 (id, entry["package"].clone())
7993 })
7994 .collect();
7995 assert_eq!(
7999 vectors.len(),
8000 3,
8001 "vendored fed corpus changed size; re-sync from subc-federation"
8002 );
8003
8004 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
8019 let richest = vectors
8020 .iter()
8021 .filter_map(|(_, package)| package.as_object())
8022 .map(|object| {
8023 object
8024 .keys()
8025 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
8026 .count()
8027 })
8028 .max()
8029 .unwrap_or(0);
8030 assert!(
8031 richest >= 2,
8032 "vendored corpus no longer carries a package with unmodelled fields, \
8033 so the relay test can no longer distinguish a verbatim relay from a lossy one"
8034 );
8035
8036 vectors
8037 }
8038
8039 #[tokio::test]
8047 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
8048 for (corpus_id, package) in fed_admission_facts_vectors() {
8049 let registry = Arc::new(Registry::default());
8050 let forwarding = Arc::new(ForwardingTable::default());
8051 let supervisor = SupervisorHandle::new();
8052 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8053 let handler =
8054 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8055 .with_supervisor(supervisor)
8056 .with_admission_facts_config(
8057 Some("fed".to_string()),
8058 Some(vec!["target".to_string()]),
8059 );
8060
8061 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8062 hello_via_sink(
8063 &handler,
8064 &target_ctx,
8065 &mut target_rx,
8066 hello_frame("target", PROTOCOL_VERSION, 1),
8067 )
8068 .await;
8069
8070 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8071 let route_handler = handler.clone();
8072 let expected = package.clone();
8073 let route_task = tokio::spawn(async move {
8074 route_handler
8075 .handle_control_frame(
8076 &client_ctx,
8077 route_open_frame_with_admission_facts(
8078 20,
8079 "target",
8080 unique_project_root("admission-facts"),
8081 Some(subc_control::ConsumerIdentity {
8082 module_id: "fed".to_string(),
8083 launch_nonce: "fed-nonce".to_string(),
8084 }),
8085 Some(package),
8086 ),
8087 )
8088 .await
8089 .unwrap()
8090 });
8091
8092 let bind_frame = target_rx.recv().await.unwrap();
8093 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8094 let ModuleControlRequest::RouteBind {
8095 admission_facts, ..
8096 } = bind
8097 else {
8098 panic!("{corpus_id}: expected route.bind")
8099 };
8100 assert_eq!(
8101 admission_facts,
8102 Some(expected),
8103 "{corpus_id}: relay must not add, drop or reshape any field"
8104 );
8105
8106 handler
8107 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8108 .await
8109 .unwrap();
8110 route_task.await.unwrap();
8111 }
8112 }
8113
8114 #[tokio::test]
8115 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8116 let registry = Arc::new(Registry::default());
8117 let forwarding = Arc::new(ForwardingTable::default());
8118 let supervisor = SupervisorHandle::new();
8119 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8120 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8121 let handler =
8122 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8123 .with_supervisor(supervisor)
8124 .with_admission_facts_config(
8125 Some("fed".to_string()),
8126 Some(vec!["target".to_string()]),
8127 );
8128
8129 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8130 hello_via_sink(
8131 &handler,
8132 &target_ctx,
8133 &mut target_rx,
8134 hello_frame("target", PROTOCOL_VERSION, 1),
8135 )
8136 .await;
8137 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8138 hello_via_sink(
8139 &handler,
8140 &other_ctx,
8141 &mut other_rx,
8142 hello_frame("other", PROTOCOL_VERSION, 2),
8143 )
8144 .await;
8145
8146 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8147 let expected_facts = facts.clone();
8148 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8149 let route_handler = handler.clone();
8150 let route_task = tokio::spawn(async move {
8151 route_handler
8152 .handle_control_frame(
8153 &client_ctx,
8154 route_open_frame_with_admission_facts(
8155 10,
8156 "target",
8157 unique_project_root("admission-facts"),
8158 Some(subc_control::ConsumerIdentity {
8159 module_id: "fed".to_string(),
8160 launch_nonce: "fed-nonce".to_string(),
8161 }),
8162 Some(facts.clone()),
8163 ),
8164 )
8165 .await
8166 .unwrap()
8167 });
8168 let bind_frame = target_rx.recv().await.unwrap();
8169 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8170 let ModuleControlRequest::RouteBind {
8171 admission_facts, ..
8172 } = bind
8173 else {
8174 panic!("expected route.bind")
8175 };
8176 assert_eq!(admission_facts, Some(expected_facts));
8177 handler
8178 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8179 .await
8180 .unwrap();
8181 assert!(route_task.await.unwrap().is_empty());
8182 assert!(matches!(
8183 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8184 .unwrap(),
8185 ClientControlResponse::RouteOpen { .. }
8186 ));
8187
8188 let direct = handler
8189 .handle_control_frame(
8190 &route_ctx(ConnectionId::new(73)).0,
8191 route_open_frame_with_admission_facts(
8192 11,
8193 "target",
8194 unique_project_root("admission-facts"),
8195 None,
8196 Some(json!({"x": 1})),
8197 ),
8198 )
8199 .await
8200 .unwrap();
8201 assert_eq!(
8202 parse_error(&direct[0])["code"],
8203 "admission_facts_not_permitted"
8204 );
8205
8206 let different_reserved = handler
8207 .handle_control_frame(
8208 &route_ctx(ConnectionId::new(77)).0,
8209 route_open_frame_with_admission_facts(
8210 15,
8211 "target",
8212 unique_project_root("admission-facts"),
8213 Some(subc_control::ConsumerIdentity {
8214 module_id: "other".to_string(),
8215 launch_nonce: "other-nonce".to_string(),
8216 }),
8217 Some(json!({"x": 1})),
8218 ),
8219 )
8220 .await
8221 .unwrap();
8222 assert_eq!(
8223 parse_error(&different_reserved[0])["code"],
8224 "admission_facts_not_permitted"
8225 );
8226
8227 let other_target = handler
8228 .handle_control_frame(
8229 &route_ctx(ConnectionId::new(74)).0,
8230 route_open_frame_with_admission_facts(
8231 12,
8232 "other",
8233 unique_project_root("admission-facts"),
8234 Some(subc_control::ConsumerIdentity {
8235 module_id: "fed".to_string(),
8236 launch_nonce: "fed-nonce".to_string(),
8237 }),
8238 Some(json!({"x": 1})),
8239 ),
8240 )
8241 .await
8242 .unwrap();
8243 assert_eq!(
8244 parse_error(&other_target[0])["code"],
8245 "admission_facts_target_not_allowed"
8246 );
8247
8248 let nonexistent = handler
8249 .handle_control_frame(
8250 &route_ctx(ConnectionId::new(75)).0,
8251 route_open_frame_with_admission_facts(
8252 13,
8253 "missing",
8254 unique_project_root("admission-facts"),
8255 None,
8256 Some(json!({"x": 1})),
8257 ),
8258 )
8259 .await
8260 .unwrap();
8261 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8262
8263 let described = handler
8264 .handle_control_frame(
8265 &route_ctx(ConnectionId::new(76)).0,
8266 Frame::build(
8267 FrameType::Request,
8268 control_flags(),
8269 0,
8270 0,
8271 14,
8272 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8273 )
8274 .unwrap(),
8275 )
8276 .await
8277 .unwrap();
8278 let ClientControlResponse::ServerDescribe { capabilities, .. } =
8279 serde_json::from_slice(&described[0].body).unwrap()
8280 else {
8281 panic!("expected server.describe response")
8282 };
8283 assert!(capabilities
8284 .iter()
8285 .any(|cap| cap == "admission_facts_relay_v1"));
8286 }
8287
8288 #[tokio::test]
8289 async fn admission_facts_without_configured_carrier_are_rejected() {
8290 let registry = Arc::new(Registry::default());
8291 let forwarding = Arc::new(ForwardingTable::default());
8292 let handler = ControlHandler::with_forwarding(registry, forwarding);
8293 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8294 hello_via_sink(
8295 &handler,
8296 &target_ctx,
8297 &mut target_rx,
8298 hello_frame("target", PROTOCOL_VERSION, 1),
8299 )
8300 .await;
8301
8302 let responses = handler
8303 .handle_control_frame(
8304 &route_ctx(ConnectionId::new(79)).0,
8305 route_open_frame_with_admission_facts(
8306 16,
8307 "target",
8308 unique_project_root("admission-facts"),
8309 None,
8310 Some(json!({"x": 1})),
8311 ),
8312 )
8313 .await
8314 .unwrap();
8315 assert_eq!(
8316 parse_error(&responses[0])["code"],
8317 "admission_facts_not_permitted"
8318 );
8319 }
8320
8321 #[tokio::test]
8322 async fn route_open_relays_consumer_capabilities_verbatim() {
8323 let registry = Arc::new(Registry::default());
8324 let forwarding = Arc::new(ForwardingTable::default());
8325 let handler =
8326 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8327 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8328 hello_via_sink(
8329 &handler,
8330 &module_ctx,
8331 &mut module_rx,
8332 hello_frame("aft", PROTOCOL_VERSION, 7),
8333 )
8334 .await;
8335
8336 let expected = vec!["elicitation".to_string(), "roots".to_string()];
8337 let expected_for_request = expected.clone();
8338 let project_root = unique_project_root("consumer-capabilities-present");
8339 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8340 let route_handler = handler.clone();
8341 let route_task = tokio::spawn(async move {
8342 route_handler
8343 .handle_control_frame(
8344 &client_ctx,
8345 route_open_frame_with_consumer_capabilities(
8346 401,
8347 "aft",
8348 project_root,
8349 Some(expected_for_request),
8350 ),
8351 )
8352 .await
8353 .unwrap()
8354 });
8355 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8356 .await
8357 .unwrap()
8358 .unwrap();
8359 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8360 let ModuleControlRequest::RouteBind {
8361 consumer_capabilities,
8362 ..
8363 } = bind
8364 else {
8365 panic!("expected route.bind request, got {bind:?}");
8366 };
8367 assert_eq!(consumer_capabilities, Some(expected.clone()));
8368
8369 handler
8370 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8371 .await
8372 .unwrap();
8373 let route_response = route_task.await.unwrap();
8374 assert!(route_response.is_empty());
8375 let published = client_rx.recv().await.unwrap();
8376 assert!(matches!(
8377 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8378 ClientControlResponse::RouteOpen { .. }
8379 ));
8380 }
8381
8382 #[tokio::test]
8383 async fn route_open_without_consumer_capabilities_relays_none() {
8384 let registry = Arc::new(Registry::default());
8385 let forwarding = Arc::new(ForwardingTable::default());
8386 let handler =
8387 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8388 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8389 hello_via_sink(
8390 &handler,
8391 &module_ctx,
8392 &mut module_rx,
8393 hello_frame("aft", PROTOCOL_VERSION, 7),
8394 )
8395 .await;
8396
8397 let project_root = unique_project_root("consumer-capabilities-absent");
8398 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8399 let route_handler = handler.clone();
8400 let route_task = tokio::spawn(async move {
8401 route_handler
8402 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
8403 .await
8404 .unwrap()
8405 });
8406 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8407 .await
8408 .unwrap()
8409 .unwrap();
8410 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8411 let ModuleControlRequest::RouteBind {
8412 consumer_capabilities,
8413 ..
8414 } = bind
8415 else {
8416 panic!("expected route.bind request, got {bind:?}");
8417 };
8418 assert_eq!(consumer_capabilities, None);
8419
8420 handler
8421 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8422 .await
8423 .unwrap();
8424 let route_response = route_task.await.unwrap();
8425 assert!(route_response.is_empty());
8426 let published = client_rx.recv().await.unwrap();
8427 assert!(matches!(
8428 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8429 ClientControlResponse::RouteOpen { .. }
8430 ));
8431 }
8432
8433 #[tokio::test]
8434 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
8435 let registry = Arc::new(Registry::default());
8436 let forwarding = Arc::new(ForwardingTable::default());
8437 let handler =
8438 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8439 .with_health_probe_timeout(Duration::from_secs(5));
8440 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
8441 hello_via_sink(
8442 &handler,
8443 &module_ctx,
8444 &mut module_rx,
8445 non_routable_hello_frame_with_control_ops(
8446 "mcp",
8447 300,
8448 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8449 ),
8450 )
8451 .await;
8452 assert!(registry
8453 .get_module("mcp")
8454 .unwrap()
8455 .unwrap()
8456 .manifest
8457 .provides
8458 .is_empty());
8459
8460 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
8461 let route_response = handler
8462 .handle_control_frame(
8463 &route_client_ctx,
8464 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
8465 )
8466 .await
8467 .unwrap();
8468 assert_eq!(route_response[0].header.ty, FrameType::Error);
8469 assert_eq!(
8470 parse_error(&route_response[0])["code"],
8471 "target_unavailable"
8472 );
8473 assert!(parse_error(&route_response[0])["message"]
8474 .as_str()
8475 .unwrap()
8476 .contains("does not provide the requested target"));
8477 assert!(module_rx.try_recv().is_err());
8478
8479 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
8480 let health_handler = handler.clone();
8481 let health_task = tokio::spawn(async move {
8482 health_handler
8483 .handle_control_frame(
8484 &health_client_ctx,
8485 supervisor_health_probe_frame(302, "mcp"),
8486 )
8487 .await
8488 .unwrap()
8489 });
8490 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8491 .await
8492 .unwrap()
8493 .unwrap();
8494 assert_eq!(
8495 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8496 ModuleControlRequest::HealthCheck {}
8497 );
8498 handler
8499 .handle_control_frame(
8500 &module_ctx,
8501 health_response(health_frame.header.corr, HealthStatus::Ok),
8502 )
8503 .await
8504 .unwrap();
8505 let health_response = health_task.await.unwrap();
8506 assert_eq!(health_response[0].header.ty, FrameType::Response);
8507 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8508 ClientControlResponse::SupervisorHealthProbe {
8509 module_id, status, ..
8510 } => {
8511 assert_eq!(module_id, "mcp");
8512 assert_eq!(status, HealthStatus::Ok);
8513 }
8514 other => panic!("unexpected health response: {other:?}"),
8515 }
8516
8517 forwarding
8522 .cleanup_connection(module_ctx.connection_id)
8523 .unwrap();
8524 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
8525 let cleanup_response = tokio::time::timeout(
8526 Duration::from_millis(200),
8527 handler.handle_control_frame(
8528 &cleanup_probe_ctx,
8529 supervisor_health_probe_frame(303, "mcp"),
8530 ),
8531 )
8532 .await
8533 .expect("probe should fail immediately when the control lane is gone")
8534 .unwrap();
8535 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
8536 assert_eq!(
8537 parse_error(&cleanup_response[0])["code"],
8538 "target_unavailable"
8539 );
8540 assert!(parse_error(&cleanup_response[0])["message"]
8541 .as_str()
8542 .unwrap()
8543 .contains("no module connection"));
8544
8545 handler
8546 .cleanup_connection(module_ctx.connection_id)
8547 .unwrap();
8548 }
8549
8550 #[tokio::test]
8551 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
8552 let registry = Arc::new(Registry::default());
8553 let supervisor_handle = SupervisorHandle::new();
8554 let supervisor =
8555 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8556 .with_handle(supervisor_handle.clone())
8557 .with_connection_file_path(
8558 std::env::temp_dir()
8559 .join(format!("subc-route-open-warming-{}", std::process::id())),
8560 );
8561 let module = supervisor
8562 .supervise_configured(
8563 ModuleSpec {
8564 module_id: "warming".to_string(),
8565 program: fake_aft_stub_path(),
8566 args: Vec::new(),
8567 env: Vec::new(),
8568 reserved: false,
8569 reserved_prefixes: Vec::new(),
8570 protocol: ModuleProtocol::Subc,
8571 overlap: Default::default(),
8572 },
8573 true,
8574 )
8575 .unwrap();
8576 assert_eq!(module.state().unwrap(), ModuleState::Running);
8577
8578 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8579 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
8580 let response = handler
8581 .handle_control_frame(
8582 &ctx,
8583 route_open_frame(304, "warming", unique_project_root("warming")),
8584 )
8585 .await
8586 .unwrap();
8587 module.stop().await.unwrap();
8588
8589 assert_eq!(response[0].header.ty, FrameType::Error);
8590 let error = parse_error(&response[0]);
8591 assert_eq!(error["code"], "module_warming");
8592 assert!(error["message"]
8593 .as_str()
8594 .unwrap()
8595 .contains("state=running, enabled=true, live=false"));
8596 }
8597
8598 #[test]
8599 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
8600 let handler = ControlHandler::new(Arc::new(Registry::default()));
8601 let capture = EventCapture::default();
8602 let _subscriber =
8603 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8604 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
8605 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
8606 let pending = (0..limit).collect::<Vec<_>>();
8607 let response = handler
8608 .route_open_capacity_refusal(
8609 &ctx,
8610 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
8611 "busy",
8612 pending.len(),
8613 limit,
8614 )
8615 .unwrap();
8616 assert_eq!(parse_error(&response)["code"], "target_unavailable");
8617 let event = capture
8618 .events()
8619 .into_iter()
8620 .find(|event| {
8621 event.target == "control"
8622 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
8623 })
8624 .expect("connection admission refusal event");
8625 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
8626 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
8627 }
8628
8629 #[test]
8630 fn route_open_target_cap_logs_admission_reason_and_capacity() {
8631 let handler = ControlHandler::new(Arc::new(Registry::default()));
8632 let capture = EventCapture::default();
8633 let _subscriber =
8634 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8635 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
8636 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
8637 let guards = (0..limit)
8638 .map(|_| {
8639 handler
8640 .route_bind_concurrency
8641 .try_admit("busy", limit)
8642 .unwrap()
8643 })
8644 .collect::<Vec<_>>();
8645 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
8646 Err(in_flight) => in_flight,
8647 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
8648 };
8649 let response = handler
8650 .route_open_target_capacity_refusal(
8651 &ctx,
8652 &route_open_frame(397, "busy", unique_project_root("target-cap")),
8653 "busy",
8654 in_flight,
8655 )
8656 .unwrap();
8657 assert_eq!(parse_error(&response)["code"], "target_unavailable");
8658 let event = capture
8659 .events()
8660 .into_iter()
8661 .find(|event| {
8662 event.target == "control"
8663 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
8664 })
8665 .expect("target admission refusal event");
8666 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
8667 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
8668 drop(guards);
8669 }
8670
8671 #[tokio::test]
8676 async fn route_open_refusal_names_the_check_that_refused() {
8677 let handler = ControlHandler::new(Arc::new(Registry::default()));
8678 let capture = EventCapture::default();
8679 let _subscriber =
8680 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8681 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8682 let response = handler
8683 .handle_control_frame(
8684 &ctx,
8685 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
8686 )
8687 .await
8688 .unwrap();
8689
8690 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8691 let event = capture
8692 .events()
8693 .into_iter()
8694 .find(|event| {
8695 event.target == "control"
8696 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8697 })
8698 .expect("route.open refusal event");
8699 assert_eq!(
8700 event.fields.get("reason"),
8701 Some(&"\"not_registered\"".to_string())
8702 );
8703 }
8704
8705 #[tokio::test]
8706 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
8707 let registry = Arc::new(Registry::default());
8708 let supervisor_handle = SupervisorHandle::new();
8709 let supervisor =
8710 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8711 .with_handle(supervisor_handle.clone())
8712 .with_connection_file_path(std::env::temp_dir().join(format!(
8713 "subc-route-open-refusal-info-{}",
8714 std::process::id()
8715 )));
8716 let module = supervisor
8717 .supervise_configured(
8718 ModuleSpec {
8719 module_id: "warming".to_string(),
8720 program: fake_aft_stub_path(),
8721 args: Vec::new(),
8722 env: Vec::new(),
8723 reserved: false,
8724 reserved_prefixes: Vec::new(),
8725 protocol: ModuleProtocol::Subc,
8726 overlap: Default::default(),
8727 },
8728 true,
8729 )
8730 .unwrap();
8731 assert_eq!(module.state().unwrap(), ModuleState::Running);
8732
8733 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8734 assert!(handler
8735 .counters()
8736 .snapshot()
8737 .get("route_open_refused_by_code")
8738 .is_none());
8739 let capture = EventCapture::default();
8740 let _subscriber =
8741 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8742 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
8743 let response = handler
8744 .handle_control_frame(
8745 &ctx,
8746 route_open_frame(394, "warming", unique_project_root("refusal-info")),
8747 )
8748 .await
8749 .unwrap();
8750 module.stop().await.unwrap();
8751
8752 assert_eq!(parse_error(&response[0])["code"], "module_warming");
8753 let event = capture
8754 .events()
8755 .into_iter()
8756 .find(|event| {
8757 event.target == "control"
8758 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
8759 })
8760 .expect("route.open refusal event");
8761 assert_eq!(
8762 event.fields.get("module_id"),
8763 Some(&"\"warming\"".to_string())
8764 );
8765 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
8766 assert_eq!(
8767 event.fields.get("reason"),
8768 Some(&"\"supervised_not_registered\"".to_string())
8769 );
8770 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
8771 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
8772 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
8773 assert_eq!(
8774 handler.counters().snapshot()["route_open_refused_by_code"],
8775 json!({ "module_warming": 1 })
8776 );
8777 }
8778
8779 #[tokio::test(flavor = "current_thread")]
8780 async fn route_open_unknown_module_escapes_target_module_id() {
8781 let handler = ControlHandler::new(Arc::new(Registry::default()));
8782 let capture = EventCapture::default();
8783 let _subscriber =
8784 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8785 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
8786 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
8787 let response = handler
8788 .handle_control_frame(
8789 &ctx,
8790 route_open_frame(
8791 395,
8792 hostile_module_id,
8793 unique_project_root("hostile-target-module-id"),
8794 ),
8795 )
8796 .await
8797 .unwrap();
8798
8799 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
8800 let event = capture
8801 .events()
8802 .into_iter()
8803 .find(|event| {
8804 event.target == "control"
8805 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
8806 })
8807 .expect("route.open unknown-module refusal event");
8808 let logged = event.fields.get("module_id").expect("module_id field");
8809 assert!(!logged.bytes().any(|byte| byte < 0x20));
8810 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8811 }
8812
8813 #[tokio::test(flavor = "current_thread")]
8814 async fn route_open_module_rejection_uses_daemon_counter_key() {
8815 let registry = Arc::new(Registry::default());
8816 let forwarding = Arc::new(ForwardingTable::default());
8817 let handler =
8818 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8819 let module_connection = ConnectionId::new(95);
8820 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8821 hello_via_sink(
8822 &handler,
8823 &module_ctx,
8824 &mut module_rx,
8825 hello_frame("aft", PROTOCOL_VERSION, 395),
8826 )
8827 .await;
8828
8829 let client_connection = ConnectionId::new(96);
8830 let (client_ctx, _client_rx) = route_ctx(client_connection);
8831 let capture = EventCapture::default();
8832 let _subscriber =
8833 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
8834 let (route_task, bind) = relay_route_open(
8835 &handler,
8836 client_connection,
8837 &client_ctx.egress,
8838 &mut module_rx,
8839 396,
8840 "aft",
8841 "hostile-module-code",
8842 )
8843 .await;
8844 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
8845 let rejection = Frame::build(
8846 FrameType::Error,
8847 control_flags(),
8848 0,
8849 0,
8850 bind.header.corr,
8851 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
8852 )
8853 .unwrap();
8854 handler
8855 .handle_control_frame(&module_ctx, rejection)
8856 .await
8857 .unwrap();
8858
8859 let response = route_task.await.unwrap();
8860 assert_eq!(parse_error(&response[0])["code"], hostile_code);
8861 let counters = handler.counters().snapshot();
8862 assert_eq!(
8863 counters["route_open_refused_by_code"],
8864 json!({ "module_rejected": 1 })
8865 );
8866 assert!(counters["route_open_refused_by_code"]
8867 .get(hostile_code)
8868 .is_none());
8869
8870 let event = capture
8871 .events()
8872 .into_iter()
8873 .find(|event| {
8874 event.target == "control"
8875 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
8876 })
8877 .expect("route.open module-rejection refusal event");
8878 let logged = event.fields.get("module_code").expect("module_code field");
8879 assert!(!logged.bytes().any(|byte| byte < 0x20));
8880 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
8881 }
8882
8883 #[tokio::test]
8884 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
8885 let registry = Arc::new(Registry::default());
8886 let supervisor_handle = SupervisorHandle::new();
8887 let missing_program = std::env::temp_dir().join(format!(
8888 "subc-route-open-missing-program-{}",
8889 std::process::id()
8890 ));
8891 let supervisor =
8892 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8893 .with_handle(supervisor_handle.clone());
8894 let module = supervisor
8895 .supervise_configured(
8896 ModuleSpec {
8897 module_id: "failed".to_string(),
8898 program: missing_program,
8899 args: Vec::new(),
8900 env: Vec::new(),
8901 reserved: false,
8902 reserved_prefixes: Vec::new(),
8903 protocol: ModuleProtocol::Subc,
8904 overlap: Default::default(),
8905 },
8906 true,
8907 )
8908 .unwrap();
8909 assert_eq!(module.state().unwrap(), ModuleState::Failed);
8910
8911 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8912 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
8913 let response = handler
8914 .handle_control_frame(
8915 &ctx,
8916 route_open_frame(305, "failed", unique_project_root("failed")),
8917 )
8918 .await
8919 .unwrap();
8920
8921 assert_eq!(response[0].header.ty, FrameType::Error);
8922 let error = parse_error(&response[0]);
8923 assert_eq!(error["code"], "target_unavailable");
8924 assert!(error["message"]
8925 .as_str()
8926 .unwrap()
8927 .contains("state=failed, enabled=true, live=false"));
8928 }
8929
8930 #[tokio::test]
8931 async fn route_open_role_mismatch_remains_target_unavailable() {
8932 let registry = Arc::new(Registry::default());
8933 let handler = ControlHandler::new(Arc::clone(®istry));
8934 handler
8935 .handle_control(
8936 ConnectionId::new(41),
8937 non_routable_hello_frame_with_control_ops("health-only", 306, None),
8938 )
8939 .unwrap();
8940
8941 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
8942 let response = handler
8943 .handle_control_frame(
8944 &ctx,
8945 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
8946 )
8947 .await
8948 .unwrap();
8949
8950 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8951 assert!(parse_error(&response[0])["message"]
8952 .as_str()
8953 .unwrap()
8954 .contains("does not provide the requested target"));
8955 }
8956
8957 #[tokio::test]
8958 async fn route_open_inactive_registration_remains_target_unavailable() {
8959 let registry = Arc::new(Registry::default());
8960 let handler = ControlHandler::new(Arc::clone(®istry));
8961 handler
8962 .handle_control(
8963 ConnectionId::new(43),
8964 hello_frame("inactive", PROTOCOL_VERSION, 308),
8965 )
8966 .unwrap();
8967 assert!(registry
8968 .set_module_state_for_test("inactive", ChannelState::Closed)
8969 .unwrap());
8970
8971 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
8972 let response = handler
8973 .handle_control_frame(
8974 &ctx,
8975 route_open_frame(309, "inactive", unique_project_root("inactive")),
8976 )
8977 .await
8978 .unwrap();
8979
8980 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8981 assert!(parse_error(&response[0])["message"]
8982 .as_str()
8983 .unwrap()
8984 .contains("is not active"));
8985 }
8986
8987 #[tokio::test]
8988 async fn late_health_reply_is_recorded_through_the_module_response_path() {
8989 let registry = Arc::new(Registry::default());
8990 let forwarding = Arc::new(ForwardingTable::default());
8991 let supervisor_handle = SupervisorHandle::new();
8992 let supervisor = Supervisor::new(Arc::clone(®istry), crate::RestartPolicy::default())
8993 .with_forwarding(Arc::clone(&forwarding))
8994 .with_handle(supervisor_handle.clone());
8995 let module = supervisor
8996 .supervise_configured(
8997 crate::ModuleSpec {
8998 module_id: "late-health-response".to_string(),
8999 program: PathBuf::from("disabled-module"),
9000 args: Vec::new(),
9001 env: Vec::new(),
9002 reserved: false,
9003 reserved_prefixes: Vec::new(),
9004 protocol: ModuleProtocol::Subc,
9005 overlap: Default::default(),
9006 },
9007 false,
9008 )
9009 .unwrap();
9010 let handler =
9011 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9012 .with_supervisor(supervisor_handle);
9013 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
9014 handler
9015 .handle_control_frame(
9016 &module_ctx,
9017 hello_frame_with_control_ops(
9018 "late-health-response",
9019 PROTOCOL_VERSION,
9020 7,
9021 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9022 ),
9023 )
9024 .await
9025 .unwrap();
9026 let probe_started_at = Instant::now() - Duration::from_millis(80);
9027 let pending = forwarding
9028 .begin_health_probe_rpc_for(
9029 "late-health-response",
9030 MODULE_CONTROL_OP_HEALTH_CHECK,
9031 probe_started_at,
9032 Instant::now() - Duration::from_millis(1),
9033 )
9034 .unwrap();
9035 assert!(forwarding
9036 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
9037 .unwrap());
9038
9039 let responses = handler
9040 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
9041 .await
9042 .unwrap();
9043
9044 assert!(responses.is_empty());
9045 let health = module.status().unwrap().health;
9046 assert_eq!(health.late_answer_count, 1);
9047 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
9048 }
9049
9050 #[tokio::test]
9051 async fn health_probe_timeout_and_module_death_are_typed() {
9052 let registry = Arc::new(Registry::default());
9053 let forwarding = Arc::new(ForwardingTable::default());
9054 let handler =
9055 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9056 .with_health_probe_timeout(Duration::from_millis(50));
9057 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9058 hello_via_sink(
9059 &handler,
9060 &module_ctx,
9061 &mut module_rx,
9062 hello_frame_with_control_ops(
9063 "aft",
9064 PROTOCOL_VERSION,
9065 7,
9066 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9067 ),
9068 )
9069 .await;
9070
9071 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9072 let responses = handler
9073 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
9074 .await
9075 .unwrap();
9076 assert_eq!(responses[0].header.ty, FrameType::Error);
9077 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
9078 let _ = module_rx.try_recv();
9079
9080 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
9081 let health_handler = handler.clone();
9082 let death_task = tokio::spawn(async move {
9083 health_handler
9084 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
9085 .await
9086 .unwrap()
9087 });
9088 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9089 .await
9090 .unwrap()
9091 .unwrap();
9092 handler
9093 .cleanup_connection(module_ctx.connection_id)
9094 .unwrap();
9095 let responses = death_task.await.unwrap();
9096 assert_eq!(responses[0].header.ty, FrameType::Error);
9097 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
9098 }
9099
9100 #[test]
9101 fn hello_requires_exact_protocol_version() {
9102 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
9103 let registry = Arc::new(Registry::default());
9104 let handler = ControlHandler::new(Arc::clone(®istry));
9105 let responses = handler
9106 .handle_control(
9107 ConnectionId::new(connection),
9108 hello_frame("aft", offered, 9),
9109 )
9110 .unwrap();
9111
9112 assert_eq!(responses.len(), 1);
9113 assert_eq!(responses[0].header.ty, FrameType::Error);
9114 let error = parse_error(&responses[0]);
9115 assert_eq!(error["code"], "version_unsupported");
9116 assert!(registry.get_module("aft").unwrap().is_none());
9117 assert_eq!(registry.active_registration_count().unwrap(), 0);
9118 }
9119 }
9120
9121 #[test]
9122 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
9123 let registry = Arc::new(Registry::default());
9124 let forwarding = Arc::new(ForwardingTable::default());
9125 let handler =
9126 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9127 let module_connection = ConnectionId::new(301);
9128 let registration = registry
9129 .register_with_control_ops(
9130 manifest("aft-push", PROTOCOL_VERSION),
9131 PROTOCOL_VERSION,
9132 module_connection,
9133 module_baseline_control_ops(),
9134 )
9135 .unwrap();
9136 let (module_tx, _module_rx) = mpsc::channel(8);
9137 let endpoint = forwarding
9138 .register_module_connection(
9139 module_connection,
9140 "aft-push".to_string(),
9141 PROTOCOL_VERSION,
9142 manifest_concurrency(®istration.manifest),
9143 FrameSink::new(module_tx),
9144 )
9145 .unwrap();
9146
9147 let unknown = Frame::build(
9149 FrameType::Push,
9150 control_flags(),
9151 0,
9152 0,
9153 5,
9154 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
9155 )
9156 .unwrap();
9157 let out = handler.handle_status_update(endpoint, unknown).unwrap();
9158 assert!(
9159 out.is_empty(),
9160 "unknown push op must be ignored, got {out:?}"
9161 );
9162
9163 let malformed = Frame::build(
9165 FrameType::Push,
9166 control_flags(),
9167 0,
9168 0,
9169 6,
9170 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
9171 )
9172 .unwrap();
9173 let out = handler.handle_status_update(endpoint, malformed).unwrap();
9174 assert_eq!(out.len(), 1);
9175 assert_eq!(out[0].header.ty, FrameType::Error);
9176 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
9177 }
9178
9179 #[test]
9180 fn hello_rejected_when_connection_already_owns_client_routes() {
9181 let registry = Arc::new(Registry::default());
9182 let forwarding = Arc::new(ForwardingTable::default());
9183 let handler =
9184 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9185 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
9187 let client_connection = ConnectionId::new(202);
9188
9189 let responses = handler
9192 .handle_control(
9193 client_connection,
9194 hello_frame("aft-second", PROTOCOL_VERSION, 9),
9195 )
9196 .unwrap();
9197 assert_eq!(responses[0].header.ty, FrameType::Error);
9198 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
9199 assert!(registry.get_module("aft-second").unwrap().is_none());
9200 }
9201
9202 #[test]
9203 fn reserved_module_hello_requires_matching_launch_nonce() {
9204 let registry = Arc::new(Registry::default());
9205 let supervisor = SupervisorHandle::new();
9206 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9209 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9210
9211 let no_nonce = handler
9213 .handle_control(
9214 ConnectionId::new(1),
9215 hello_frame("vault", PROTOCOL_VERSION, 1),
9216 )
9217 .unwrap();
9218 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
9219 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
9220 assert!(registry.get_module("vault").unwrap().is_none());
9221
9222 let wrong = handler
9224 .handle_control(
9225 ConnectionId::new(2),
9226 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
9227 )
9228 .unwrap();
9229 assert_eq!(wrong[0].header.ty, FrameType::Error);
9230 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
9231 assert!(registry.get_module("vault").unwrap().is_none());
9232
9233 let ok = handler
9235 .handle_control(
9236 ConnectionId::new(3),
9237 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
9238 )
9239 .unwrap();
9240 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
9241 assert!(registry.get_module("vault").unwrap().is_some());
9242 }
9243
9244 #[test]
9245 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
9246 let registry = Arc::new(Registry::default());
9247 let supervisor = SupervisorHandle::new();
9248 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9249 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9250 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9251
9252 let squat = handler
9253 .handle_control(
9254 ConnectionId::new(1),
9255 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
9256 )
9257 .unwrap();
9258 assert_eq!(squat[0].header.ty, FrameType::Error);
9259 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
9260 assert!(parse_error(&squat[0])["message"]
9261 .as_str()
9262 .unwrap()
9263 .contains("fed:"));
9264
9265 let accepted_peer = handler
9266 .handle_control(
9267 ConnectionId::new(2),
9268 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
9269 )
9270 .unwrap();
9271 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
9272
9273 let accepted_short = handler
9274 .handle_control(
9275 ConnectionId::new(3),
9276 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
9277 )
9278 .unwrap();
9279 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
9280
9281 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
9282 let response = handler
9283 .handle_control(
9284 ConnectionId::new(conn),
9285 hello_frame(module_id, PROTOCOL_VERSION, conn),
9286 )
9287 .unwrap();
9288 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
9289 }
9290 }
9291
9292 #[test]
9293 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
9294 let registry = Arc::new(Registry::default());
9295 let supervisor = SupervisorHandle::new();
9296 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9297 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9298 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
9299 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9300
9301 let owner_nonce = handler
9302 .handle_control(
9303 ConnectionId::new(1),
9304 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
9305 )
9306 .unwrap();
9307 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
9308 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
9309 assert!(registry.get_module("fed:special").unwrap().is_none());
9310
9311 let exact_nonce = handler
9312 .handle_control(
9313 ConnectionId::new(2),
9314 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
9315 )
9316 .unwrap();
9317 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
9318 assert!(registry.get_module("fed:special").unwrap().is_some());
9319 }
9320
9321 #[test]
9322 fn non_reserved_module_ignores_launch_nonce() {
9323 let registry = Arc::new(Registry::default());
9324 let handler = ControlHandler::new(Arc::clone(®istry));
9327 let no_nonce = handler
9328 .handle_control(
9329 ConnectionId::new(1),
9330 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
9331 )
9332 .unwrap();
9333 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
9334 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
9335
9336 let echoed_nonce = handler
9337 .handle_control(
9338 ConnectionId::new(2),
9339 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
9340 )
9341 .unwrap();
9342 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
9343 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
9344 }
9345
9346 #[test]
9347 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
9348 let handler = ControlHandler::default();
9349 let conn = ConnectionId::new(1);
9350 let malformed = Frame::build(
9351 FrameType::Hello,
9352 control_flags(),
9353 0,
9354 0,
9355 3,
9356 b"{not json".to_vec(),
9357 )
9358 .unwrap();
9359
9360 let error = handler.handle_control(conn, malformed).unwrap();
9361 assert_eq!(error[0].header.ty, FrameType::Error);
9362 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
9363
9364 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
9365 let pong = handler.handle_control(conn, ping).unwrap();
9366 assert_eq!(pong[0].header.ty, FrameType::Pong);
9367 assert_eq!(pong[0].header.corr, 4);
9368 }
9369
9370 #[test]
9371 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
9372 let registry = Arc::new(Registry::default());
9373 let handler = ControlHandler::new(Arc::clone(®istry));
9374
9375 handler
9376 .handle_control(
9377 ConnectionId::new(1),
9378 hello_frame("aft", PROTOCOL_VERSION, 1),
9379 )
9380 .unwrap();
9381 let duplicate = handler
9382 .handle_control(
9383 ConnectionId::new(2),
9384 hello_frame("aft", PROTOCOL_VERSION, 2),
9385 )
9386 .unwrap();
9387
9388 assert_eq!(duplicate[0].header.ty, FrameType::Error);
9389 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9390 let registration = registry.get_module("aft").unwrap().unwrap();
9391 assert_eq!(registration.connection_id, ConnectionId::new(1));
9392 }
9393
9394 #[test]
9395 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
9396 let registry = Arc::new(Registry::default());
9397 let forwarding = Arc::new(ForwardingTable::default());
9398 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
9399 let handler =
9400 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9401 .with_process_liveness(process_liveness);
9402 let (ctx, route_channel, route_epoch) =
9403 bind_liveness_route(®istry, &forwarding, "aft-dead");
9404 let responses = handler
9405 .handle_route_poll(
9406 &ctx,
9407 route_poll_frame(41, PollKind::Liveness, route_channel),
9408 route_channel,
9409 route_epoch,
9410 PollKind::Liveness,
9411 )
9412 .unwrap();
9413
9414 assert_eq!(responses.len(), 1);
9415 assert_eq!(responses[0].header.ty, FrameType::Response);
9416 assert_route_poll_liveness(&responses[0], false);
9417 }
9418
9419 #[test]
9420 fn liveness_poll_without_process_source_uses_bound_route() {
9421 let registry = Arc::new(Registry::default());
9422 let forwarding = Arc::new(ForwardingTable::default());
9423 let handler =
9424 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9425 let (ctx, route_channel, route_epoch) =
9426 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
9427 let responses = handler
9428 .handle_route_poll(
9429 &ctx,
9430 route_poll_frame(42, PollKind::Liveness, route_channel),
9431 route_channel,
9432 route_epoch,
9433 PollKind::Liveness,
9434 )
9435 .unwrap();
9436
9437 assert_route_poll_liveness(&responses[0], true);
9438 }
9439
9440 #[test]
9441 fn liveness_poll_untracked_process_source_uses_bound_route() {
9442 let registry = Arc::new(Registry::default());
9443 let forwarding = Arc::new(ForwardingTable::default());
9444 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
9445 let handler =
9446 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9447 .with_process_liveness(process_liveness);
9448 let (ctx, route_channel, route_epoch) =
9449 bind_liveness_route(®istry, &forwarding, "aft-untracked");
9450 let responses = handler
9451 .handle_route_poll(
9452 &ctx,
9453 route_poll_frame(43, PollKind::Liveness, route_channel),
9454 route_channel,
9455 route_epoch,
9456 PollKind::Liveness,
9457 )
9458 .unwrap();
9459
9460 assert_route_poll_liveness(&responses[0], true);
9461 }
9462
9463 #[tokio::test]
9464 async fn unknown_op_returns_unknown_control_op() {
9465 let handler = ControlHandler::default();
9466 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
9467 let request = Frame::build(
9468 FrameType::Request,
9469 control_flags(),
9470 0,
9471 0,
9472 55,
9473 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
9474 )
9475 .unwrap();
9476
9477 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9478
9479 assert_eq!(response.len(), 1);
9480 assert_eq!(response[0].header.ty, FrameType::Error);
9481 assert_eq!(response[0].header.corr, 55);
9482 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
9483 }
9484
9485 #[tokio::test]
9486 async fn supervisor_provenance_rejects_unknown_exact_module() {
9487 let handler = ControlHandler::default();
9488 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
9489 let request = Frame::build(
9490 FrameType::Request,
9491 control_flags(),
9492 0,
9493 0,
9494 57,
9495 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
9496 )
9497 .unwrap();
9498
9499 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9500
9501 assert_eq!(response.len(), 1);
9502 assert_eq!(response[0].header.ty, FrameType::Error);
9503 assert_eq!(response[0].header.corr, 57);
9504 let error = parse_error(&response[0]);
9505 assert_eq!(error["code"], "unknown_module");
9506 assert_eq!(error["message"], "module_id 'missing' is not supervised");
9507 }
9508
9509 #[test]
9510 fn provenance_probe_override_keeps_handler_tests_deterministic() {
9511 let expected = subc_control::RunningImageAgreement::Unavailable {
9512 reason: subc_control::RunningImageUnavailableReason::HashFailed,
9513 };
9514 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
9515 assert_eq!(handler.provenance_probe_override, Some(expected));
9516 }
9517
9518 #[test]
9519 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
9520 let verdict = reload_verdict(
9521 std::path::Path::new("/bin/new"),
9522 Some(std::path::Path::new("/bin/old")),
9523 subc_control::RunningImageAgreement::Unavailable {
9524 reason: subc_control::RunningImageUnavailableReason::HashFailed,
9525 },
9526 );
9527 assert!(matches!(
9528 verdict.path,
9529 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
9530 if configured == std::path::Path::new("/bin/new")
9531 && spawned_from == std::path::Path::new("/bin/old")
9532 ));
9533 }
9534
9535 #[test]
9536 fn reload_verdict_detects_replaced_image_at_same_path() {
9537 let image = subc_control::RunningImageAgreement::Mismatch {
9538 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
9539 digest: "old".into(),
9540 },
9541 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
9542 digest: "new".into(),
9543 },
9544 };
9545 let verdict = reload_verdict(
9546 std::path::Path::new("/bin/same"),
9547 Some(std::path::Path::new("/bin/same")),
9548 image.clone(),
9549 );
9550 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9551 assert_eq!(verdict.image, image);
9552 }
9553
9554 #[test]
9555 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
9556 let image = subc_control::RunningImageAgreement::Unavailable {
9557 reason: subc_control::RunningImageUnavailableReason::NotRunning,
9558 };
9559 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
9560 assert_eq!(
9561 verdict.path,
9562 subc_control::ReloadPathAgreement::Unavailable {
9563 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
9564 }
9565 );
9566 assert_eq!(verdict.image, image);
9567
9568 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
9569 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
9570 };
9571 let verdict = reload_verdict(
9572 std::path::Path::new("/bin/same"),
9573 Some(std::path::Path::new("/bin/same")),
9574 unconfirmed.clone(),
9575 );
9576 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9577 assert_eq!(verdict.image, unconfirmed);
9578 }
9579
9580 #[test]
9581 fn reload_verdict_preserves_each_image_unavailability_reason() {
9582 use subc_control::RunningImageUnavailableReason as Reason;
9583
9584 for reason in [
9585 Reason::NotRunning,
9586 Reason::UnsupportedPlatform,
9587 Reason::RunningExecutableUnreadable,
9588 Reason::SpawnedPathUnreadable,
9589 Reason::HashFailed,
9590 Reason::ProcessIdentityUnconfirmed,
9591 Reason::Unknown("future_probe_reason".to_string()),
9592 ] {
9593 let image = subc_control::RunningImageAgreement::Unavailable {
9594 reason: reason.clone(),
9595 };
9596 let verdict = reload_verdict(
9597 std::path::Path::new("/bin/same"),
9598 Some(std::path::Path::new("/bin/same")),
9599 image.clone(),
9600 );
9601 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
9602 assert_eq!(verdict.image, image, "{reason:?}");
9603 }
9604 }
9605
9606 #[tokio::test]
9607 async fn malformed_control_bodies_return_invalid_control_body() {
9608 let handler = ControlHandler::default();
9609 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
9610
9611 for (corr, body) in [
9612 (56, br#"{"route_channel":1}"#.as_slice()),
9613 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
9614 (
9615 58,
9616 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
9617 ),
9618 ] {
9619 let request = Frame::build(
9620 FrameType::Request,
9621 control_flags(),
9622 0,
9623 0,
9624 corr,
9625 body.to_vec(),
9626 )
9627 .unwrap();
9628 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
9629
9630 assert_eq!(response.len(), 1);
9631 assert_eq!(response[0].header.ty, FrameType::Error);
9632 assert_eq!(response[0].header.corr, corr);
9633 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
9634 }
9635 }
9636
9637 #[tokio::test]
9638 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
9639 let registry = Arc::new(Registry::default());
9640 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
9641 let router = Router::with_control_handler(Arc::clone(&control));
9642 let connection = router.begin_connection();
9643 let (ctx, mut rx) = route_ctx(connection.id());
9644
9645 router
9646 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
9647 .await
9648 .unwrap();
9649 let response = rx.recv().await.unwrap();
9650 let ack = parse_ack(&response);
9651 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9652 let channel = 1;
9653
9654 let goodbye =
9655 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
9656 router.route_for_connection(&ctx, goodbye).await.unwrap();
9657 assert!(rx.try_recv().is_err());
9658 assert!(registry.get_module("aft").unwrap().is_none());
9659
9660 router
9661 .route_for_connection(&ctx, channel_request(channel, 13))
9662 .await
9663 .unwrap();
9664 let error_frame = rx.recv().await.unwrap();
9665 assert_eq!(error_frame.header.ty, FrameType::Error);
9666 assert_eq!(error_frame.header.channel, channel);
9667 }
9668
9669 #[tokio::test]
9670 async fn dropping_router_connection_releases_registration() {
9671 let registry = Arc::new(Registry::default());
9672 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
9673 let router = Router::with_control_handler(control);
9674 let connection = router.begin_connection();
9675 let (ctx, mut rx) = route_ctx(connection.id());
9676
9677 router
9678 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
9679 .await
9680 .unwrap();
9681 let response = rx.recv().await.unwrap();
9682 let ack = parse_ack(&response);
9683 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9684 assert!(registry.get_module("aft").unwrap().is_some());
9685
9686 drop(connection);
9687
9688 assert!(registry.get_module("aft").unwrap().is_none());
9689 assert_eq!(registry.active_registration_count().unwrap(), 0);
9690 }
9691
9692 fn capability_manifest(
9693 module_id: &str,
9694 provides: &[&str],
9695 must_never_reach: &[&str],
9696 ) -> ModuleManifest {
9697 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9698 manifest.capabilities = Some(CapabilityDeclarations {
9699 provides: provides
9700 .iter()
9701 .map(|capability| (*capability).to_string())
9702 .collect(),
9703 requires: Vec::new(),
9704 must_never_reach: must_never_reach
9705 .iter()
9706 .map(|capability| (*capability).to_string())
9707 .collect(),
9708 });
9709 manifest
9710 }
9711
9712 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
9713 Frame::build(
9714 FrameType::Hello,
9715 control_flags(),
9716 0,
9717 0,
9718 corr,
9719 serde_json::to_vec(&ModuleHelloBody {
9720 protocol_ver: manifest.protocol_ver,
9721 manifest,
9722 control_ops: None,
9723 launch_nonce: None,
9724 })
9725 .expect("capability test HELLO serializes"),
9726 )
9727 .expect("capability test HELLO frame builds")
9728 }
9729
9730 fn catalog_update_with_capabilities_frame(
9731 corr: u64,
9732 capabilities: CapabilityDeclarations,
9733 ) -> Frame {
9734 Frame::build(
9735 FrameType::Request,
9736 control_flags(),
9737 0,
9738 0,
9739 corr,
9740 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
9741 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
9742 capabilities: Some(capabilities),
9743 ready: None,
9744 })
9745 .expect("capability catalog.update serializes"),
9746 )
9747 .expect("capability catalog.update frame builds")
9748 }
9749
9750 async fn register_capability_manifest(
9751 handler: &ControlHandler,
9752 ctx: &RouteCtx,
9753 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9754 manifest: ModuleManifest,
9755 corr: u64,
9756 ) {
9757 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
9758 }
9759
9760 async fn open_route_for_capability_test(
9761 handler: &ControlHandler,
9762 target_ctx: &RouteCtx,
9763 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9764 client_connection_id: u64,
9765 corr: u64,
9766 target_module_id: &str,
9767 consumer_identity: Option<ConsumerIdentity>,
9768 ) -> (
9769 mpsc::Receiver<crate::router::OutboundFrame>,
9770 ModuleControlRequest,
9771 ) {
9772 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
9773 let route_handler = handler.clone();
9774 let target_module_id = target_module_id.to_string();
9775 let route_task = tokio::spawn(async move {
9776 route_handler
9777 .handle_control_frame(
9778 &client_ctx,
9779 route_open_frame_with_admission_facts(
9780 corr,
9781 &target_module_id,
9782 unique_project_root("admission-facts"),
9783 consumer_identity,
9784 None,
9785 ),
9786 )
9787 .await
9788 .expect("capability test route.open succeeds")
9789 });
9790 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
9791 .await
9792 .expect("capability test route.open must reach route.bind")
9793 .expect("target control receiver stays open");
9794 let bind_request: ModuleControlRequest =
9795 serde_json::from_slice(&bind.body).expect("route.bind decodes");
9796 handler
9797 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
9798 .await
9799 .expect("capability test route.bind ACK succeeds");
9800 assert!(route_task.await.expect("route.open task joins").is_empty());
9801 let opened = client_rx
9802 .recv()
9803 .await
9804 .expect("successful route.open publishes a response");
9805 assert!(matches!(
9806 serde_json::from_slice::<ClientControlResponse>(&opened.body),
9807 Ok(ClientControlResponse::RouteOpen { .. })
9808 ));
9809 (client_rx, bind_request)
9810 }
9811
9812 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
9813 assert_eq!(frame.header.ty, FrameType::Push);
9814 assert_eq!(frame.header.channel, 0);
9815 assert_eq!(
9816 serde_json::from_slice::<ClientControlPush>(&frame.body)
9817 .expect("route.closed control push decodes"),
9818 ClientControlPush::RouteClosed {
9819 module_id: target_module_id.to_string(),
9820 reason: RouteCloseReason::CapabilityDenied,
9821 drained: false,
9822 abandoned: 0,
9823 excluded_subscriptions: 0,
9824 terminal: Some(false),
9825 }
9826 );
9827 }
9828
9829 #[tokio::test]
9830 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
9831 let registry = Arc::new(Registry::default());
9832 let forwarding = Arc::new(ForwardingTable::default());
9833 let supervisor = SupervisorHandle::new();
9834 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9835 let handler =
9836 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9837 .with_supervisor(supervisor);
9838 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
9839 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
9840 register_capability_manifest(
9841 &handler,
9842 &target_ctx,
9843 &mut target_rx,
9844 capability_manifest("target", &["credentials-provider/v1"], &[]),
9845 1,
9846 )
9847 .await;
9848 register_capability_manifest(
9849 &handler,
9850 &opener_ctx,
9851 &mut opener_rx,
9852 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9853 2,
9854 )
9855 .await;
9856
9857 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
9858 let replies = handler
9859 .handle_control_frame(
9860 &client_ctx,
9861 route_open_frame_with_admission_facts(
9862 3,
9863 "target",
9864 unique_project_root("admission-facts"),
9865 Some(ConsumerIdentity {
9866 module_id: "opener".to_string(),
9867 launch_nonce: "opener-nonce".to_string(),
9868 }),
9869 None,
9870 ),
9871 )
9872 .await
9873 .expect("denied route.open returns a typed frame");
9874 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
9875 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9876 assert!(
9877 target_rx.try_recv().is_err(),
9878 "forbidden route.open must not relay route.bind"
9879 );
9880 }
9881
9882 #[tokio::test]
9883 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
9884 let registry = Arc::new(Registry::default());
9885 let forwarding = Arc::new(ForwardingTable::default());
9886 let supervisor = SupervisorHandle::new();
9887 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9888 let handler =
9889 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9890 .with_supervisor(supervisor);
9891 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
9892 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
9893 register_capability_manifest(
9894 &handler,
9895 &target_ctx,
9896 &mut target_rx,
9897 capability_manifest("target", &["credentials-provider/v1"], &[]),
9898 1,
9899 )
9900 .await;
9901 register_capability_manifest(
9902 &handler,
9903 &old_opener_ctx,
9904 &mut old_opener_rx,
9905 capability_manifest("opener", &[], &[]),
9906 2,
9907 )
9908 .await;
9909 let (mut client_rx, _) = open_route_for_capability_test(
9910 &handler,
9911 &target_ctx,
9912 &mut target_rx,
9913 712,
9914 3,
9915 "target",
9916 Some(ConsumerIdentity {
9917 module_id: "opener".to_string(),
9918 launch_nonce: "opener-nonce".to_string(),
9919 }),
9920 )
9921 .await;
9922 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9923
9924 handler
9925 .cleanup_connection(old_opener_ctx.connection_id)
9926 .expect("old opener registration cleans up");
9927 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
9928 register_capability_manifest(
9929 &handler,
9930 &new_opener_ctx,
9931 &mut new_opener_rx,
9932 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9933 4,
9934 )
9935 .await;
9936
9937 assert_capability_denied_push(
9938 client_rx
9939 .try_recv()
9940 .expect("HELLO deny addition must emit route.closed")
9941 .frame,
9942 "target",
9943 );
9944 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
9945 assert!(matches!(
9946 target_rx.try_recv(),
9947 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
9948 ));
9949 }
9950
9951 #[tokio::test]
9952 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
9953 let registry = Arc::new(Registry::default());
9954 let forwarding = Arc::new(ForwardingTable::default());
9955 let supervisor = SupervisorHandle::new();
9956 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
9957 let handler =
9958 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9959 .with_supervisor(supervisor);
9960 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
9961 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
9962 register_capability_manifest(
9963 &handler,
9964 &target_ctx,
9965 &mut target_rx,
9966 capability_manifest("target", &[], &[]),
9967 1,
9968 )
9969 .await;
9970 register_capability_manifest(
9971 &handler,
9972 &opener_ctx,
9973 &mut opener_rx,
9974 capability_manifest("opener", &[], &["credentials-provider/v1"]),
9975 2,
9976 )
9977 .await;
9978 let (mut client_rx, _) = open_route_for_capability_test(
9979 &handler,
9980 &target_ctx,
9981 &mut target_rx,
9982 722,
9983 3,
9984 "target",
9985 Some(ConsumerIdentity {
9986 module_id: "opener".to_string(),
9987 launch_nonce: "opener-nonce".to_string(),
9988 }),
9989 )
9990 .await;
9991 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
9992
9993 let replies = handler
9994 .handle_control_frame(
9995 &target_ctx,
9996 catalog_update_with_capabilities_frame(
9997 4,
9998 CapabilityDeclarations {
9999 provides: vec!["credentials-provider/v1".to_string()],
10000 requires: Vec::new(),
10001 must_never_reach: Vec::new(),
10002 },
10003 ),
10004 )
10005 .await
10006 .expect("claim catalog.update succeeds");
10007 assert!(matches!(
10008 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
10009 Ok(ModuleControlResponseToModule::CatalogUpdate {})
10010 ));
10011 assert_capability_denied_push(
10012 client_rx
10013 .try_recv()
10014 .expect("claim addition must emit route.closed")
10015 .frame,
10016 "target",
10017 );
10018 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10019 assert!(matches!(
10020 target_rx.try_recv(),
10021 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10022 ));
10023 }
10024
10025 #[tokio::test]
10026 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
10027 let registry = Arc::new(Registry::default());
10028 let forwarding = Arc::new(ForwardingTable::default());
10029 let supervisor = SupervisorHandle::new();
10030 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10031 let handler =
10032 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10033 .with_supervisor(supervisor);
10034 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
10035 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
10036 register_capability_manifest(
10037 &handler,
10038 &target_ctx,
10039 &mut target_rx,
10040 capability_manifest("target", &["credentials-provider/v1"], &[]),
10041 1,
10042 )
10043 .await;
10044 register_capability_manifest(
10045 &handler,
10046 &opener_ctx,
10047 &mut opener_rx,
10048 capability_manifest("opener", &[], &[]),
10049 2,
10050 )
10051 .await;
10052 let (mut client_rx, _) = open_route_for_capability_test(
10053 &handler,
10054 &target_ctx,
10055 &mut target_rx,
10056 732,
10057 3,
10058 "target",
10059 Some(ConsumerIdentity {
10060 module_id: "opener".to_string(),
10061 launch_nonce: "opener-nonce".to_string(),
10062 }),
10063 )
10064 .await;
10065 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10066
10067 handler
10068 .handle_control_frame(
10069 &target_ctx,
10070 catalog_update_with_capabilities_frame(
10071 4,
10072 CapabilityDeclarations {
10073 provides: Vec::new(),
10074 requires: Vec::new(),
10075 must_never_reach: Vec::new(),
10076 },
10077 ),
10078 )
10079 .await
10080 .expect("claim removal catalog.update succeeds");
10081 assert_eq!(
10082 forwarding.active_binding_count().unwrap(),
10083 1,
10084 "removing an attested target claim must leave the route census unchanged"
10085 );
10086 assert!(
10087 client_rx.try_recv().is_err(),
10088 "claim removal must not emit route.closed capability_denied"
10089 );
10090 assert!(
10091 target_rx.try_recv().is_err(),
10092 "claim removal must not send the target a route GOODBYE"
10093 );
10094 }
10095
10096 #[tokio::test]
10099 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
10100 let registry = Arc::new(Registry::default());
10101 let forwarding = Arc::new(ForwardingTable::default());
10102 let supervisor = SupervisorHandle::new();
10103 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10104 let handler =
10105 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10106 .with_supervisor(supervisor);
10107 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
10108 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
10109 register_capability_manifest(
10110 &handler,
10111 &target_ctx,
10112 &mut target_rx,
10113 capability_manifest("target", &["credentials-provider/v1"], &[]),
10114 1,
10115 )
10116 .await;
10117 register_capability_manifest(
10118 &handler,
10119 &opener_ctx,
10120 &mut opener_rx,
10121 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10122 2,
10123 )
10124 .await;
10125
10126 let (_client_rx, bind) = open_route_for_capability_test(
10127 &handler,
10128 &target_ctx,
10129 &mut target_rx,
10130 742,
10131 3,
10132 "target",
10133 None,
10134 )
10135 .await;
10136 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10137 panic!("direct scope-honesty route must bind");
10138 };
10139 assert_eq!(principal, Some(Principal::Direct));
10140 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10141 }
10142
10143 #[tokio::test]
10146 async fn must_never_reach_self_route_is_capability_forbidden() {
10147 let registry = Arc::new(Registry::default());
10148 let forwarding = Arc::new(ForwardingTable::default());
10149 let supervisor = SupervisorHandle::new();
10150 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
10151 let handler =
10152 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10153 .with_supervisor(supervisor);
10154 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
10155 register_capability_manifest(
10156 &handler,
10157 &self_ctx,
10158 &mut self_rx,
10159 capability_manifest(
10160 "self-provider",
10161 &["credentials-provider/v1"],
10162 &["credentials-provider/v1"],
10163 ),
10164 1,
10165 )
10166 .await;
10167
10168 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
10169 let replies = handler
10170 .handle_control_frame(
10171 &client_ctx,
10172 route_open_frame_with_admission_facts(
10173 2,
10174 "self-provider",
10175 unique_project_root("admission-facts"),
10176 Some(ConsumerIdentity {
10177 module_id: "self-provider".to_string(),
10178 launch_nonce: "self-nonce".to_string(),
10179 }),
10180 None,
10181 ),
10182 )
10183 .await
10184 .expect("self-route refusal returns a typed frame");
10185 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10186 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10187 assert!(
10188 self_rx.try_recv().is_err(),
10189 "self denial must not relay route.bind"
10190 );
10191 }
10192
10193 #[test]
10194 fn unsupported_channel_zero_frame_returns_error() {
10195 let handler = ControlHandler::default();
10196 let request = Frame::build(
10197 FrameType::Request,
10198 control_flags(),
10199 0,
10200 0,
10201 21,
10202 b"opaque".to_vec(),
10203 )
10204 .unwrap();
10205
10206 let response = handler
10207 .handle_control(ConnectionId::new(1), request)
10208 .unwrap();
10209
10210 assert_eq!(response[0].header.ty, FrameType::Error);
10211 assert_eq!(
10212 parse_error(&response[0])["code"],
10213 "unsupported_control_frame"
10214 );
10215 }
10216
10217 mod swap {
10222 use super::*;
10223
10224 const INCUMBENT: ConnectionId = ConnectionId::new(30);
10225 const CANDIDATE: ConnectionId = ConnectionId::new(40);
10226
10227 struct Swap {
10228 registry: Arc<Registry>,
10229 forwarding: Arc<ForwardingTable>,
10230 handler: ControlHandler,
10231 incumbent_ctx: RouteCtx,
10232 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10233 candidate_ctx: RouteCtx,
10234 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10235 }
10236
10237 async fn swap_with_incumbent() -> Swap {
10238 let registry = Arc::new(Registry::default());
10239 let forwarding = Arc::new(ForwardingTable::default());
10240 let handler =
10241 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10242 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
10243 hello_via_sink(
10244 &handler,
10245 &incumbent_ctx,
10246 &mut incumbent_rx,
10247 hello_frame("aft", PROTOCOL_VERSION, 7),
10248 )
10249 .await;
10250 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
10251 Swap {
10252 registry,
10253 forwarding,
10254 handler,
10255 incumbent_ctx,
10256 incumbent_rx,
10257 candidate_ctx,
10258 candidate_rx,
10259 }
10260 }
10261
10262 fn register_candidate(swap: &Swap, ready: Option<bool>) {
10263 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
10264 candidate_manifest.ready = ready;
10265 let registration = swap
10266 .registry
10267 .register_candidate_with_control_ops(
10268 candidate_manifest,
10269 PROTOCOL_VERSION,
10270 CANDIDATE,
10271 module_baseline_control_ops(),
10272 )
10273 .unwrap();
10274 swap.forwarding
10275 .register_candidate_module_connection(
10276 CANDIDATE,
10277 "aft".to_string(),
10278 PROTOCOL_VERSION,
10279 manifest_concurrency(®istration.manifest),
10280 swap.candidate_ctx.egress.clone(),
10281 )
10282 .unwrap();
10283 }
10284
10285 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
10286 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
10287 swap.registry.promote_candidate("aft").unwrap().unwrap();
10288 cutover.incumbent.unwrap()
10289 }
10290
10291 fn keyed_total(counters: &Value, key: &str) -> u64 {
10292 counters[key]
10293 .as_object()
10294 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
10295 .unwrap_or(0)
10296 }
10297
10298 #[tokio::test]
10304 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
10305 let mut swap = swap_with_incumbent().await;
10306 let handler = swap.handler.clone();
10307
10308 let cotenant = ConnectionId::new(31);
10310 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10311 let (cotenant_task, cotenant_bind) = relay_route_open(
10312 &handler,
10313 cotenant,
10314 &cotenant_ctx.egress,
10315 &mut swap.incumbent_rx,
10316 100,
10317 "aft",
10318 "swap-cotenant",
10319 )
10320 .await;
10321 handler
10322 .handle_control_frame(
10323 &swap.incumbent_ctx,
10324 route_bind_ack(cotenant_bind.header.corr),
10325 )
10326 .await
10327 .unwrap();
10328 assert!(cotenant_task.await.unwrap().is_empty());
10329 let (cotenant_channel, cotenant_epoch) =
10330 published_route(&cotenant_rx.recv().await.unwrap());
10331
10332 let caller = ConnectionId::new(32);
10334 let (caller_ctx, mut caller_rx) = route_ctx(caller);
10335 let (caller_task, caller_bind) = relay_route_open(
10336 &handler,
10337 caller,
10338 &caller_ctx.egress,
10339 &mut swap.incumbent_rx,
10340 101,
10341 "aft",
10342 "swap-caller",
10343 )
10344 .await;
10345 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
10346
10347 register_candidate(&swap, None);
10348 cutover(&swap);
10349
10350 let ack = handler
10352 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
10353 .await;
10354 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10355 if module_loop_error.is_some() {
10356 handler.cleanup_connection(INCUMBENT).unwrap();
10359 }
10360
10361 assert!(
10363 cotenant_rx.try_recv().is_err(),
10364 "the co-tenant route on the incumbent was torn down by one late ack: \
10365 {module_loop_error:?}"
10366 );
10367 assert!(matches!(
10368 swap.forwarding
10369 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10370 .unwrap(),
10371 DataRoute::Client(DataRouteState::Bound(_))
10372 ));
10373 assert_eq!(module_loop_error, None);
10374 assert!(swap
10375 .registry
10376 .get_module_by_connection(INCUMBENT)
10377 .unwrap()
10378 .is_some());
10379
10380 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
10382 .await
10383 .expect("the incumbent is told to drop the abandoned binding")
10384 .unwrap()
10385 .frame;
10386 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10387 assert_eq!(goodbye.header.channel, abandoned_channel);
10388 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10389 assert!(swap.incumbent_rx.try_recv().is_err());
10390
10391 let response = caller_task.await.unwrap();
10393 assert_eq!(response.len(), 1);
10394 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
10395 assert!(caller_rx.try_recv().is_err());
10396
10397 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
10401 let counters = handler.counters().snapshot();
10402 assert_eq!(
10403 keyed_total(&counters, "route_open_accepted_by_principal"),
10404 1
10405 );
10406 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
10407 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
10408 }
10409
10410 #[tokio::test]
10414 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
10415 let mut swap = swap_with_incumbent().await;
10416 register_candidate(&swap, None);
10417 let incumbent = cutover(&swap);
10418 swap.forwarding
10419 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
10420 .unwrap()
10421 .expect("the incumbent is still registered");
10422
10423 let client = ConnectionId::new(33);
10424 let (client_ctx, mut client_rx) = route_ctx(client);
10425 let route_handler = swap.handler.clone();
10426 let open_ctx = RouteCtx {
10427 connection_id: client,
10428 egress: client_ctx.egress.clone(),
10429 };
10430 let mut route_task = tokio::spawn(async move {
10431 route_handler
10432 .handle_control_frame(
10433 &open_ctx,
10434 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
10435 )
10436 .await
10437 .unwrap()
10438 });
10439 let bind = tokio::select! {
10440 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
10441 response = &mut route_task => {
10442 let response = response.unwrap();
10443 panic!(
10444 "post-cutover route.open was refused instead of relayed to the candidate: {}",
10445 parse_error(&response[0])["code"]
10446 );
10447 }
10448 };
10449 swap.handler
10450 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
10451 .await
10452 .unwrap();
10453 assert!(route_task.await.unwrap().is_empty());
10454 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
10455 match swap
10456 .forwarding
10457 .lookup_data_route(client, channel, epoch)
10458 .unwrap()
10459 {
10460 DataRoute::Client(DataRouteState::Bound(route)) => {
10461 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
10462 }
10463 other => panic!("expected a bound route on the candidate, got {other:?}"),
10464 }
10465 assert!(swap.incumbent_rx.try_recv().is_err());
10466 }
10467
10468 #[tokio::test]
10473 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
10474 let swap = swap_with_incumbent().await;
10475 register_candidate(&swap, Some(false));
10476 let update = Frame::build(
10477 FrameType::Request,
10478 control_flags(),
10479 0,
10480 0,
10481 55,
10482 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10483 provides: manifest("aft", PROTOCOL_VERSION).provides,
10484 capabilities: None,
10485 ready: Some(true),
10486 })
10487 .unwrap(),
10488 )
10489 .unwrap();
10490
10491 let replies = swap
10492 .handler
10493 .handle_control_frame(&swap.candidate_ctx, update)
10494 .await
10495 .unwrap();
10496
10497 assert_eq!(replies.len(), 1);
10498 assert_eq!(
10499 replies[0].header.ty,
10500 FrameType::Response,
10501 "candidate catalog.update was refused: {:?}",
10502 serde_json::from_slice::<Value>(&replies[0].body).ok()
10503 );
10504 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
10505 assert_eq!(
10506 swap.registry
10507 .get_module("aft")
10508 .unwrap()
10509 .unwrap()
10510 .connection_id,
10511 INCUMBENT
10512 );
10513 }
10514 }
10515
10516 mod swap_admission {
10520 use super::*;
10521
10522 const INCUMBENT_NONCE: &str = "incumbent-nonce";
10523 const CANDIDATE_NONCE: &str = "candidate-nonce";
10524
10525 fn handler_with_incumbent(
10526 module_id: &str,
10527 reserved: bool,
10528 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
10529 let registry = Arc::new(Registry::default());
10530 let supervisor = SupervisorHandle::new();
10531 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
10532 if reserved {
10533 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
10534 }
10535 let handler =
10536 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
10537 let incumbent = handler
10538 .handle_control(
10539 ConnectionId::new(1),
10540 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
10541 )
10542 .unwrap();
10543 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
10544 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
10545 (registry, supervisor, handler)
10546 }
10547
10548 #[test]
10554 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
10555 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
10556
10557 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
10558 let replies = handler
10559 .handle_control(
10560 ConnectionId::new(connection),
10561 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
10562 )
10563 .unwrap();
10564 assert_eq!(replies[0].header.ty, FrameType::Error);
10565 assert_eq!(
10566 parse_error(&replies[0])["code"],
10567 "swap_token_invalid",
10568 "nonce {nonce:?}"
10569 );
10570 }
10571 assert!(registry.get_candidate("aft").unwrap().is_none());
10572 assert_eq!(
10573 registry.get_module("aft").unwrap().unwrap().connection_id,
10574 ConnectionId::new(1)
10575 );
10576
10577 let admitted = handler
10580 .handle_control(
10581 ConnectionId::new(4),
10582 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
10583 )
10584 .unwrap();
10585 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
10586 assert_eq!(
10587 registry
10588 .get_candidate("aft")
10589 .unwrap()
10590 .unwrap()
10591 .connection_id,
10592 ConnectionId::new(4)
10593 );
10594 assert_eq!(
10595 registry.get_module("aft").unwrap().unwrap().connection_id,
10596 ConnectionId::new(1),
10597 "the candidate must not take the active slot"
10598 );
10599 let replayed = handler
10600 .handle_control(
10601 ConnectionId::new(5),
10602 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
10603 )
10604 .unwrap();
10605 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
10606
10607 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
10611 let squatter = handler
10612 .handle_control(
10613 ConnectionId::new(6),
10614 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
10615 )
10616 .unwrap();
10617 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
10618 assert!(
10619 registry.get_module("aft").unwrap().is_none(),
10620 "a squatter took the active slot of an id being swapped"
10621 );
10622 }
10623
10624 #[test]
10629 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
10630 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
10631
10632 let replies = handler
10633 .handle_control(
10634 ConnectionId::new(2),
10635 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10636 )
10637 .unwrap();
10638
10639 assert_eq!(
10640 replies[0].header.ty,
10641 FrameType::HelloAck,
10642 "reserved candidate refused: {:?}",
10643 serde_json::from_slice::<Value>(&replies[0].body).ok()
10644 );
10645 assert_eq!(
10646 registry
10647 .get_candidate("vault")
10648 .unwrap()
10649 .unwrap()
10650 .connection_id,
10651 ConnectionId::new(2)
10652 );
10653 }
10654
10655 #[test]
10658 fn without_an_open_swap_the_ordinary_gates_decide() {
10659 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
10660 supervisor.close_swap("vault");
10661
10662 let candidate = handler
10663 .handle_control(
10664 ConnectionId::new(2),
10665 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
10666 )
10667 .unwrap();
10668 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
10669 let duplicate = handler
10670 .handle_control(
10671 ConnectionId::new(3),
10672 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
10673 )
10674 .unwrap();
10675 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10676 assert!(registry.get_candidate("vault").unwrap().is_none());
10677 }
10678 }
10679}
10680
10681#[cfg(test)]
10682mod concurrency_default_exposure_tests {
10683 use super::*;
10684
10685 fn hello_body(role_json: &str) -> Vec<u8> {
10686 format!(
10687 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
10688 )
10689 .into_bytes()
10690 }
10691
10692 fn manifest_from(body: &[u8]) -> ModuleManifest {
10693 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
10694 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
10695 .expect("manifest parses")
10696 }
10697
10698 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
10699
10700 #[test]
10701 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
10702 let body = hello_body(&format!(
10703 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
10704 ));
10705 let manifest = manifest_from(&body);
10706 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10709 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
10710 }
10711
10712 #[test]
10713 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
10714 let body = hello_body(&format!(
10715 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
10716 ));
10717 let manifest = manifest_from(&body);
10718 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
10719 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10720 }
10721
10722 #[test]
10723 fn non_management_roles_are_never_reported() {
10724 let body = hello_body(
10725 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
10726 );
10727 let manifest = manifest_from(&body);
10728 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
10729 }
10730}