1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24 ProviderRole,
25 },
26 scope::{
27 ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector, CAP_SCOPES_V1,
28 SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP,
29 },
30 session::{
31 HealthReport, ModuleControlPush, ModuleControlRequest, ModuleControlRequestFromModule,
32 ModuleControlResponse, ModuleControlResponseToModule, MODULE_CONTROL_OP_HEALTH_CHECK,
33 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
34 },
35 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
36 Priority, RouteTarget, PROTOCOL_VERSION,
37};
38use tokio::time::{timeout_at, Instant};
39use tracing::{debug, info, warn};
40
41use crate::{
42 capability_requirements::{
43 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
44 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
45 RuntimeModule,
46 },
47 daemon_config::RestartRequiredSection,
48 forwarding::{
49 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
50 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
51 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
52 },
53 observability::{
54 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
55 },
56 provenance::{
57 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
58 },
59 registry::{ChannelState, ConnectionId, Registry, RegistryError},
60 router::{RouteCtx, RouterError},
61 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
62 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
63 stderr_tail::{CaptureState, TailEntry},
64 supervise::{
65 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
66 SupervisorHandle, SwapHelloAdmission,
67 },
68 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
69};
70
71pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
77
78const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
79const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
80const CAP_PING_PONG: &str = "ping_pong_v1";
81const CAP_SESSION_ATTACH: &str = "session_attach_v1";
82const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
83
84const SUBC_CONTROL_OPS: &[&str] = &[
85 ops::SERVER_DESCRIBE,
86 ops::CATALOG_LIST,
87 ops::ROUTE_OPEN,
88 ops::ROUTE_POLL,
89 ops::ROUTE_CLOSING,
90 ops::ROUTE_CLOSED,
91 ops::SUPERVISOR_LIST,
92 ops::SUPERVISOR_RESTART,
93 ops::SUPERVISOR_SWAP,
94 ops::SUPERVISOR_RELOAD,
95 ops::SUPERVISOR_RESCAN,
96 ops::SUPERVISOR_RELEASE_RESERVED,
97 ops::SUPERVISOR_SET_ENABLED,
98 ops::SUPERVISOR_HEALTH_PROBE,
99 ops::SUPERVISOR_HEALTH,
100 ops::SUPERVISOR_STDERR_TAIL,
101 ops::SUPERVISOR_TERMINALS,
102 ops::SUPERVISOR_ROUTES,
103 ops::SUPERVISOR_PROVENANCE,
104 ops::SUPERVISOR_SPAWN_SNAPSHOT,
105 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
106];
107
108const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
109 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
110 "supervisor.live_roots",
111 SCOPE_SYNC_OP,
112 SCOPE_DESCRIBE_OP,
113];
114
115const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
119
120const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
121
122pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
130
131pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
146
147pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
165
166const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
167const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
168
169fn reload_verdict(
170 configured: &Path,
171 spawned_from: Option<&Path>,
172 image: subc_control::RunningImageAgreement,
173) -> PendingReloadVerdict {
174 let path = match spawned_from {
175 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
176 Some(spawned_from) => ReloadPathAgreement::Mismatch {
177 configured: configured.to_path_buf(),
178 spawned_from: spawned_from.to_path_buf(),
179 },
180 None => ReloadPathAgreement::Unavailable {
181 reason: if matches!(
182 image,
183 subc_control::RunningImageAgreement::Unavailable {
184 reason: subc_control::RunningImageUnavailableReason::NotRunning
185 }
186 ) {
187 ReloadPathUnavailableReason::NotRunning
188 } else {
189 ReloadPathUnavailableReason::SpawnedPathUnavailable
190 },
191 },
192 };
193 PendingReloadVerdict { path, image }
194}
195
196#[derive(Clone)]
197struct DaemonProvenanceFacts {
198 build: DaemonBuildProvenance,
199 pid: Option<u32>,
200 started_at_ms: Option<u64>,
201 start_clock: Option<crate::clock::StartClock>,
202 executable_path: Option<PathBuf>,
203 executable_identity: Option<SpawnedFileIdentity>,
204 process_start_time: Option<u64>,
205 probe: ExecutableIdentityProbe,
206}
207
208impl Default for DaemonProvenanceFacts {
209 fn default() -> Self {
210 Self {
211 build: DaemonBuildProvenance {
212 build_git_sha: None,
213 build_lock_digest: None,
214 },
215 pid: None,
216 started_at_ms: None,
217 start_clock: None,
218 executable_path: None,
219 executable_identity: None,
220 process_start_time: None,
221 probe: ExecutableIdentityProbe::default(),
222 }
223 }
224}
225
226#[derive(Debug, Clone)]
227struct SupervisorRescanContext {
228 supervisor: Supervisor,
229 config_path: PathBuf,
230 configured_port: Option<u16>,
231 storage_config: Option<crate::daemon_config::StorageConfig>,
232 admission_facts_carrier_module_id: Option<String>,
233 admission_facts_targets: Option<Vec<String>>,
234 scope_authority_owners: Vec<String>,
235}
236
237const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
258 "reloading",
259 "supervisor_not_live",
260 "registration_not_active",
261 "no_forwarding_connection",
262 "relay_send_failed",
263];
264
265#[derive(Clone)]
267pub struct ControlHandler {
268 registry: Arc<Registry>,
269 forwarding: Arc<ForwardingTable>,
270 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
271 supervisor: SupervisorHandle,
272 subc_capabilities: Arc<[String]>,
273 route_bind_relay_timeout: Duration,
277 route_bind_relay_timeouts: BTreeMap<String, Duration>,
281 route_bind_breakers: RouteBindBreakers,
284 route_bind_concurrency: RouteBindConcurrency,
287 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
292 route_bind_breaker_threshold: u32,
294 route_bind_breaker_cooldown: Duration,
296 health_probe_timeout: Duration,
297 storage_config: Option<crate::daemon_config::StorageConfig>,
300 machine_id: Option<crate::machine_id::MachineId>,
304 admission_facts_carrier_module_id: Option<String>,
305 admission_facts_targets: Option<Vec<String>>,
306 scopes: Arc<RwLock<ScopeTable>>,
310 scope_authority_owners: Vec<String>,
313 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
316 rescan: Option<SupervisorRescanContext>,
317 connected_clients: ConnectedClients,
318 counters: DaemonCounters,
319 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
320 daemon_provenance: DaemonProvenanceFacts,
321 #[cfg(test)]
322 control_dispatch_delay: Option<Duration>,
323 #[cfg(test)]
324 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
325}
326
327impl fmt::Debug for ControlHandler {
328 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
329 f.debug_struct("ControlHandler")
330 .field("registry", &self.registry)
331 .field("forwarding", &self.forwarding)
332 .field("process_liveness", &self.process_liveness.is_some())
333 .field("supervisor", &self.supervisor)
334 .field("subc_capabilities", &self.subc_capabilities)
335 .finish()
336 }
337}
338
339struct RouteOpenRequest {
340 target: RouteTarget,
341 identity: BindIdentity,
342 consumer_identity: Option<ConsumerIdentity>,
343 consumer_capabilities: Option<Vec<String>>,
344 admission_facts: Option<serde_json::Value>,
345 scope: Option<ScopeSelector>,
346}
347
348struct RouteBindReservationGuard {
349 forwarding: Arc<ForwardingTable>,
350 endpoint: ModuleEndpointId,
351 relay_corr: u64,
352 armed: bool,
353}
354
355struct ModuleControlRpcGuard {
356 forwarding: Arc<ForwardingTable>,
357 endpoint: ModuleEndpointId,
358 corr: u64,
359 armed: bool,
360}
361
362impl ModuleControlRpcGuard {
363 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
364 Self {
365 forwarding,
366 endpoint,
367 corr,
368 armed: true,
369 }
370 }
371
372 fn disarm(&mut self) {
373 self.armed = false;
374 }
375}
376
377impl Drop for ModuleControlRpcGuard {
378 fn drop(&mut self) {
379 if self.armed {
380 let _ = self
381 .forwarding
382 .cancel_module_control_rpc(self.endpoint, self.corr);
383 }
384 }
385}
386
387impl RouteBindReservationGuard {
388 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
389 Self {
390 forwarding,
391 endpoint,
392 relay_corr,
393 armed: true,
394 }
395 }
396
397 fn release_and_disarm(&mut self) {
398 if !self.armed {
399 return;
400 }
401 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
402 self.endpoint,
403 self.relay_corr,
404 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
405 ) {
406 send_goodbye_target_best_effort(
407 &self.forwarding.counters(),
408 &target,
409 "canceled route.bind",
410 );
411 }
412 self.armed = false;
413 }
414
415 fn disarm(&mut self) {
416 self.armed = false;
417 }
418}
419
420impl Drop for RouteBindReservationGuard {
421 fn drop(&mut self) {
422 self.release_and_disarm();
423 }
424}
425
426#[derive(Debug, Clone, Default)]
456pub(crate) struct RouteBindBreakers {
457 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
458}
459
460#[derive(Debug, Clone, Default)]
461pub(crate) struct RouteBindConcurrency {
462 modules: Arc<Mutex<HashMap<String, usize>>>,
463}
464
465struct RouteBindConcurrencyGuard {
466 concurrency: RouteBindConcurrency,
467 module_id: String,
468}
469
470impl RouteBindConcurrency {
471 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
474 let mut modules = self
475 .modules
476 .lock()
477 .expect("route.bind concurrency mutex poisoned");
478 let in_flight = modules.entry(module_id.to_string()).or_default();
479 if *in_flight >= limit {
480 return Err(*in_flight);
481 }
482 *in_flight += 1;
483 Ok(RouteBindConcurrencyGuard {
484 concurrency: self.clone(),
485 module_id: module_id.to_string(),
486 })
487 }
488}
489
490impl Drop for RouteBindConcurrencyGuard {
491 fn drop(&mut self) {
492 let mut modules = self
493 .concurrency
494 .modules
495 .lock()
496 .expect("route.bind concurrency mutex poisoned");
497 let remove = {
498 let in_flight = modules
499 .get_mut(&self.module_id)
500 .expect("admitted route.bind has a concurrency entry");
501 *in_flight -= 1;
502 *in_flight == 0
503 };
504 if remove {
505 modules.remove(&self.module_id);
506 }
507 }
508}
509
510#[derive(Debug, Default)]
511struct ModuleBreakerState {
512 consecutive_timeouts: u32,
514 cooldown_until: Option<Instant>,
517 probe_in_flight: bool,
522}
523
524enum RouteBindAdmission<'a> {
526 Admitted {
527 guard: RouteBindBreakerGuard<'a>,
528 probe: bool,
531 },
532 Refused {
533 consecutive_timeouts: u32,
534 retry_in: Duration,
538 probe_in_flight: bool,
539 },
540}
541
542struct RouteBindBreakerGuard<'a> {
549 breakers: RouteBindBreakers,
550 module_id: &'a str,
551 settled: bool,
552}
553
554impl RouteBindBreakerGuard<'_> {
555 fn record_accepted(&mut self) -> bool {
559 self.settled = true;
560 self.breakers.record_accepted(self.module_id)
561 }
562
563 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
566 self.settled = true;
567 self.breakers
568 .record_timeout(self.module_id, threshold, cooldown)
569 }
570
571 fn record_inconclusive(&mut self) {
580 self.settled = true;
581 self.breakers.record_inconclusive(self.module_id);
582 }
583}
584
585impl Drop for RouteBindBreakerGuard<'_> {
586 fn drop(&mut self) {
587 if !self.settled {
588 self.breakers.record_inconclusive(self.module_id);
589 }
590 }
591}
592
593struct BreakerOpened {
597 consecutive_timeouts: u32,
598 reopened_after_probe: bool,
601}
602
603impl RouteBindBreakers {
604 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
605 self.modules
606 .lock()
607 .expect("route.bind breaker mutex poisoned")
608 }
609
610 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
613 let admitted = |probe| RouteBindAdmission::Admitted {
614 guard: RouteBindBreakerGuard {
615 breakers: self.clone(),
616 module_id,
617 settled: false,
618 },
619 probe,
620 };
621
622 let mut modules = self.lock();
623 let Some(state) = modules.get_mut(module_id) else {
624 return admitted(false);
625 };
626 let Some(cooldown_until) = state.cooldown_until else {
627 return admitted(false);
628 };
629 if state.probe_in_flight {
630 return RouteBindAdmission::Refused {
631 consecutive_timeouts: state.consecutive_timeouts,
632 retry_in: Duration::ZERO,
633 probe_in_flight: true,
634 };
635 }
636 let now = Instant::now();
637 if now < cooldown_until {
638 return RouteBindAdmission::Refused {
639 consecutive_timeouts: state.consecutive_timeouts,
640 retry_in: cooldown_until - now,
641 probe_in_flight: false,
642 };
643 }
644 state.probe_in_flight = true;
645 admitted(true)
646 }
647
648 fn record_accepted(&self, module_id: &str) -> bool {
649 self.lock()
650 .remove(module_id)
651 .is_some_and(|state| state.cooldown_until.is_some())
652 }
653
654 fn record_timeout(
655 &self,
656 module_id: &str,
657 threshold: u32,
658 cooldown: Duration,
659 ) -> Option<BreakerOpened> {
660 let mut modules = self.lock();
661 let state = modules.entry(module_id.to_string()).or_default();
662 let was_open = state.cooldown_until.is_some();
663 let was_probe = state.probe_in_flight;
664 state.probe_in_flight = false;
665 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
666 if state.consecutive_timeouts < threshold {
667 return None;
668 }
669 state.cooldown_until = Some(Instant::now() + cooldown);
670 Some(BreakerOpened {
671 consecutive_timeouts: state.consecutive_timeouts,
672 reopened_after_probe: was_open && was_probe,
673 })
674 }
675
676 fn record_inconclusive(&self, module_id: &str) {
677 if let Some(state) = self.lock().get_mut(module_id) {
678 state.probe_in_flight = false;
679 }
680 }
681
682 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
697 self.lock()
698 .remove(module_id)
699 .map(|state| state.consecutive_timeouts)
700 .filter(|discarded| *discarded > 0)
701 }
702
703 fn open_snapshot(&self) -> Option<serde_json::Value> {
710 let now = Instant::now();
711 let modules = self.lock();
712 let open = modules
713 .iter()
714 .filter_map(|(module_id, state)| {
715 let cooldown_until = state.cooldown_until?;
716 Some((
717 module_id.clone(),
718 serde_json::json!({
719 "consecutive_timeouts": state.consecutive_timeouts,
720 "cooldown_remaining_ms":
721 cooldown_until.saturating_duration_since(now).as_millis() as u64,
722 "probe_in_flight": state.probe_in_flight,
723 }),
724 ))
725 })
726 .collect::<serde_json::Map<String, serde_json::Value>>();
727 (!open.is_empty()).then_some(serde_json::Value::Object(open))
728 }
729}
730
731impl ControlHandler {
732 pub fn new(registry: Arc<Registry>) -> Self {
733 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
734 }
735
736 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
737 let counters = forwarding.counters();
738 let route_bind_breakers = forwarding.route_bind_breakers();
742 let route_bind_concurrency = forwarding.route_bind_concurrency();
743 let route_outages = forwarding.route_outages();
744 Self {
745 registry,
746 forwarding,
747 process_liveness: None,
748 supervisor: SupervisorHandle::new(),
749 subc_capabilities: Arc::from([
750 CAP_MANIFEST_REGISTRATION.to_string(),
751 CAP_CHANNEL_LIFECYCLE.to_string(),
752 CAP_PING_PONG.to_string(),
753 CAP_SESSION_ATTACH.to_string(),
754 CAP_ADMISSION_FACTS_RELAY.to_string(),
755 CAP_SCOPES_V1.to_string(),
756 ]),
757 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
758 route_bind_relay_timeouts: BTreeMap::new(),
759 route_bind_breakers,
760 route_bind_concurrency,
761 route_outages,
762 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
763 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
764 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
765 storage_config: None,
766 machine_id: None,
767 admission_facts_carrier_module_id: None,
768 admission_facts_targets: None,
769 scopes: Arc::new(RwLock::new(ScopeTable::new(
770 crate::daemon_config::default_scope_authority_owners(),
771 ))),
772 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
773 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
774 rescan: None,
775 connected_clients: ConnectedClients::new(),
776 counters,
777 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
778 daemon_provenance: DaemonProvenanceFacts::default(),
779 #[cfg(test)]
780 control_dispatch_delay: None,
781 #[cfg(test)]
782 provenance_probe_override: None,
783 }
784 }
785
786 pub fn with_storage_config(
789 mut self,
790 storage_config: Option<crate::daemon_config::StorageConfig>,
791 ) -> Self {
792 self.storage_config = storage_config;
793 self
794 }
795
796 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
799 self.machine_id = machine_id;
800 self
801 }
802
803 pub fn with_admission_facts_config(
807 mut self,
808 carrier_module_id: Option<String>,
809 targets: Option<Vec<String>>,
810 ) -> Self {
811 self.admission_facts_carrier_module_id = carrier_module_id;
812 self.admission_facts_targets = targets;
813 self
814 }
815
816 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
820 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
821 self.scope_authority_owners = owners;
822 self
823 }
824
825 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
828 self.route_bind_relay_timeout = timeout;
829 self
830 }
831
832 pub fn with_route_bind_relay_timeouts(
838 mut self,
839 timeouts: impl IntoIterator<Item = (String, Duration)>,
840 ) -> Self {
841 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
842 self
843 }
844
845 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
851 self.route_bind_relay_timeouts
852 .get(module_id)
853 .copied()
854 .unwrap_or(self.route_bind_relay_timeout)
855 }
856
857 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
865 self.route_bind_breaker_threshold = threshold.max(1);
866 self.route_bind_breaker_cooldown = cooldown;
867 self
868 }
869
870 #[cfg(test)]
871 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
872 self.health_probe_timeout = timeout;
873 self
874 }
875
876 #[cfg(test)]
877 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
878 self.control_dispatch_delay = Some(delay);
879 self
880 }
881
882 pub fn with_process_liveness(
883 mut self,
884 process_liveness: Arc<dyn ModuleProcessLiveness>,
885 ) -> Self {
886 self.process_liveness = Some(process_liveness);
887 self
888 }
889
890 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
891 self.supervisor = supervisor;
892 self
893 }
894
895 pub fn with_daemon_provenance(
896 mut self,
897 pid: u32,
898 started_at_ms: u64,
899 executable_path: Option<PathBuf>,
900 build_git_sha: Option<String>,
901 build_lock_digest: Option<String>,
902 ) -> Self {
903 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
904 let process_start_time = process_start_time(pid);
905 self.daemon_provenance = DaemonProvenanceFacts {
906 build: DaemonBuildProvenance {
907 build_git_sha,
908 build_lock_digest,
909 },
910 pid: Some(pid),
911 started_at_ms: Some(started_at_ms),
912 start_clock: None,
913 executable_path,
914 executable_identity,
915 process_start_time,
916 probe: ExecutableIdentityProbe::default(),
917 };
918 self
919 }
920
921 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
922 self.daemon_provenance.start_clock = Some(clock);
923 self
924 }
925
926 #[cfg(test)]
927 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
928 self.provenance_probe_override = Some(result);
929 self
930 }
931
932 pub fn with_capability_config(
936 self,
937 modules: impl IntoIterator<Item = (String, bool)>,
938 reserved_capabilities: BTreeMap<String, String>,
939 ) -> Self {
940 self.capability_evaluator
941 .configure(modules, reserved_capabilities);
942 self
943 }
944
945 pub fn with_supervisor_rescan(
946 mut self,
947 supervisor: Supervisor,
948 config_path: impl Into<PathBuf>,
949 configured_port: Option<u16>,
950 ) -> Self {
951 self.rescan = Some(SupervisorRescanContext {
952 supervisor,
953 config_path: config_path.into(),
954 configured_port,
955 storage_config: self.storage_config.clone(),
956 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
957 admission_facts_targets: self.admission_facts_targets.clone(),
958 scope_authority_owners: self.scope_authority_owners.clone(),
959 });
960 self
961 }
962
963 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
964 self.connected_clients = connected_clients;
965 self
966 }
967
968 pub fn forwarding(&self) -> Arc<ForwardingTable> {
969 Arc::clone(&self.forwarding)
970 }
971
972 pub(crate) fn counters(&self) -> DaemonCounters {
973 self.counters.clone()
974 }
975
976 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
979 tokio::spawn(async move {
980 loop {
981 self.capability_evaluator
982 .wait_for_change_or_deadline()
983 .await;
984 self.refresh_capability_requirements();
985 }
986 });
987 }
988
989 fn runtime_capability_snapshot(
990 &self,
991 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
992 let runtime = self
993 .supervisor
994 .list()
995 .into_iter()
996 .map(|module| {
997 let status = module.status().map_err(|err| {
998 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
999 })?;
1000 Ok(RuntimeModule {
1001 module_id: status.module_id,
1002 state: status.state,
1003 enabled: status.enabled,
1004 })
1005 })
1006 .collect::<Result<Vec<_>, RouterError>>()?;
1007 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1008 RouterError::backend(
1009 0,
1010 0,
1011 format!("failed to list capability registrations: {err}"),
1012 )
1013 })?;
1014 let registrations = registrations
1015 .into_iter()
1016 .map(|registration| RegisteredModule {
1017 module_id: registration.manifest.module_id,
1018 module_version: registration.manifest.module_version,
1019 capabilities: registration.manifest.capabilities,
1020 })
1021 .collect();
1022 Ok((runtime, registrations))
1023 }
1024
1025 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1032 let cached_registration = RegisteredModule {
1033 module_id: registration.manifest.module_id.clone(),
1034 module_version: registration.manifest.module_version.clone(),
1035 capabilities: registration.manifest.capabilities.clone(),
1036 };
1037 if self.capability_evaluator.record_hello(&cached_registration) {
1038 warn!(
1039 module_id = %cached_registration.module_id,
1040 "capability claims drifted from the cached manifest"
1041 );
1042 }
1043 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1044 self.enforce_capability_denies();
1045 }
1046 self.refresh_capability_requirements();
1047 }
1048
1049 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1053 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1054 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1055 self.supervisor.set_swap_promotion_observer(observer);
1056 }
1057
1058 pub fn refresh_capability_requirements(&self) {
1059 match self.runtime_capability_snapshot() {
1060 Ok((runtime, registrations)) => {
1061 log_requirement_events(
1062 self.capability_evaluator
1063 .evaluate_now(&runtime, ®istrations),
1064 );
1065 }
1066 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1067 }
1068 }
1069
1070 fn enforce_capability_denies(&self) {
1074 let (_, registrations) = match self.registry.list_modules() {
1075 Ok(snapshot) => snapshot,
1076 Err(err) => {
1077 warn!(error = %err, "failed to read registrations for capability deny census");
1078 return;
1079 }
1080 };
1081 let manifests = registrations
1082 .into_iter()
1083 .map(|registration| {
1084 (
1085 registration.manifest.module_id.clone(),
1086 registration.manifest,
1087 )
1088 })
1089 .collect::<BTreeMap<_, _>>();
1090 let census = match self.forwarding.route_census(None) {
1091 Ok(census) => census,
1092 Err(err) => {
1093 warn!(error = %err, "failed to read route census for capability deny enforcement");
1094 return;
1095 }
1096 };
1097
1098 for (target_module_id, routes) in census {
1099 let Some(target_manifest) = manifests.get(&target_module_id) else {
1100 continue;
1101 };
1102 let mut closed_routes = Vec::new();
1103 let mut module_goodbyes = Vec::new();
1104 for route in routes {
1105 let Principal::Reserved {
1106 module_id: opening_module_id,
1107 } = &route.principal
1108 else {
1109 continue;
1110 };
1111 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1112 continue;
1113 };
1114 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1115 continue;
1116 };
1117
1118 match self.forwarding.release_client_route(
1119 route.goodbye_target.connection_id,
1120 route.goodbye_target.channel,
1121 route.goodbye_target.epoch,
1122 ) {
1123 Ok(RouteRelease::Removed(module_goodbye)) => {
1124 warn!(
1125 opening_module_id,
1126 target_module_id,
1127 capability,
1128 "force-closing route because an attested capability deny edge now matches"
1129 );
1130 closed_routes.push(route);
1131 module_goodbyes.push(module_goodbye);
1132 }
1133 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1134 Err(err) => warn!(
1135 opening_module_id,
1136 target_module_id,
1137 capability,
1138 error = %err,
1139 "failed to force-close capability-denied route"
1140 ),
1141 }
1142 }
1143
1144 if closed_routes.is_empty() {
1145 continue;
1146 }
1147 send_route_control_pushes(
1148 &self.forwarding,
1149 closed_routes,
1150 ClientControlPush::RouteClosed {
1151 module_id: target_module_id,
1152 reason: RouteCloseReason::CapabilityDenied,
1153 drained: false,
1154 abandoned: 0,
1155 excluded_subscriptions: 0,
1156 terminal: Some(false),
1157 },
1158 );
1159 self.emit_route_goodbyes(module_goodbyes);
1160 }
1161 }
1162
1163 fn not_ready_reason(
1169 &self,
1170 registration: &crate::registry::ModuleRegistration,
1171 ) -> Option<NotReadyReason> {
1172 if !registration.ready {
1173 return Some(NotReadyReason {
1174 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1175 capability: None,
1176 });
1177 }
1178 self.first_unprovided_required_capability(registration)
1179 .map(|capability| NotReadyReason {
1180 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1181 capability: Some(capability),
1182 })
1183 }
1184
1185 fn first_unprovided_required_capability(
1203 &self,
1204 registration: &crate::registry::ModuleRegistration,
1205 ) -> Option<String> {
1206 let required = registration
1207 .manifest
1208 .capabilities
1209 .iter()
1210 .flat_map(|declarations| declarations.requires.iter())
1211 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1212 .map(|requirement| requirement.capability.as_str())
1213 .collect::<BTreeSet<_>>();
1214 if required.is_empty() {
1215 return None;
1216 }
1217 let module_id = registration.manifest.module_id.as_str();
1218 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1219 if required
1220 .iter()
1221 .any(|capability| verdict(capability).is_none())
1222 {
1223 self.refresh_capability_requirements();
1224 }
1225 required
1226 .into_iter()
1227 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1228 .map(str::to_string)
1229 }
1230
1231 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1232 self.capability_evaluator
1233 .statuses()
1234 .into_iter()
1235 .map(capability_requirement_status)
1236 .collect()
1237 }
1238
1239 fn deregister_connection(
1246 &self,
1247 connection_id: ConnectionId,
1248 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1249 self.registry.deregister_connection(connection_id)
1250 }
1251
1252 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1253 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1254 return None;
1255 }
1256 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1257 parse_client_control_request(&frame.body)
1258 else {
1259 return None;
1260 };
1261 Some(target_module_id(&target).to_string())
1262 }
1263
1264 pub(crate) fn route_open_capacity_refusal(
1265 &self,
1266 ctx: &RouteCtx,
1267 frame: &Frame,
1268 target_module_id: &str,
1269 in_flight: usize,
1270 limit: usize,
1271 ) -> Result<Frame, RouterError> {
1272 self.route_open_admission_refusal_frame(
1273 ctx,
1274 frame,
1275 target_module_id,
1276 "open_admission_full",
1277 (in_flight, limit),
1278 format!(
1279 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1280 ),
1281 )
1282 }
1283
1284 fn route_open_target_capacity_refusal(
1285 &self,
1286 ctx: &RouteCtx,
1287 frame: &Frame,
1288 target_module_id: &str,
1289 in_flight: usize,
1290 ) -> Result<Frame, RouterError> {
1291 self.route_open_admission_refusal_frame(
1292 ctx,
1293 frame,
1294 target_module_id,
1295 "target_binds_full",
1296 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1297 format!(
1298 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1299 ),
1300 )
1301 }
1302
1303 fn route_open_admission_refusal_frame(
1311 &self,
1312 ctx: &RouteCtx,
1313 frame: &Frame,
1314 target_module_id: &str,
1315 reason: &'static str,
1316 (in_flight, limit): (usize, usize),
1317 message: impl Into<String>,
1318 ) -> Result<Frame, RouterError> {
1319 let code = error_codes::TARGET_UNAVAILABLE;
1320 self.counters.increment_route_open_refused(code);
1321 info!(
1322 target: "control",
1323 code,
1324 reason,
1325 module_id = ?target_module_id,
1326 connection_id = ctx.connection_id.get(),
1327 in_flight,
1328 limit,
1329 "route.open refused"
1330 );
1331 control_error_frame(frame, code, message.into())
1332 }
1333
1334 #[cfg(test)]
1340 pub fn handle_control(
1341 &self,
1342 connection_id: ConnectionId,
1343 frame: Frame,
1344 ) -> Result<Vec<Frame>, RouterError> {
1345 match frame.header.ty {
1346 FrameType::Ping => Ok(vec![pong(&frame)?]),
1347 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1348 FrameType::Goodbye => self.handle_goodbye(connection_id),
1349 ty => Ok(vec![control_error_frame(
1350 &frame,
1351 "unsupported_control_frame",
1352 format!("unsupported channel-0 frame {ty:?}"),
1353 )?]),
1354 }
1355 }
1356
1357 pub async fn handle_control_frame(
1358 &self,
1359 ctx: &RouteCtx,
1360 frame: Frame,
1361 ) -> Result<Vec<Frame>, RouterError> {
1362 self.handle_control_frame_timed(ctx, frame, None).await
1363 }
1364
1365 pub(crate) async fn handle_control_frame_timed(
1366 &self,
1367 ctx: &RouteCtx,
1368 frame: Frame,
1369 dispatch_started_at: Option<StdInstant>,
1370 ) -> Result<Vec<Frame>, RouterError> {
1371 match frame.header.ty {
1372 FrameType::Ping => Ok(vec![pong(&frame)?]),
1373 FrameType::Hello => {
1374 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1375 }
1376 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1377 FrameType::Cancel => {
1378 if self
1379 .supervisor
1380 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1381 {
1382 Ok(Vec::new())
1383 } else {
1384 Ok(vec![control_error_frame(
1385 &frame,
1386 "unknown_subscription",
1387 "no supervisor spawn subscription has this correlation id",
1388 )?])
1389 }
1390 }
1391 FrameType::Request => {
1392 if self
1393 .forwarding
1394 .module_endpoint_for_connection(ctx.connection_id)
1395 .map_err(RouterError::Forwarding)?
1396 .is_some()
1397 {
1398 if !is_known_module_request_op(&frame.body) {
1399 return Ok(vec![control_error_frame(
1400 &frame,
1401 "unsupported_control_frame",
1402 "module-originated channel-0 REQUEST is not supported",
1403 )?]);
1404 }
1405 let request = match parse_module_control_request_from_module(&frame.body) {
1406 Ok(request) => request,
1407 Err((err, ControlRequestBodyError::UnknownOp)) => {
1408 return Ok(vec![control_error_frame(
1409 &frame,
1410 "unsupported_control_frame",
1411 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1412 )?])
1413 }
1414 Err((err, ControlRequestBodyError::InvalidBody)) => {
1415 return Ok(vec![control_error_frame(
1416 &frame,
1417 "invalid_control_body",
1418 format!("malformed module control body: {err}"),
1419 )?])
1420 }
1421 };
1422 let op = module_control_request_op(&request);
1423 let corr = frame.header.corr;
1424 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1425 let result =
1426 self.handle_module_control_request(ctx.connection_id, frame, request);
1427 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1428 return result;
1429 }
1430
1431 if is_known_module_request_op(&frame.body) {
1432 return Ok(vec![control_error_frame(
1433 &frame,
1434 "not_registered",
1435 "catalog.update requires an active module registration owned by this connection",
1436 )?]);
1437 }
1438
1439 let request = match parse_client_control_request(&frame.body) {
1440 Ok(request) => request,
1441 Err((err, ControlRequestBodyError::UnknownOp)) => {
1442 return Ok(vec![control_error_frame(
1443 &frame,
1444 "unknown_control_op",
1445 format!("unknown client control op: {err}"),
1446 )?])
1447 }
1448 Err((err, ControlRequestBodyError::InvalidBody)) => {
1449 return Ok(vec![control_error_frame(
1450 &frame,
1451 "invalid_control_body",
1452 format!("malformed client control body: {err}"),
1453 )?])
1454 }
1455 };
1456 let op = client_control_request_op(&request);
1457 let corr = frame.header.corr;
1458 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1459 #[cfg(test)]
1460 if let Some(delay) = self.control_dispatch_delay {
1461 tokio::time::sleep(delay).await;
1462 }
1463 let result = self
1464 .handle_client_control_request(ctx, frame, request)
1465 .await;
1466 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1467 result
1468 }
1469 FrameType::Push => {
1470 let Some(endpoint) = self
1471 .forwarding
1472 .module_endpoint_for_connection(ctx.connection_id)
1473 .map_err(RouterError::Forwarding)?
1474 else {
1475 return Ok(vec![control_error_frame(
1476 &frame,
1477 "unsupported_control_frame",
1478 "client-originated channel-0 PUSH is not supported",
1479 )?]);
1480 };
1481 self.handle_status_update(endpoint, frame)
1482 }
1483 FrameType::Response | FrameType::Error
1484 if self
1485 .forwarding
1486 .module_endpoint_for_connection(ctx.connection_id)
1487 .map_err(RouterError::Forwarding)?
1488 .is_some() =>
1489 {
1490 self.handle_module_relay_response(ctx.connection_id, frame)
1491 }
1492 ty => Ok(vec![control_error_frame(
1493 &frame,
1494 "unsupported_control_frame",
1495 format!("unsupported channel-0 frame {ty:?}"),
1496 )?]),
1497 }
1498 }
1499
1500 pub fn cleanup_connection(
1501 &self,
1502 connection_id: ConnectionId,
1503 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1504 let crash_closed = self
1505 .registry
1506 .get_module_by_connection(connection_id)?
1507 .and_then(|registration| {
1508 self.forwarding
1509 .module_endpoint_for_connection(connection_id)
1510 .ok()
1511 .flatten()
1512 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1513 .map(|routes| (registration.manifest.module_id, routes))
1514 });
1515 let crash_closed = crash_closed.map(|(module_id, routes)| {
1516 let terminal = match self.supervisor.get(&module_id) {
1517 None => false,
1518 Some(module) => match module.will_recover_after_connection_loss() {
1519 Ok(will_recover) => !will_recover,
1520 Err(err) => {
1521 warn!(
1522 %module_id,
1523 error = %err,
1524 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1525 );
1526 false
1527 }
1528 },
1529 };
1530 let reason = match self.forwarding.is_daemon_draining() {
1534 Ok(true) => RouteCloseReason::Restart,
1535 Ok(false) => RouteCloseReason::Crash,
1536 Err(err) => {
1537 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1538 RouteCloseReason::Crash
1539 }
1540 };
1541 (module_id, routes, reason, terminal)
1542 });
1543 let registrations = self.deregister_connection(connection_id);
1544 let cleanup = self.forwarding.cleanup_connection_counted(connection_id);
1545 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1550 let abandoned = cleanup
1551 .as_ref()
1552 .map_or(0, |cleanup| cleanup.abandoned_relays);
1553 send_route_control_pushes(
1554 &self.forwarding,
1555 routes,
1556 ClientControlPush::RouteClosed {
1557 module_id,
1558 reason,
1559 drained: false,
1560 abandoned,
1561 excluded_subscriptions: 0,
1562 terminal: Some(terminal),
1563 },
1564 );
1565 }
1566 if let Ok(cleanup) = cleanup {
1567 self.emit_route_goodbyes(cleanup.released);
1568 }
1569 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1573 crate::supervise::notify_registration_release();
1574 self.capability_evaluator.wake_deadline_loop();
1575 self.refresh_capability_requirements();
1576 }
1577 self.supervisor.remove_spawn_subscribers(connection_id);
1578 self.hello_launch_nonces
1581 .lock()
1582 .unwrap_or_else(|poisoned| poisoned.into_inner())
1583 .forget(connection_id);
1584 self.scopes
1585 .write()
1586 .unwrap_or_else(|poisoned| poisoned.into_inner())
1587 .release_connection(connection_id);
1588 registrations
1589 }
1590
1591 pub(crate) fn handle_route_goodbye(
1592 &self,
1593 connection_id: ConnectionId,
1594 route_channel: u16,
1595 route_epoch: u32,
1596 ) -> Result<bool, RouterError> {
1597 debug!(
1598 connection_id = connection_id.get(),
1599 route_channel, route_epoch, "handling route GOODBYE"
1600 );
1601 let RouteRelease::Removed(released_route) = self
1602 .forwarding
1603 .release_client_route(connection_id, route_channel, route_epoch)
1604 .map_err(RouterError::Forwarding)?
1605 else {
1606 return Ok(false);
1607 };
1608 self.emit_route_goodbyes(vec![released_route]);
1609 Ok(true)
1610 }
1611
1612 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1613 for released in released_routes {
1614 let frame = match Frame::build_with_version(
1615 released.negotiated_ver,
1616 FrameType::Goodbye,
1617 control_flags(),
1618 released.channel,
1619 released.epoch,
1620 0,
1621 Vec::new(),
1622 ) {
1623 Ok(frame) => frame,
1624 Err(err) => {
1625 warn!(
1626 route_channel = released.channel,
1627 error = %err,
1628 "failed to build route GOODBYE frame"
1629 );
1630 continue;
1631 }
1632 };
1633 if !released.close_on_delivery_failure() {
1634 crate::forwarding::send_module_route_goodbye(
1635 &self.counters,
1636 &released.sink,
1637 frame,
1638 released.module_id.as_deref(),
1639 "client route released",
1640 );
1641 continue;
1642 }
1643 if let Err(err) = released.sink.try_send(frame) {
1644 warn!(
1645 target_connection_id = released.connection_id.get(),
1646 route_channel = released.channel,
1647 error = %err,
1648 "route GOODBYE was not delivered to client; closing target connection"
1649 );
1650 if self
1651 .forwarding
1652 .escalate_client_delivery_failure(
1653 released.connection_id,
1654 released.channel,
1655 released.epoch,
1656 CloseReason::new(
1657 "route_goodbye_delivery_failed",
1658 format!(
1659 "failed to enqueue route GOODBYE for channel {}: {err}",
1660 released.channel
1661 ),
1662 ),
1663 crate::forwarding::UndeliveredFrame {
1664 module_id: released.module_id.as_deref(),
1665 sink: &released.sink,
1666 },
1667 )
1668 .unwrap_or(false)
1669 {
1670 self.counters.increment_goodbye_relay_client_failed();
1671 }
1672 }
1673 }
1674 }
1675
1676 fn send_abandoned_route_bind_goodbye(
1688 &self,
1689 module_sink: &crate::FrameSink,
1690 negotiated_ver: u8,
1691 module_channel: u16,
1692 module_epoch: u32,
1693 ) {
1694 let frame = match Frame::build_with_version(
1695 negotiated_ver,
1696 FrameType::Goodbye,
1697 control_flags(),
1698 module_channel,
1699 module_epoch,
1700 0,
1701 Vec::new(),
1702 ) {
1703 Ok(frame) => frame,
1704 Err(err) => {
1705 warn!(
1706 route_channel = module_channel,
1707 error = %err,
1708 "failed to build GOODBYE for abandoned route.bind"
1709 );
1710 return;
1711 }
1712 };
1713 crate::forwarding::send_module_route_goodbye(
1714 &self.counters,
1715 module_sink,
1716 frame,
1717 None,
1718 "abandoned route.bind",
1719 );
1720 }
1721
1722 fn handle_hello(
1723 &self,
1724 connection_id: ConnectionId,
1725 sink: Option<crate::FrameSink>,
1726 frame: Frame,
1727 ) -> Result<Vec<Frame>, RouterError> {
1728 debug!(
1729 connection_id = connection_id.get(),
1730 corr = frame.header.corr,
1731 "handling HELLO"
1732 );
1733 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1734 Ok(value) => value,
1735 Err(err) => {
1736 return Ok(vec![control_error_frame(
1737 &frame,
1738 "invalid_hello",
1739 format!("malformed HELLO body: {err}"),
1740 )?])
1741 }
1742 };
1743 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1744 return Ok(vec![control_error_frame(
1745 &frame,
1746 "invalid_capability_grammar",
1747 err.to_string(),
1748 )?]);
1749 }
1750 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1751 return Ok(vec![control_error_frame(
1752 &frame,
1753 "invalid_manifest",
1754 err.to_string(),
1755 )?]);
1756 }
1757 if let Some(provenance) = hello_value
1758 .get("manifest")
1759 .and_then(|manifest| manifest.get("provenance"))
1760 {
1761 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1762 return Ok(vec![control_error_frame(
1763 &frame,
1764 "invalid_manifest",
1765 format!("malformed manifest provenance: {err}"),
1766 )?]);
1767 }
1768 }
1769 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1770 Ok(hello) => hello,
1771 Err(err) => {
1772 return Ok(vec![control_error_frame(
1773 &frame,
1774 "invalid_hello",
1775 format!("malformed HELLO body: {err}"),
1776 )?])
1777 }
1778 };
1779
1780 if hello.protocol_ver != hello.manifest.protocol_ver {
1781 return Ok(vec![control_error_frame(
1782 &frame,
1783 "invalid_manifest",
1784 format!(
1785 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1786 hello.protocol_ver, hello.manifest.protocol_ver
1787 ),
1788 )?]);
1789 }
1790
1791 if hello.manifest.module_id.trim().is_empty() {
1792 return Ok(vec![control_error_frame(
1793 &frame,
1794 "invalid_manifest",
1795 "manifest module_id must not be empty",
1796 )?]);
1797 }
1798
1799 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1800 Ok(negotiated_ver) => negotiated_ver,
1801 Err(message) => {
1802 return Ok(vec![control_error_frame(
1803 &frame,
1804 "version_unsupported",
1805 message,
1806 )?])
1807 }
1808 };
1809
1810 let swap_admission = self
1818 .supervisor
1819 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1820 if swap_admission == SwapHelloAdmission::Refused {
1821 warn!(
1822 module_id = %hello.manifest.module_id,
1823 connection_id = connection_id.get(),
1824 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1825 );
1826 return Ok(vec![control_error_frame(
1827 &frame,
1828 "swap_token_invalid",
1829 format!(
1830 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1831 hello.manifest.module_id
1832 ),
1833 )?]);
1834 }
1835 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1836
1837 if let Some(rejection) = (!swap_candidate)
1845 .then(|| {
1846 self.supervisor.reserved_hello_rejection(
1847 &hello.manifest.module_id,
1848 hello.launch_nonce.as_deref(),
1849 )
1850 })
1851 .flatten()
1852 {
1853 let message = match rejection {
1854 ReservedHelloRejection::Exact { module_id } => format!(
1855 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1856 ),
1857 ReservedHelloRejection::Prefix {
1858 prefix,
1859 owner_module_id,
1860 } => format!(
1861 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1862 hello.manifest.module_id
1863 ),
1864 };
1865 return Ok(vec![control_error_frame(
1866 &frame,
1867 "reserved_module",
1868 message,
1869 )?]);
1870 }
1871
1872 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1873 &hello.manifest.module_id,
1874 hello.manifest.capabilities.as_ref(),
1875 );
1876 if let Some(refusal) = reserved_capability_refusals.first() {
1877 let capability = refusal.capability.clone();
1878 let bound_module = refusal.claimants[0].clone();
1879 log_duplicate_claim_events(reserved_capability_refusals);
1880 return Ok(vec![control_error_frame(
1881 &frame,
1882 "reserved_capability",
1883 format!(
1884 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1885 capability, bound_module, hello.manifest.module_id
1886 ),
1887 )?]);
1888 }
1889
1890 if self
1893 .forwarding
1894 .connection_has_client_routes(connection_id)
1895 .map_err(RouterError::Forwarding)?
1896 {
1897 return Ok(vec![control_error_frame(
1898 &frame,
1899 "invalid_hello",
1900 "connection has open client routes and cannot also register as a module",
1901 )?]);
1902 }
1903
1904 self.hello_launch_nonces
1909 .lock()
1910 .unwrap_or_else(|poisoned| poisoned.into_inner())
1911 .record(connection_id, hello.launch_nonce.as_deref());
1912 let control_ops = effective_module_control_ops(hello.control_ops);
1913 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1916 if swap_candidate {
1917 return self.register_swap_candidate(
1918 connection_id,
1919 sink,
1920 &frame,
1921 hello.manifest,
1922 negotiated_ver,
1923 control_ops,
1924 hello_ack,
1925 );
1926 }
1927 let registration = match self.registry.register_with_control_ops(
1928 hello.manifest,
1929 negotiated_ver,
1930 connection_id,
1931 control_ops,
1932 ) {
1933 Ok(registration) => registration,
1934 Err(RegistryError::DuplicateModuleId { module_id }) => {
1935 return Ok(vec![control_error_frame(
1936 &frame,
1937 "duplicate_module_id",
1938 format!(
1939 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1940 ),
1941 )?])
1942 }
1943 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1944 return Ok(vec![control_error_frame(
1945 &frame,
1946 "invalid_module_id",
1947 err.to_string(),
1948 )?])
1949 }
1950 Err(err) => {
1951 return Ok(vec![control_error_frame(
1952 &frame,
1953 "registry_error",
1954 err.to_string(),
1955 )?])
1956 }
1957 };
1958
1959 let reply = if let Some(sink) = sink {
1960 let concurrency = manifest_concurrency(®istration.manifest);
1976 if let Err(err) = self.forwarding.register_module_connection_acked(
1977 connection_id,
1978 registration.manifest.module_id.clone(),
1979 negotiated_ver,
1980 concurrency,
1981 sink,
1982 hello_ack,
1983 ) {
1984 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1988 crate::supervise::notify_registration_release();
1989 }
1990 return Ok(vec![control_error_frame(
1991 &frame,
1992 forwarding_error_code(&err),
1993 err.to_string(),
1994 )?]);
1995 }
1996 Vec::new()
1997 } else {
1998 vec![hello_ack]
2001 };
2002
2003 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2012 info!(
2013 module_id = %registration.manifest.module_id,
2014 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2015 );
2016 }
2017
2018 self.apply_registration_capabilities(®istration);
2019
2020 info!(
2021 module_id = %registration.manifest.module_id,
2022 module_version = %registration.manifest.module_version,
2023 negotiated_ver,
2024 routable_provider = manifest_provides_routable_role(®istration.manifest),
2025 connection_id = connection_id.get(),
2026 "module registered"
2027 );
2028
2029 Ok(reply)
2030 }
2031
2032 #[allow(clippy::too_many_arguments)]
2042 fn register_swap_candidate(
2043 &self,
2044 connection_id: ConnectionId,
2045 sink: Option<crate::FrameSink>,
2046 frame: &Frame,
2047 manifest: ModuleManifest,
2048 negotiated_ver: u8,
2049 control_ops: Vec<String>,
2050 hello_ack: Frame,
2051 ) -> Result<Vec<Frame>, RouterError> {
2052 let module_id = manifest.module_id.clone();
2053 let registration = match self.registry.register_candidate_with_control_ops(
2054 manifest,
2055 negotiated_ver,
2056 connection_id,
2057 control_ops,
2058 ) {
2059 Ok(registration) => registration,
2060 Err(RegistryError::DuplicateModuleId { module_id }) => {
2061 return Ok(vec![control_error_frame(
2062 frame,
2063 "duplicate_module_id",
2064 format!(
2065 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2066 ),
2067 )?])
2068 }
2069 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2070 return Ok(vec![control_error_frame(
2071 frame,
2072 "invalid_module_id",
2073 err.to_string(),
2074 )?])
2075 }
2076 Err(err) => {
2077 return Ok(vec![control_error_frame(
2078 frame,
2079 "registry_error",
2080 err.to_string(),
2081 )?])
2082 }
2083 };
2084 let reply = if let Some(sink) = sink {
2085 let concurrency = manifest_concurrency(®istration.manifest);
2089 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2090 connection_id,
2091 module_id.clone(),
2092 negotiated_ver,
2093 concurrency,
2094 sink,
2095 hello_ack,
2096 ) {
2097 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2098 crate::supervise::notify_registration_release();
2099 }
2100 return Ok(vec![control_error_frame(
2101 frame,
2102 forwarding_error_code(&err),
2103 err.to_string(),
2104 )?]);
2105 }
2106 Vec::new()
2107 } else {
2108 vec![hello_ack]
2109 };
2110 self.supervisor.mark_swap_candidate_admitted(&module_id);
2111 info!(
2112 module_id = %module_id,
2113 module_version = %registration.manifest.module_version,
2114 negotiated_ver,
2115 ready = registration.ready,
2116 connection_id = connection_id.get(),
2117 "swap candidate registered; not routable until cutover"
2118 );
2119 Ok(reply)
2120 }
2121
2122 fn build_hello_ack(
2123 &self,
2124 frame: &Frame,
2125 negotiated_ver: u8,
2126 module_id: &str,
2127 ) -> Result<Frame, RouterError> {
2128 let ack = ModuleHelloAckBody {
2129 negotiated_ver,
2130 subc_ops: module_subc_ops(),
2131 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2132 storage: self
2133 .storage_config
2134 .as_ref()
2135 .map(|cfg| cfg.descriptor_for(module_id)),
2136 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2137 };
2138 let body = serde_json::to_vec(&ack).map_err(|err| {
2139 RouterError::backend(
2140 0,
2141 frame.header.corr,
2142 format!("failed to encode HELLO_ACK: {err}"),
2143 )
2144 })?;
2145
2146 Frame::build_with_version(
2147 negotiated_ver,
2148 FrameType::HelloAck,
2149 control_flags(),
2150 0,
2151 0,
2152 frame.header.corr,
2153 body,
2154 )
2155 .map_err(RouterError::FrameBuild)
2156 }
2157
2158 async fn handle_client_control_request(
2159 &self,
2160 ctx: &RouteCtx,
2161 frame: Frame,
2162 request: ClientControlRequest,
2163 ) -> Result<Vec<Frame>, RouterError> {
2164 match request {
2165 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2166 ClientControlRequest::CatalogList { module_id } => {
2167 self.handle_catalog_list(frame, module_id)
2168 }
2169 ClientControlRequest::RouteOpen {
2170 target,
2171 identity,
2172 consumer_identity,
2173 consumer_capabilities,
2174 admission_facts,
2175 scope,
2176 } => {
2177 self.handle_route_open(
2178 ctx,
2179 frame,
2180 RouteOpenRequest {
2181 target,
2182 identity,
2183 consumer_identity,
2184 consumer_capabilities,
2185 admission_facts,
2186 scope,
2187 },
2188 )
2189 .await
2190 }
2191 ClientControlRequest::RoutePoll {
2192 route_channel,
2193 route_epoch,
2194 kind,
2195 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2196 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2197 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2198 self.handle_supervisor_spawn_snapshot(frame)
2199 }
2200 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2201 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2202 }
2203 ClientControlRequest::SupervisorRestart {
2204 module_id,
2205 drain_timeout_ms,
2206 } => {
2207 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2208 .await
2209 }
2210 ClientControlRequest::SupervisorSwap {
2211 module_id,
2212 ready_timeout_ms,
2213 } => {
2214 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2215 .await
2216 }
2217 ClientControlRequest::SupervisorReload { module_id } => {
2218 self.handle_supervisor_reload(frame, module_id).await
2219 }
2220 ClientControlRequest::SupervisorRescan { preview } => {
2221 self.handle_supervisor_rescan(frame, preview).await
2222 }
2223 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2224 self.handle_supervisor_release_reserved(frame, module_id)
2225 .await
2226 }
2227 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2228 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2229 .await
2230 }
2231 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2232 self.handle_supervisor_health_probe(frame, module_id).await
2233 }
2234 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2235 ClientControlRequest::SupervisorRoutes { module_id } => {
2236 self.handle_supervisor_routes(frame, module_id)
2237 }
2238 ClientControlRequest::SupervisorProvenance { module_id } => {
2239 self.handle_supervisor_provenance(frame, module_id).await
2240 }
2241 ClientControlRequest::SupervisorStderrTail {
2242 module_id,
2243 max_lines,
2244 max_bytes,
2245 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2246 ClientControlRequest::SupervisorTerminals { module_id } => {
2247 self.handle_supervisor_terminals(frame, module_id).await
2248 }
2249 }
2250 }
2251
2252 fn handle_module_control_request(
2253 &self,
2254 connection_id: ConnectionId,
2255 frame: Frame,
2256 request: ModuleControlRequestFromModule,
2257 ) -> Result<Vec<Frame>, RouterError> {
2258 match request {
2259 ModuleControlRequestFromModule::CatalogUpdate {
2260 provides,
2261 capabilities,
2262 ready,
2263 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2264 ModuleControlRequestFromModule::LiveRoots {} => {
2265 let registered = self
2266 .registry
2267 .get_module_by_connection(connection_id)
2268 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2269 let Some(registration) = registered else {
2270 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2271 };
2272 let response = self
2273 .forwarding
2274 .live_roots(®istration.manifest.module_id)
2275 .map_err(RouterError::Forwarding)?;
2276 Ok(vec![control_response_body_frame(
2277 &frame,
2278 &response,
2279 "ModuleControlResponseToModule::LiveRoots",
2280 )?])
2281 }
2282 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2283 self.handle_scope_sync(connection_id, frame, generation, scopes)
2284 }
2285 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2286 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2287 }
2288 }
2289 }
2290
2291 fn handle_scope_sync(
2295 &self,
2296 connection_id: ConnectionId,
2297 frame: Frame,
2298 generation: u64,
2299 scopes: Vec<ScopeRecord>,
2300 ) -> Result<Vec<Frame>, RouterError> {
2301 let Some(registration) = self
2302 .registry
2303 .get_module_by_connection(connection_id)
2304 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2305 else {
2306 return Ok(vec![control_error_frame(
2307 &frame,
2308 "not_registered",
2309 "scope.sync requires an active module registration owned by this connection",
2310 )?]);
2311 };
2312 let owner = registration.manifest.module_id;
2313 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2314 let is_current_launch = |connection: ConnectionId| {
2315 self.hello_launch_nonces
2316 .lock()
2317 .unwrap_or_else(|poisoned| poisoned.into_inner())
2318 .presented(connection, current_nonce.as_deref())
2319 };
2320 let mut table = self
2325 .scopes
2326 .write()
2327 .unwrap_or_else(|poisoned| poisoned.into_inner());
2328 let outcome = table.sync(&owner, connection_id, is_current_launch, generation, scopes);
2329 let drained = match &outcome {
2330 Ok(applied) => self
2331 .forwarding
2332 .publish_scope_changes(&applied.tag_changes)
2333 .map_err(RouterError::Forwarding)?,
2334 Err(_) => Vec::new(),
2335 };
2336 drop(table);
2337 match outcome {
2338 Ok(applied) => {
2339 let counts = ScopeOutcomeCounts::of(&applied.results);
2340 info!(
2341 owner = %owner,
2342 generation,
2343 records = applied.results.len(),
2344 created = counts.created,
2345 replaced = counts.replaced,
2346 updated = counts.updated,
2347 unchanged = counts.unchanged,
2348 refused = counts.refused,
2349 ended = applied.ended.len(),
2350 tag_changes = applied.tag_changes.len(),
2351 routes_closed = drained.len(),
2352 "scope sync accepted"
2353 );
2354 for refused in applied
2360 .results
2361 .iter()
2362 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2363 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2364 {
2365 warn!(
2366 owner = %owner,
2367 generation,
2368 scope_ref = %refused.scope_ref,
2369 scope_epoch = refused.scope_epoch,
2370 code = refused.code.as_deref().unwrap_or(""),
2371 "scope record refused"
2372 );
2373 }
2374 self.close_scope_drained_routes(drained);
2375 let response = ModuleControlResponseToModule::ScopeSync {
2376 generation,
2377 results: applied.results,
2378 ended: applied.ended,
2379 };
2380 Ok(vec![control_response_body_frame(
2381 &frame,
2382 &response,
2383 "ModuleControlResponseToModule::ScopeSync",
2384 )?])
2385 }
2386 Err(refusal) => {
2387 info!(
2388 owner = %owner,
2389 generation,
2390 code = refusal.code,
2391 "scope sync refused"
2392 );
2393 Ok(vec![control_error_frame(
2394 &frame,
2395 refusal.code,
2396 refusal.message,
2397 )?])
2398 }
2399 }
2400 }
2401
2402 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2408 if drained.is_empty() {
2409 return;
2410 }
2411 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2412 BTreeMap::new();
2413 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2414 for route in drained {
2415 warn!(
2416 module_id = %route.module_id,
2417 reason = ?route.reason,
2418 client_connection_id = route.client.connection_id.get(),
2419 route_channel = route.client.channel,
2420 "closing route because its scope changed"
2421 );
2422 pushes
2423 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2424 .or_insert_with(|| (route.reason, Vec::new()))
2425 .1
2426 .push(EndpointRoute {
2427 goodbye_target: route.client.clone(),
2428 principal: Principal::Unverified,
2429 bound_at: Instant::now(),
2430 draining: false,
2431 drain_reason: None,
2432 });
2433 goodbyes.push(route.module);
2434 goodbyes.push(route.client);
2435 }
2436 for ((module_id, _), (reason, routes)) in pushes {
2437 send_route_control_pushes(
2438 &self.forwarding,
2439 routes,
2440 ClientControlPush::RouteClosed {
2441 module_id,
2442 reason,
2443 drained: false,
2444 abandoned: 0,
2445 excluded_subscriptions: 0,
2446 terminal: Some(false),
2447 },
2448 );
2449 }
2450 self.emit_route_goodbyes(goodbyes);
2451 }
2452
2453 fn handle_scope_describe(
2456 &self,
2457 connection_id: ConnectionId,
2458 frame: Frame,
2459 owner: Principal,
2460 scope_ref: String,
2461 ) -> Result<Vec<Frame>, RouterError> {
2462 let registered = self
2463 .registry
2464 .get_module_by_connection(connection_id)
2465 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2466 if registered.is_none() {
2467 return Ok(vec![control_error_frame(
2468 &frame,
2469 "not_registered",
2470 "scope.describe requires an active module registration owned by this connection",
2471 )?]);
2472 }
2473 let description = self
2474 .scopes
2475 .read()
2476 .unwrap_or_else(|poisoned| poisoned.into_inner())
2477 .describe(&owner, &scope_ref);
2478 let owner_configured = match &owner {
2479 Principal::Reserved { module_id } => self.supervisor.get(module_id).is_some(),
2480 _ => false,
2481 };
2482 let response = ModuleControlResponseToModule::ScopeDescribe {
2483 status: description.status,
2484 scope_epoch: description.scope_epoch,
2485 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2486 owner_synced: description.owner_synced,
2487 owner_configured,
2488 scope: description.stamp,
2489 };
2490 Ok(vec![control_response_body_frame(
2491 &frame,
2492 &response,
2493 "ModuleControlResponseToModule::ScopeDescribe",
2494 )?])
2495 }
2496
2497 fn handle_catalog_update(
2498 &self,
2499 connection_id: ConnectionId,
2500 frame: Frame,
2501 provides: Vec<ProviderRole>,
2502 capabilities: Option<CapabilityDeclarations>,
2503 ready: Option<bool>,
2504 ) -> Result<Vec<Frame>, RouterError> {
2505 self.refresh_capability_requirements();
2506 let Some(registration) = self
2507 .registry
2508 .get_module_by_connection(connection_id)
2509 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2510 else {
2511 return Ok(vec![control_error_frame(
2512 &frame,
2513 "not_registered",
2514 "catalog.update requires an active module registration owned by this connection",
2515 )?]);
2516 };
2517
2518 if let Some(message) =
2519 catalog_update_frozen_field_message(®istration.manifest, &provides)
2520 {
2521 return Ok(vec![control_error_frame(
2522 &frame,
2523 "catalog_update_frozen_field",
2524 message,
2525 )?]);
2526 }
2527
2528 let mut candidate = registration.manifest.clone();
2529 candidate.provides = provides.clone();
2530 candidate.capabilities = capabilities
2531 .clone()
2532 .or_else(|| registration.manifest.capabilities.clone());
2533 if let Err(err) = candidate.validate_capability_grammar() {
2534 return Ok(vec![control_error_frame(
2535 &frame,
2536 "invalid_capability_grammar",
2537 err.to_string(),
2538 )?]);
2539 }
2540
2541 let updated = self
2542 .registry
2543 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2544 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2545 if updated.is_none() {
2546 return Ok(vec![control_error_frame(
2547 &frame,
2548 "not_registered",
2549 "catalog.update requires an active module registration owned by this connection",
2550 )?]);
2551 }
2552 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2553 log_duplicate_claim_events(
2554 self.capability_evaluator
2555 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2556 );
2557 }
2558 if capability_census_trigger(
2559 registration.manifest.capabilities.as_ref(),
2560 updated
2561 .as_ref()
2562 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2563 ) {
2564 self.enforce_capability_denies();
2565 }
2566 self.refresh_capability_requirements();
2567
2568 let response = ModuleControlResponseToModule::CatalogUpdate {};
2569 control_response_body_frame(
2570 &frame,
2571 &response,
2572 "ModuleControlResponseToModule::CatalogUpdate",
2573 )
2574 .map(|frame| vec![frame])
2575 }
2576
2577 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2578 self.refresh_capability_requirements();
2579 let mut counters = self.counters.snapshot();
2585 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2586 self.forwarding.client_route_concentration(),
2587 counters.as_object_mut(),
2588 ) {
2589 obj.insert(
2590 "client_connections_with_routes".into(),
2591 connections_with_routes.into(),
2592 );
2593 obj.insert("max_routes_on_one_connection".into(), max.into());
2594 }
2595 if let (Some(open_breakers), Some(obj)) = (
2601 self.route_bind_breakers.open_snapshot(),
2602 counters.as_object_mut(),
2603 ) {
2604 obj.insert("route_bind_breakers_open".into(), open_breakers);
2605 }
2606 let response = ClientControlResponse::ServerDescribe {
2607 protocol_ver: PROTOCOL_VERSION,
2608 subc_ops: subc_ops(),
2609 capabilities: self.subc_capabilities.as_ref().to_vec(),
2610 connected_clients: self.connected_clients.count(),
2611 counters: Some(counters),
2612 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2613 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2614 capability_requirements: self.capability_requirement_statuses(),
2615 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2616 };
2617 Ok(vec![control_response_body_frame(
2618 &frame,
2619 &response,
2620 "ClientControlResponse::ServerDescribe",
2621 )?])
2622 }
2623
2624 fn handle_catalog_list(
2625 &self,
2626 frame: Frame,
2627 module_id: Option<String>,
2628 ) -> Result<Vec<Frame>, RouterError> {
2629 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2630 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2631 })?;
2632 let entries = modules
2633 .into_iter()
2634 .filter(|registration| {
2635 module_id
2636 .as_deref()
2637 .map(|wanted| registration.manifest.module_id == wanted)
2638 .unwrap_or(true)
2639 })
2640 .map(|registration| {
2641 let not_ready = self.not_ready_reason(®istration);
2642 let roles = registration.manifest.provides;
2643 CatalogEntry {
2644 module_id: registration.manifest.module_id,
2645 ready: not_ready.is_none(),
2646 not_ready,
2647 module_version: Some(registration.manifest.module_version),
2648 roles,
2649 control_ops: registration.control_ops,
2650 capabilities: registration.manifest.capabilities,
2651 self_signals: registration.manifest.self_signals,
2652 }
2653 })
2654 .collect();
2655 let response = ClientControlResponse::CatalogList {
2656 generation,
2657 modules: entries,
2658 subc_ops: subc_ops(),
2659 };
2660 Ok(vec![control_response_body_frame(
2661 &frame,
2662 &response,
2663 "ClientControlResponse::CatalogList",
2664 )?])
2665 }
2666
2667 fn route_open_principal(
2668 &self,
2669 frame: &Frame,
2670 consumer_identity: Option<ConsumerIdentity>,
2671 ) -> Result<Result<Principal, Frame>, RouterError> {
2672 let Some(consumer_identity) = consumer_identity else {
2673 return Ok(Ok(Principal::Direct));
2674 };
2675
2676 if self.supervisor.spawned_consumer_authorized(
2677 &consumer_identity.module_id,
2678 &consumer_identity.launch_nonce,
2679 ) {
2680 return Ok(Ok(Principal::Reserved {
2681 module_id: consumer_identity.module_id,
2682 }));
2683 }
2684
2685 Ok(Err(control_error_frame(
2686 frame,
2687 "bad_consumer_identity",
2688 format!(
2689 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2690 consumer_identity.module_id
2691 ),
2692 )?))
2693 }
2694
2695 fn route_open_refusal_frame(
2702 &self,
2703 ctx: &RouteCtx,
2704 frame: &Frame,
2705 module_id: &str,
2706 reason: &'static str,
2707 code: &'static str,
2708 message: impl Into<String>,
2709 ) -> Result<Frame, RouterError> {
2710 self.observe_route_open_refusal(ctx, module_id, reason, code);
2711 control_error_frame(frame, code, message.into())
2712 }
2713
2714 fn route_open_breaker_refusal_frame(
2732 &self,
2733 ctx: &RouteCtx,
2734 frame: &Frame,
2735 module_id: &str,
2736 consecutive_timeouts: u32,
2737 retry_in: Duration,
2738 probe_in_flight: bool,
2739 ) -> Result<Frame, RouterError> {
2740 self.counters
2741 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2742 debug!(
2743 target: "control",
2744 code = "module_timeout",
2745 module_id = ?module_id,
2746 connection_id = ctx.connection_id.get(),
2747 consecutive_timeouts,
2748 retry_in_ms = retry_in.as_millis() as u64,
2749 probe_in_flight,
2750 "route.open refused by open bind-relay breaker"
2751 );
2752 let detail = if probe_in_flight {
2753 "one probe bind is already in flight; retry once it settles".to_string()
2754 } else {
2755 format!("not relaying for another {retry_in:?}")
2756 };
2757 control_error_frame(
2758 frame,
2759 "module_timeout",
2760 format!(
2761 "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2762 relays; {detail}"
2763 ),
2764 )
2765 }
2766
2767 fn observe_route_open_refusal(
2780 &self,
2781 ctx: &RouteCtx,
2782 module_id: &str,
2783 reason: &'static str,
2784 code: &'static str,
2785 ) {
2786 self.counters.increment_route_open_refused(code);
2787 info!(
2788 target: "control",
2789 code,
2790 reason,
2791 module_id = ?module_id,
2792 connection_id = ctx.connection_id.get(),
2793 "route.open refused"
2794 );
2795 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
2796 self.route_outages.record_not_serving(module_id, reason);
2797 }
2798 }
2799
2800 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2845 self.route_outages.record_accepted(module_id);
2846 self.counters.increment_route_open_accepted(principal);
2847 info!(
2848 target: "control",
2849 principal,
2850 module_id,
2851 connection_id = ctx.connection_id.get(),
2852 "route.open accepted"
2853 );
2854 }
2855
2856 fn supervised_absent_route_open_refusal_frame(
2857 &self,
2858 ctx: &RouteCtx,
2859 frame: &Frame,
2860 module_id: &str,
2861 code: &'static str,
2862 status: &crate::supervise::ModuleStatus,
2863 ) -> Result<Frame, RouterError> {
2864 self.counters.increment_route_open_refused(code);
2865 info!(
2866 target: "control",
2867 code,
2868 reason = "supervised_not_registered",
2869 module_id = ?module_id,
2870 connection_id = ctx.connection_id.get(),
2871 state = %status.state,
2872 enabled = status.enabled,
2873 live = status.live,
2874 "route.open refused"
2875 );
2876 self.route_outages
2880 .record_not_serving(module_id, "supervised_not_registered");
2881 control_error_frame(
2882 frame,
2883 code,
2884 format!(
2885 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2886 status.state, status.enabled, status.live
2887 ),
2888 )
2889 }
2890
2891 async fn handle_route_open(
2892 &self,
2893 ctx: &RouteCtx,
2894 frame: Frame,
2895 request: RouteOpenRequest,
2896 ) -> Result<Vec<Frame>, RouterError> {
2897 let RouteOpenRequest {
2898 target,
2899 mut identity,
2900 consumer_identity,
2901 consumer_capabilities,
2902 admission_facts,
2903 scope,
2904 } = request;
2905 let target_module_id = target_module_id(&target).to_string();
2906 debug!(
2907 connection_id = ctx.connection_id.get(),
2908 corr = frame.header.corr,
2909 module_id = %target_module_id,
2910 "handling route.open"
2911 );
2912
2913 let Some(registration) = self
2932 .registry
2933 .get_module(&target_module_id)
2934 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2935 else {
2936 if let Some((status, warming)) =
2937 self.supervisor_status(&target_module_id, frame.header.corr)?
2938 {
2939 if status.protocol == ModuleProtocol::None {
2946 return Ok(vec![self.route_open_refusal_frame(
2947 ctx,
2948 &frame,
2949 &target_module_id,
2950 "protocol_none",
2951 error_codes::MODULE_NO_PROTOCOL,
2952 format!(
2953 "module_id '{target_module_id}' is declared protocol: none; \
2954 it speaks no subc wire and serves no routes"
2955 ),
2956 )?]);
2957 }
2958 let code = if warming {
2959 "module_warming"
2960 } else {
2961 "target_unavailable"
2962 };
2963 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2964 ctx,
2965 &frame,
2966 &target_module_id,
2967 code,
2968 &status,
2969 )?]);
2970 }
2971 if let Some(removed_ago_ms) =
2972 self.supervisor.removal_tombstone_age_ms(&target_module_id)
2973 {
2974 return Ok(vec![self.route_open_refusal_frame(
2975 ctx,
2976 &frame,
2977 &target_module_id,
2978 "removed",
2979 error_codes::MODULE_REMOVED,
2980 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
2981 )?]);
2982 }
2983 return Ok(vec![self.route_open_refusal_frame(
2984 ctx,
2985 &frame,
2986 &target_module_id,
2987 "not_registered",
2988 error_codes::UNKNOWN_MODULE,
2989 format!("module_id '{target_module_id}' is not registered"),
2990 )?]);
2991 };
2992
2993 if !registration.ready {
2997 self.counters
2998 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
2999 info!(
3000 target: "control",
3001 code = error_codes::MODULE_WARMING,
3002 module_id = ?target_module_id,
3003 connection_id = ctx.connection_id.get(),
3004 reason = "declared_not_ready",
3005 "route.open refused"
3006 );
3007 self.route_outages
3010 .record_not_serving(&target_module_id, "declared_not_ready");
3011 return Ok(vec![control_error_body_frame(
3012 &frame,
3013 ErrorBody {
3014 code: error_codes::MODULE_WARMING.to_string(),
3015 message: format!(
3016 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3017 ),
3018 detail: Some(serde_json::json!({
3019 "reason": "declared_not_ready"
3020 })),
3021 },
3022 )?]);
3023 }
3024
3025 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3044 self.counters
3045 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3046 info!(
3047 target: "control",
3048 code = error_codes::MODULE_WARMING,
3049 module_id = ?target_module_id,
3050 connection_id = ctx.connection_id.get(),
3051 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3052 capability = %capability,
3053 "route.open refused"
3054 );
3055 return Ok(vec![control_error_body_frame(
3056 &frame,
3057 ErrorBody {
3058 code: error_codes::MODULE_WARMING.to_string(),
3059 message: format!(
3060 "module_id '{target_module_id}' requires capability '{capability}', \
3061 which no registered module provides; retry"
3062 ),
3063 detail: Some(serde_json::json!({
3064 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3065 "capability": capability,
3066 })),
3067 },
3068 )?]);
3069 }
3070
3071 if !target_has_required_role(&target, ®istration.manifest.provides) {
3072 return Ok(vec![self.route_open_refusal_frame(
3073 ctx,
3074 &frame,
3075 &target_module_id,
3076 "role_not_provided",
3077 "target_unavailable",
3078 format!("module_id '{target_module_id}' does not provide the requested target"),
3079 )?]);
3080 }
3081
3082 if registration.state != ChannelState::Active {
3083 return Ok(vec![self.route_open_refusal_frame(
3084 ctx,
3085 &frame,
3086 &target_module_id,
3087 "registration_not_active",
3088 "target_unavailable",
3089 format!("module_id '{target_module_id}' is not active"),
3090 )?]);
3091 }
3092
3093 if self
3094 .forwarding
3095 .module_is_draining(&target_module_id)
3096 .map_err(RouterError::Forwarding)?
3097 {
3098 return Ok(vec![self.route_open_refusal_frame(
3099 ctx,
3100 &frame,
3101 &target_module_id,
3102 "reloading",
3103 "module_reloading",
3104 format!("module_id '{target_module_id}' is reloading"),
3105 )?]);
3106 }
3107
3108 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3109 process_liveness.process_live(&target_module_id) == Some(false)
3110 }) {
3111 if process_liveness.process_replacing(&target_module_id) {
3118 return Ok(vec![self.route_open_refusal_frame(
3119 ctx,
3120 &frame,
3121 &target_module_id,
3122 "reloading",
3123 "module_reloading",
3124 format!("module_id '{target_module_id}' is reloading"),
3125 )?]);
3126 }
3127 return Ok(vec![self.route_open_refusal_frame(
3128 ctx,
3129 &frame,
3130 &target_module_id,
3131 "supervisor_not_live",
3132 "target_unavailable",
3133 format!("module_id '{target_module_id}' is not live"),
3134 )?]);
3135 }
3136
3137 if !self
3138 .forwarding
3139 .has_live_module_connection(&target_module_id)
3140 .map_err(RouterError::Forwarding)?
3141 {
3142 return Ok(vec![self.route_open_refusal_frame(
3143 ctx,
3144 &frame,
3145 &target_module_id,
3146 "no_forwarding_connection",
3147 "target_unavailable",
3148 format!("module_id '{target_module_id}' has no live forwarding connection"),
3149 )?]);
3150 }
3151
3152 if let Some(error) =
3153 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3154 {
3155 self.observe_route_open_refusal(
3156 ctx,
3157 &target_module_id,
3158 "op_not_allowed",
3159 "op_not_allowed",
3160 );
3161 return Ok(vec![error]);
3162 }
3163
3164 let principal = match self.route_open_principal(&frame, consumer_identity)? {
3165 Ok(principal) => principal,
3166 Err(error) => {
3167 self.observe_route_open_refusal(
3168 ctx,
3169 &target_module_id,
3170 "bad_consumer_identity",
3171 "bad_consumer_identity",
3172 );
3173 return Ok(vec![error]);
3174 }
3175 };
3176
3177 if let Principal::Reserved {
3181 module_id: opening_module_id,
3182 } = &principal
3183 {
3184 if let Some(opening_registration) = self
3185 .registry
3186 .get_module(opening_module_id)
3187 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3188 {
3189 if let Some(capability) =
3190 denied_capability(&opening_registration.manifest, ®istration.manifest)
3191 {
3192 warn!(
3193 opening_module_id,
3194 target_module_id,
3195 capability,
3196 "refusing route.open because an attested capability deny edge matches"
3197 );
3198 return Ok(vec![self.route_open_refusal_frame(
3199 ctx,
3200 &frame,
3201 &target_module_id,
3202 "capability_deny_edge",
3203 "capability_forbidden",
3204 format!(
3205 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3206 ),
3207 )?]);
3208 }
3209 }
3210 }
3211
3212 if admission_facts.is_some() {
3213 let carrier_matches = matches!(
3214 &principal,
3215 Principal::Reserved { module_id }
3216 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3217 );
3218 if !carrier_matches {
3219 return Ok(vec![self.route_open_refusal_frame(
3220 ctx,
3221 &frame,
3222 &target_module_id,
3223 "admission_facts_carrier_not_permitted",
3224 "admission_facts_not_permitted",
3225 "admission facts may only be carried by the configured reserved module",
3226 )?]);
3227 }
3228
3229 let target_allowed = self
3230 .admission_facts_targets
3231 .as_ref()
3232 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3233 if !target_allowed {
3234 return Ok(vec![self.route_open_refusal_frame(
3235 ctx,
3236 &frame,
3237 &target_module_id,
3238 "admission_facts_target_not_listed",
3239 "admission_facts_target_not_allowed",
3240 format!(
3241 "admission facts are not permitted for target module_id '{target_module_id}'"
3242 ),
3243 )?]);
3244 }
3245
3246 }
3250
3251 let (bound_scope, scope_stamp) = match scope {
3257 None => (None, None),
3258 Some(selector) => {
3259 let owner_configured = match &selector.owner {
3260 Principal::Reserved { module_id } => self.supervisor.get(module_id).is_some(),
3261 _ => false,
3262 };
3263 let admitted = self
3264 .scopes
3265 .read()
3266 .unwrap_or_else(|poisoned| poisoned.into_inner())
3267 .admit(&principal, &target_module_id, &selector, owner_configured);
3268 match admitted {
3269 Ok(admission) => (
3270 Some(BoundScope {
3271 owner: admission.owner,
3272 scope_ref: admission.stamp.scope_ref.clone(),
3273 tag: admission.tag,
3274 }),
3275 Some(admission.stamp),
3276 ),
3277 Err(refusal) => {
3278 return Ok(vec![self.route_open_refusal_frame(
3279 ctx,
3280 &frame,
3281 &target_module_id,
3282 refusal.code,
3283 refusal.code,
3284 refusal.message,
3285 )?]);
3286 }
3287 }
3288 }
3289 };
3290
3291 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3313 Ok(project_root) => project_root,
3314 Err(err) => {
3315 return Ok(vec![control_error_frame(
3316 &frame,
3317 "invalid_project_root",
3318 err.to_string(),
3319 )?])
3320 }
3321 };
3322 identity.project_root = project_root.as_path().to_path_buf();
3323
3324 let _concurrency_guard = match self
3335 .route_bind_concurrency
3336 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3337 {
3338 Ok(guard) => guard,
3339 Err(in_flight) => {
3340 return Ok(vec![self.route_open_target_capacity_refusal(
3341 ctx,
3342 &frame,
3343 &target_module_id,
3344 in_flight,
3345 )?]);
3346 }
3347 };
3348
3349 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3352 RouteBindAdmission::Admitted { guard, probe } => {
3353 if probe {
3354 info!(
3355 module_id = %target_module_id,
3356 connection_id = ctx.connection_id.get(),
3357 "route.bind breaker half-open: admitting one probe"
3358 );
3359 }
3360 guard
3361 }
3362 RouteBindAdmission::Refused {
3363 consecutive_timeouts,
3364 retry_in,
3365 probe_in_flight,
3366 } => {
3367 return Ok(vec![self.route_open_breaker_refusal_frame(
3368 ctx,
3369 &frame,
3370 &target_module_id,
3371 consecutive_timeouts,
3372 retry_in,
3373 probe_in_flight,
3374 )?]);
3375 }
3376 };
3377
3378 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3383 let relay_deadline = Instant::now() + route_bind_relay_timeout;
3384 let pending = match self
3385 .forwarding
3386 .begin_route_bind_relay_for(
3387 ctx.connection_id,
3388 ctx.egress.clone(),
3389 response_version(&frame),
3390 frame.header.corr,
3391 &target_module_id,
3392 principal.clone(),
3393 bound_scope,
3394 Some(project_root),
3395 relay_deadline,
3396 )
3397 .await
3398 {
3399 Ok(pending) => pending,
3400 Err(err) => {
3401 return Ok(vec![self.route_open_refusal_frame(
3402 ctx,
3403 &frame,
3404 &target_module_id,
3405 "relay_reservation_failed",
3406 forwarding_error_code(&err),
3407 err.to_string(),
3408 )?])
3409 }
3410 };
3411 let crate::forwarding::PendingRouteBindRelay {
3412 endpoint,
3413 module_sink,
3414 negotiated_ver,
3415 client_channel,
3416 client_epoch,
3417 module_channel,
3418 module_epoch,
3419 corr: relay_corr,
3420 receiver,
3421 } = pending;
3422 let mut reservation =
3423 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3424
3425 debug!(
3426 connection_id = ctx.connection_id.get(),
3427 client_channel,
3428 client_epoch,
3429 module_channel,
3430 module_epoch,
3431 "reserved route handle pair"
3432 );
3433 let principal_label = match &principal {
3436 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3437 Principal::Direct => "direct".to_string(),
3438 other => format!("{other:?}"),
3439 };
3440 let relay = ModuleControlRequest::RouteBind {
3441 route_channel: module_channel,
3442 epoch: module_epoch,
3443 target,
3444 identity,
3445 principal: Some(principal),
3446 consumer_capabilities,
3447 admission_facts,
3448 scope: scope_stamp,
3449 };
3450 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3451 RouterError::backend(
3452 0,
3453 frame.header.corr,
3454 format!("failed to encode route.bind request: {err}"),
3455 )
3456 })?;
3457 let relay_frame = Frame::build_with_version(
3458 negotiated_ver,
3459 FrameType::Request,
3460 control_flags(),
3461 0,
3462 0,
3463 relay_corr,
3464 relay_body,
3465 )
3466 .map_err(RouterError::FrameBuild)?;
3467
3468 if let Err(err) = module_sink.send(relay_frame).await {
3469 reservation.release_and_disarm();
3470 return Ok(vec![self.route_open_refusal_frame(
3471 ctx,
3472 &frame,
3473 &target_module_id,
3474 "relay_send_failed",
3475 "target_unavailable",
3476 err.to_string(),
3477 )?]);
3478 }
3479
3480 if !self
3481 .forwarding
3482 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3483 .map_err(RouterError::Forwarding)?
3484 {
3485 self.send_abandoned_route_bind_goodbye(
3486 &module_sink,
3487 negotiated_ver,
3488 module_channel,
3489 module_epoch,
3490 );
3491 }
3492
3493 match timeout_at(relay_deadline, receiver).await {
3494 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3495 reservation.disarm();
3496 if breaker.record_accepted() {
3497 info!(
3498 module_id = %target_module_id,
3499 "route.bind breaker closed: the probe was accepted"
3500 );
3501 }
3502 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3503 Ok(Vec::new())
3504 }
3505 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3506 reservation.release_and_disarm();
3507 breaker.record_inconclusive();
3511 let scope_code = match body.code.as_str() {
3515 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
3516 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
3517 _ => None,
3518 };
3519 if let Some(code) = scope_code {
3520 self.observe_route_open_refusal(
3521 ctx,
3522 &target_module_id,
3523 "scope_changed_before_commit",
3524 code,
3525 );
3526 return Ok(vec![control_error_body_frame(&frame, body)?]);
3527 }
3528 self.counters
3529 .increment_route_open_refused("module_rejected");
3530 info!(
3531 target: "control",
3532 code = "module_rejected",
3533 module_code = ?body.code,
3534 module_id = ?target_module_id,
3535 connection_id = ctx.connection_id.get(),
3536 "route.open refused"
3537 );
3538 Ok(vec![control_error_body_frame(&frame, body)?])
3539 }
3540 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3541 reservation.release_and_disarm();
3542 breaker.record_inconclusive();
3543 tracing::warn!(
3550 module_id = %target_module_id,
3551 "route.bind relay abandoned: {message}"
3552 );
3553 Ok(vec![self.route_open_refusal_frame(
3554 ctx,
3555 &frame,
3556 &target_module_id,
3557 "relay_abandoned",
3558 "target_unavailable",
3559 message,
3560 )?])
3561 }
3562 Ok(Err(_)) => {
3563 reservation.release_and_disarm();
3564 breaker.record_inconclusive();
3565 Ok(vec![self.route_open_refusal_frame(
3566 ctx,
3567 &frame,
3568 &target_module_id,
3569 "relay_waiter_canceled",
3570 "target_unavailable",
3571 "route.bind relay waiter was canceled before the module responded",
3572 )?])
3573 }
3574 Err(_) => {
3575 reservation.release_and_disarm();
3576 if let Some(opened) = breaker.record_timeout(
3580 self.route_bind_breaker_threshold,
3581 self.route_bind_breaker_cooldown,
3582 ) {
3583 warn!(
3584 module_id = %target_module_id,
3585 consecutive_timeouts = opened.consecutive_timeouts,
3586 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3587 reopened_after_probe = opened.reopened_after_probe,
3588 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3589 );
3590 }
3591 tracing::warn!(
3599 module_id = %target_module_id,
3600 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3601 "route.bind relay timed out: module did not ack within budget"
3602 );
3603 Ok(vec![self.route_open_refusal_frame(
3604 ctx,
3605 &frame,
3606 &target_module_id,
3607 "relay_timed_out",
3608 "module_timeout",
3609 format!(
3610 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3611 route_bind_relay_timeout
3612 ),
3613 )?])
3614 }
3615 }
3616 }
3617
3618 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3619 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3620 snapshot: self.supervisor.spawn_snapshot(),
3621 };
3622 Ok(vec![control_response_body_frame(
3623 &frame,
3624 &response,
3625 "ClientControlResponse::SupervisorSpawnSnapshot",
3626 )?])
3627 }
3628
3629 fn handle_supervisor_spawn_subscribe(
3630 &self,
3631 ctx: &RouteCtx,
3632 frame: Frame,
3633 since: Option<SpawnCursor>,
3634 ) -> Result<Vec<Frame>, RouterError> {
3635 match self.supervisor.subscribe_spawns(
3636 ctx.connection_id,
3637 frame.header.corr,
3638 response_version(&frame),
3639 since,
3640 ctx.egress.clone(),
3641 ) {
3642 Ok(()) => Ok(Vec::new()),
3643 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3644 Ok(vec![control_error_body_frame(
3645 &frame,
3646 ErrorBody {
3647 code: "spawn_cursor_incarnation_mismatch".to_string(),
3648 message: "spawn cursor belongs to a different daemon incarnation"
3649 .to_string(),
3650 detail: Some(serde_json::json!({
3651 "current_daemon_incarnation": current
3652 })),
3653 },
3654 )?])
3655 }
3656 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3657 &frame,
3658 ErrorBody {
3659 code: "spawn_cursor_too_old".to_string(),
3660 message: "spawn cursor predates the retained event ring".to_string(),
3661 detail: Some(serde_json::json!({
3662 "oldest_retained_cursor": oldest
3663 })),
3664 },
3665 )?]),
3666 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3667 0,
3668 frame.header.corr,
3669 format!("failed to open supervisor spawn subscription: {error}"),
3670 )),
3671 }
3672 }
3673
3674 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3675 let generation = self
3676 .registry
3677 .generation()
3678 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3679 let mut modules = Vec::new();
3680 for module in self.supervisor.list() {
3681 let status = module.status_for_control("list").map_err(|err| {
3682 RouterError::backend(
3683 0,
3684 frame.header.corr,
3685 format!("failed to read supervisor status: {err}"),
3686 )
3687 })?;
3688 let (configured, _) = module.configuration().map_err(|err| {
3689 RouterError::backend(
3690 0,
3691 frame.header.corr,
3692 format!("failed to read module configuration: {err}"),
3693 )
3694 })?;
3695 let image = module.running_image_agreement().await;
3697 let resources = Some(module.child_resource_usage());
3700 let pending_reload = Some(reload_verdict(
3701 &configured.program,
3702 status.spawned_from.as_deref(),
3703 image,
3704 ));
3705 modules.push(SupervisorEntry {
3706 launch_nonce_env: Some(
3707 configured.protocol != subc_control::ModuleProtocol::None
3708 && (!cfg!(unix) || configured.launch_nonce_env),
3709 ),
3710 module_id: status.module_id,
3711 state: status.state.to_string(),
3712 enabled: status.enabled,
3713 live: status.live,
3714 protocol: status.protocol,
3715 health: status.health.status,
3716 pending_reload,
3717 last_probe_ms: status.health.last_probe_ms,
3718 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3719 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3720 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3721 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3722 restart_count: Some(status.restart_count),
3723 max_restarts: Some(status.max_restarts),
3724 lifetime_restarts: Some(status.lifetime_restarts),
3725 spawn_generation: Some(status.spawn_generation),
3726 restart_window_secs: Some(status.restart_window.as_secs()),
3727 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3728 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3729 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3730 resources,
3731 });
3732 }
3733 let response = ClientControlResponse::SupervisorList {
3734 generation,
3735 modules,
3736 };
3737 Ok(vec![control_response_body_frame(
3738 &frame,
3739 &response,
3740 "ClientControlResponse::SupervisorList",
3741 )?])
3742 }
3743
3744 fn handle_supervisor_stderr_tail(
3745 &self,
3746 frame: Frame,
3747 module_id: String,
3748 max_lines: Option<u32>,
3749 max_bytes: Option<u32>,
3750 ) -> Result<Vec<Frame>, RouterError> {
3751 let Some(module) = self.supervisor.get(&module_id) else {
3752 return Ok(vec![control_error_frame(
3753 &frame,
3754 "unknown_module",
3755 format!("module_id '{module_id}' is not supervised"),
3756 )?]);
3757 };
3758
3759 let snapshot = module.stderr_tail(
3760 max_lines.map(|value| value as usize),
3761 max_bytes.map(|value| value as usize),
3762 );
3763
3764 let response = ClientControlResponse::SupervisorStderrTail {
3765 module_id,
3766 tail: StderrTail {
3767 capture: match snapshot.capture {
3768 CaptureState::Captured => StderrCaptureState::Captured,
3769 CaptureState::Incomplete { reason } => {
3770 StderrCaptureState::Incomplete { reason }
3771 }
3772 CaptureState::NotCaptured { reason } => {
3773 StderrCaptureState::NotCaptured { reason }
3774 }
3775 },
3776 entries: snapshot
3777 .entries
3778 .into_iter()
3779 .map(|entry| match entry {
3780 TailEntry::Line {
3781 text,
3782 truncated,
3783 at_ms,
3784 } => StderrTailEntry::Line {
3785 text,
3786 truncated,
3787 at_ms,
3788 },
3789 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3790 })
3791 .collect(),
3792 dropped_lines: snapshot.dropped_lines,
3793 },
3794 };
3795 Ok(vec![control_response_body_frame(
3796 &frame,
3797 &response,
3798 "ClientControlResponse::SupervisorStderrTail",
3799 )?])
3800 }
3801
3802 async fn handle_supervisor_terminals(
3803 &self,
3804 frame: Frame,
3805 module_id: String,
3806 ) -> Result<Vec<Frame>, RouterError> {
3807 let Some(module) = self.supervisor.get(&module_id) else {
3808 return Ok(vec![control_error_frame(
3809 &frame,
3810 "unknown_module",
3811 format!("module_id '{module_id}' is not supervised"),
3812 )?]);
3813 };
3814
3815 let terminals = module
3818 .read_durable_terminal_history()
3819 .await
3820 .map_err(|error| {
3821 RouterError::backend(
3822 0,
3823 frame.header.corr,
3824 format!("failed to read terminal history: {error}"),
3825 )
3826 })?;
3827 let response = ClientControlResponse::SupervisorTerminals {
3828 module_id,
3829 terminals,
3830 };
3831 Ok(vec![control_response_body_frame(
3832 &frame,
3833 &response,
3834 "ClientControlResponse::SupervisorTerminals",
3835 )?])
3836 }
3837
3838 fn handle_supervisor_routes(
3839 &self,
3840 frame: Frame,
3841 module_id: Option<String>,
3842 ) -> Result<Vec<Frame>, RouterError> {
3843 let modules = self
3844 .forwarding
3845 .route_census(module_id.as_deref())
3846 .map_err(RouterError::Forwarding)?
3847 .into_iter()
3848 .map(|(module_id, routes)| SupervisorRouteModule {
3849 module_id,
3850 routes: routes
3851 .into_iter()
3852 .map(|route| SupervisorRoute {
3853 consumer: match route.principal {
3854 Principal::Reserved { module_id } => {
3855 SupervisorRouteConsumer::Reserved { module_id }
3856 }
3857 Principal::Direct | Principal::Unverified => {
3858 SupervisorRouteConsumer::Direct {
3859 connection_id: route.goodbye_target.connection_id.get(),
3860 }
3861 }
3862 },
3863 age_ms: Instant::now()
3864 .saturating_duration_since(route.bound_at)
3865 .as_millis()
3866 .try_into()
3867 .unwrap_or(u64::MAX),
3868 draining: route.draining,
3869 drain_reason: route.drain_reason,
3870 })
3871 .collect(),
3872 })
3873 .collect();
3874 let response = ClientControlResponse::SupervisorRoutes { modules };
3875 Ok(vec![control_response_body_frame(
3876 &frame,
3877 &response,
3878 "ClientControlResponse::SupervisorRoutes",
3879 )?])
3880 }
3881
3882 async fn handle_supervisor_provenance(
3883 &self,
3884 frame: Frame,
3885 module_id: Option<String>,
3886 ) -> Result<Vec<Frame>, RouterError> {
3887 let mut selected = if let Some(module_id) = module_id {
3888 let Some(module) = self.supervisor.get(&module_id) else {
3889 return Ok(vec![control_error_frame(
3890 &frame,
3891 "unknown_module",
3892 format!("module_id '{module_id}' is not supervised"),
3893 )?]);
3894 };
3895 vec![module]
3896 } else {
3897 self.supervisor.list()
3898 };
3899
3900 let mut modules = Vec::with_capacity(selected.len());
3901 for module in selected.drain(..) {
3902 let status = module.status().map_err(|err| {
3903 RouterError::backend(
3904 0,
3905 frame.header.corr,
3906 format!("failed to read supervisor status: {err}"),
3907 )
3908 })?;
3909 let module_declared = self
3910 .registry
3911 .get_module(&status.module_id)
3912 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3913 .and_then(|registration| registration.manifest.provenance)
3914 .map(|build| ModuleDeclaredProvenance::Reported { build })
3915 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3916 #[cfg(test)]
3917 let running_image = match &self.provenance_probe_override {
3918 Some(result) => result.clone(),
3919 None => module.running_image_agreement().await,
3920 };
3921 #[cfg(not(test))]
3922 let running_image = module.running_image_agreement().await;
3923 modules.push(SupervisorModuleProvenance {
3924 module_id: status.module_id,
3925 module_declared,
3926 daemon_observed: SupervisorObservedProcess {
3927 pid: status.pid,
3928 spawned_at_ms: status.spawned_at_ms,
3929 spawned_from: status.spawned_from,
3930 running_image,
3931 },
3932 });
3933 }
3934 let daemon = SupervisorDaemonProvenance {
3935 daemon_build: self.daemon_provenance.build.clone(),
3936 daemon_observed: DaemonObservedProcess {
3937 pid: self.daemon_provenance.pid,
3938 started_at_ms: self
3939 .daemon_provenance
3940 .start_clock
3941 .map(|clock| clock.started_at_ms())
3942 .or(self.daemon_provenance.started_at_ms),
3943 running_image: self
3944 .daemon_provenance
3945 .probe
3946 .observe(
3947 self.daemon_provenance.pid,
3948 self.daemon_provenance.executable_path.as_deref(),
3949 self.daemon_provenance.executable_identity,
3950 self.daemon_provenance.process_start_time,
3951 )
3952 .await,
3953 },
3954 };
3955 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3956 Ok(vec![control_response_body_frame(
3957 &frame,
3958 &response,
3959 "ClientControlResponse::SupervisorProvenance",
3960 )?])
3961 }
3962
3963 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3964 self.refresh_capability_requirements();
3965 let generation = self
3966 .registry
3967 .generation()
3968 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3969 let modules = self
3970 .supervisor
3971 .list()
3972 .into_iter()
3973 .map(|module| {
3974 let status = module.status_for_control("health").map_err(|err| {
3975 RouterError::backend(
3976 0,
3977 frame.header.corr,
3978 format!("failed to read supervisor health: {err}"),
3979 )
3980 })?;
3981 let module_id = status.module_id;
3982 let capability_detail = self
3983 .capability_evaluator
3984 .required_problem_detail(&module_id);
3985 Ok(SupervisorHealthEntry {
3986 module_id,
3987 status: status.health.status,
3988 detail: append_capability_problem_detail(
3989 status.health.detail,
3990 capability_detail,
3991 ),
3992 metrics: status.health.metrics,
3993 consecutive_failures: status.health.consecutive_failures,
3994 late_answer_count: status.health.late_answer_count,
3995 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
3996 last_action: status.health.last_action,
3997 last_action_ms: status.health.last_action_ms,
3998 last_probe_ms: status.health.last_probe_ms,
3999 })
4000 })
4001 .collect::<Result<Vec<_>, RouterError>>()?;
4002 let response = ClientControlResponse::SupervisorHealth {
4003 generation,
4004 modules,
4005 };
4006 Ok(vec![control_response_body_frame(
4007 &frame,
4008 &response,
4009 "ClientControlResponse::SupervisorHealth",
4010 )?])
4011 }
4012
4013 async fn handle_supervisor_restart(
4014 &self,
4015 frame: Frame,
4016 module_id: String,
4017 drain_timeout_ms: Option<u64>,
4018 ) -> Result<Vec<Frame>, RouterError> {
4019 let operation_lock = self.supervisor.operation_lock();
4020 let _operation_guard = operation_lock.lock().await;
4021 let Some(module) = self.supervisor.get(&module_id) else {
4022 return Ok(vec![control_error_frame(
4023 &frame,
4024 "unknown_module",
4025 format!("module_id '{module_id}' is not supervised"),
4026 )?]);
4027 };
4028
4029 self.route_outages.mark_operator_action(&module_id);
4030 if let Err(err) = module.restart(drain_timeout_ms).await {
4031 self.route_outages
4032 .operator_action_ended_unrefused(&module_id);
4033 let (code, message) = match err {
4034 crate::supervise::SuperviseError::Disabled { .. } => {
4035 ("module_disabled", err.to_string())
4036 }
4037 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4038 ("swap_in_progress", err.to_string())
4039 }
4040 _ => (
4041 "target_unavailable",
4042 format!("failed to restart module_id '{module_id}': {err}"),
4043 ),
4044 };
4045 return Ok(vec![control_error_frame(&frame, code, message)?]);
4046 }
4047
4048 let response = ClientControlResponse::SupervisorAck {
4049 module_id,
4050 applied: true,
4051 };
4052 Ok(vec![control_response_body_frame(
4053 &frame,
4054 &response,
4055 "ClientControlResponse::SupervisorAck",
4056 )?])
4057 }
4058
4059 async fn handle_supervisor_swap(
4063 &self,
4064 frame: Frame,
4065 module_id: String,
4066 ready_timeout_ms: Option<u64>,
4067 ) -> Result<Vec<Frame>, RouterError> {
4068 let module = {
4075 let operation_lock = self.supervisor.operation_lock();
4076 let _operation_guard = operation_lock.lock().await;
4077 self.supervisor.get(&module_id)
4078 };
4079 let Some(module) = module else {
4080 return Ok(vec![control_error_frame(
4081 &frame,
4082 "unknown_module",
4083 format!("module_id '{module_id}' is not supervised"),
4084 )?]);
4085 };
4086
4087 self.route_outages.mark_operator_action(&module_id);
4088 if let Err(err) = module
4089 .swap(ready_timeout_ms.map(Duration::from_millis))
4090 .await
4091 {
4092 self.route_outages
4093 .operator_action_ended_unrefused(&module_id);
4094 use crate::supervise::SuperviseError;
4095 let message = err.to_string();
4096 let error = match err {
4097 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4098 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4099 code: "swap_refused".to_string(),
4100 message,
4101 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4102 },
4103 SuperviseError::SwapFailed {
4104 arm,
4105 candidate_exit,
4106 ..
4107 } => ErrorBody {
4108 code: "swap_failed".to_string(),
4109 message,
4110 detail: Some(serde_json::json!({
4111 "arm": arm.as_str(),
4112 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4113 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4114 })),
4115 },
4116 _ => ErrorBody::new(
4117 "target_unavailable",
4118 format!("failed to swap module_id '{module_id}': {message}"),
4119 ),
4120 };
4121 return Ok(vec![control_error_body_frame(&frame, error)?]);
4122 }
4123 self.route_outages
4127 .operator_action_ended_unrefused(&module_id);
4128
4129 let response = ClientControlResponse::SupervisorAck {
4130 module_id,
4131 applied: true,
4132 };
4133 Ok(vec![control_response_body_frame(
4134 &frame,
4135 &response,
4136 "ClientControlResponse::SupervisorAck",
4137 )?])
4138 }
4139
4140 async fn handle_supervisor_reload(
4141 &self,
4142 frame: Frame,
4143 module_id: String,
4144 ) -> Result<Vec<Frame>, RouterError> {
4145 let operation_lock = self.supervisor.operation_lock();
4146 let _operation_guard = operation_lock.lock().await;
4147 let Some(module) = self.supervisor.get(&module_id) else {
4148 return Ok(vec![control_error_frame(
4149 &frame,
4150 "unknown_module",
4151 format!("module_id '{module_id}' is not supervised"),
4152 )?]);
4153 };
4154
4155 self.route_outages.mark_operator_action(&module_id);
4156 if let Err(err) = module.reload().await {
4157 self.route_outages
4158 .operator_action_ended_unrefused(&module_id);
4159 let (code, message) = match err {
4160 crate::supervise::SuperviseError::Disabled { .. } => {
4161 ("module_disabled", err.to_string())
4162 }
4163 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4164 ("swap_in_progress", err.to_string())
4165 }
4166 _ => (
4167 "reload_failed",
4168 format!("failed to reload module_id '{module_id}': {err}"),
4169 ),
4170 };
4171 return Ok(vec![control_error_frame(&frame, code, message)?]);
4172 }
4173
4174 let response = ClientControlResponse::SupervisorAck {
4175 module_id,
4176 applied: true,
4177 };
4178 Ok(vec![control_response_body_frame(
4179 &frame,
4180 &response,
4181 "ClientControlResponse::SupervisorAck",
4182 )?])
4183 }
4184
4185 async fn handle_supervisor_rescan(
4186 &self,
4187 frame: Frame,
4188 preview: bool,
4189 ) -> Result<Vec<Frame>, RouterError> {
4190 let Some(context) = self.rescan.clone() else {
4191 return Ok(vec![control_error_frame(
4192 &frame,
4193 "rescan_unavailable",
4194 "the daemon was not started with a reloadable config path".to_string(),
4195 )?]);
4196 };
4197
4198 let operation_lock = self.supervisor.operation_lock();
4199 let _operation_guard = operation_lock.lock().await;
4200 let loaded = match crate::daemon_config::load(&context.config_path) {
4201 Ok(config) => config,
4202 Err(err) => {
4203 return Ok(vec![control_error_frame(
4204 &frame,
4205 "invalid_daemon_config",
4206 format!("supervisor rescan rejected daemon config: {err}"),
4207 )?])
4208 }
4209 };
4210 let Some(config) = loaded else {
4218 return Ok(vec![control_error_frame(
4219 &frame,
4220 "invalid_daemon_config",
4221 format!(
4222 "daemon config not found at {}; refusing to rescan (an absent config would \
4223 retire every supervised module)",
4224 context.config_path.display()
4225 ),
4226 )?]);
4227 };
4228 let (
4229 configured_port,
4230 storage_config,
4231 admission_facts_carrier_module_id,
4232 admission_facts_targets,
4233 scope_authority_owners,
4234 modules,
4235 reserved_capabilities,
4236 ) = (
4237 config.port,
4238 config.storage,
4239 config.admission_facts_carrier_module_id,
4240 config.admission_facts_targets,
4241 config.scope_authority_owners,
4242 config.modules,
4243 config.reserved_capabilities,
4244 );
4245
4246 let mut restart_required = Vec::new();
4255 for section in RestartRequiredSection::ALL {
4256 let changed = match section {
4257 RestartRequiredSection::Port => configured_port != context.configured_port,
4258 RestartRequiredSection::Storage => storage_config != context.storage_config,
4259 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4260 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4261 }
4262 RestartRequiredSection::AdmissionFactsTargets => {
4263 admission_facts_targets != context.admission_facts_targets
4264 }
4265 RestartRequiredSection::ScopeAuthorityOwners => {
4266 scope_authority_owners != context.scope_authority_owners
4267 }
4268 };
4269 if changed {
4270 restart_required.push(section.label().to_string());
4271 }
4272 }
4273 if !restart_required.is_empty() {
4274 warn!(
4275 config_path = %context.config_path.display(),
4276 sections = %restart_required.join(", "),
4277 "daemon config changed outside the modules section; restart the daemon to apply those changes"
4278 );
4279 }
4280
4281 for configured in &modules {
4282 if let Err(err) = validate_spec(&configured.module_spec()) {
4283 return Ok(vec![control_error_frame(
4284 &frame,
4285 "invalid_daemon_config",
4286 format!("supervisor rescan rejected daemon config: {err}"),
4287 )?]);
4288 }
4289 }
4290
4291 let configured_capabilities = modules
4292 .iter()
4293 .map(|module| (module.module_id.clone(), module.enabled))
4294 .collect::<Vec<_>>();
4295 let preview_capability_warnings = if preview {
4296 let (_, registrations) = self.runtime_capability_snapshot()?;
4297 let current_modules = self
4298 .supervisor
4299 .list()
4300 .into_iter()
4301 .map(|module| module.module_id().to_string())
4302 .collect::<BTreeSet<_>>();
4303 let resulting_modules = configured_capabilities.clone();
4304 let removed = current_modules
4305 .into_iter()
4306 .filter(|module_id| {
4307 !resulting_modules
4308 .iter()
4309 .any(|(configured_id, _)| configured_id == module_id)
4310 })
4311 .collect::<Vec<_>>();
4312 self.capability_evaluator.preview_removal_warnings(
4313 resulting_modules,
4314 &removed,
4315 ®istrations,
4316 )
4317 } else {
4318 Vec::new()
4319 };
4320 let result = match self
4321 .reconcile_supervised_modules(&context.supervisor, modules, preview)
4322 .await
4323 {
4324 Ok(result) => result,
4325 Err(message) => {
4326 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4327 }
4328 };
4329 if !preview {
4330 self.capability_evaluator
4331 .configure(configured_capabilities, reserved_capabilities);
4332 self.capability_evaluator.wake_deadline_loop();
4333 self.refresh_capability_requirements();
4334 }
4335 let mut result = result;
4336 result.restart_required = restart_required;
4337 result.capability_warnings = preview_capability_warnings;
4338 let response = ClientControlResponse::SupervisorRescan { result };
4339 Ok(vec![control_response_body_frame(
4340 &frame,
4341 &response,
4342 "ClientControlResponse::SupervisorRescan",
4343 )?])
4344 }
4345
4346 async fn handle_supervisor_release_reserved(
4347 &self,
4348 frame: Frame,
4349 module_id: String,
4350 ) -> Result<Vec<Frame>, RouterError> {
4351 let Some(context) = self.rescan.clone() else {
4352 return Ok(vec![control_error_frame(
4353 &frame,
4354 "release_unavailable",
4355 "reserved-id release requires a daemon started with a reloadable config path",
4356 )?]);
4357 };
4358 let operation_lock = self.supervisor.operation_lock();
4359 let _operation_guard = operation_lock.lock().await;
4360 let loaded = match crate::daemon_config::load(&context.config_path) {
4361 Ok(Some(config)) => config,
4362 Ok(None) => {
4363 return Ok(vec![control_error_frame(
4364 &frame,
4365 "invalid_daemon_config",
4366 format!(
4367 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4368 context.config_path.display()
4369 ),
4370 )?])
4371 }
4372 Err(err) => {
4373 return Ok(vec![control_error_frame(
4374 &frame,
4375 "invalid_daemon_config",
4376 format!("unable to verify reserved-id release against daemon config: {err}"),
4377 )?])
4378 }
4379 };
4380 if loaded
4381 .modules
4382 .iter()
4383 .any(|configured| configured.module_id == module_id)
4384 {
4385 return Ok(vec![control_error_frame(
4386 &frame,
4387 "reserved_module_configured",
4388 format!(
4389 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4390 ),
4391 )?]);
4392 }
4393 if !self.supervisor.release_retained_reserved_gate(&module_id) {
4394 return Ok(vec![control_error_frame(
4395 &frame,
4396 "reserved_gate_not_retained",
4397 format!(
4398 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4399 ),
4400 )?]);
4401 }
4402
4403 let response = ClientControlResponse::SupervisorAck {
4404 module_id,
4405 applied: true,
4406 };
4407 Ok(vec![control_response_body_frame(
4408 &frame,
4409 &response,
4410 "ClientControlResponse::SupervisorAck",
4411 )?])
4412 }
4413
4414 async fn reconcile_supervised_modules(
4423 &self,
4424 supervisor: &Supervisor,
4425 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4426 preview: bool,
4427 ) -> Result<SupervisorRescanResult, String> {
4428 let mut current = BTreeMap::new();
4429 for module in self.supervisor.list() {
4430 let (spec, health) = module.configuration().map_err(|err| {
4431 format!(
4432 "failed to read configuration for module_id '{}': {err}",
4433 module.module_id()
4434 )
4435 })?;
4436 let enabled = module
4437 .status()
4438 .map_err(|err| {
4439 format!(
4440 "failed to read status for module_id '{}': {err}",
4441 module.module_id()
4442 )
4443 })?
4444 .enabled;
4445 current.insert(
4446 module.module_id().to_string(),
4447 (module, spec, health, enabled),
4448 );
4449 }
4450 let configured = configured_modules
4451 .into_iter()
4452 .map(|module| (module.module_id.clone(), module))
4453 .collect::<BTreeMap<_, _>>();
4454
4455 let added = configured
4456 .keys()
4457 .filter(|module_id| !current.contains_key(*module_id))
4458 .cloned()
4459 .collect::<Vec<_>>();
4460 let removed = current
4461 .keys()
4462 .filter(|module_id| !configured.contains_key(*module_id))
4463 .cloned()
4464 .collect::<Vec<_>>();
4465 let mut changed_pending_reload = Vec::new();
4466 let mut configuration_changes = BTreeSet::new();
4467 let mut enabled_changes = BTreeSet::new();
4468 let mut unchanged = 0_u32;
4469
4470 for (module_id, configured_module) in &configured {
4471 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4472 else {
4473 continue;
4474 };
4475 let configuration_changed = *current_spec != configured_module.module_spec()
4476 || *current_health != configured_module.health;
4477 let enabled_changed = *current_enabled != configured_module.enabled;
4478 if configuration_changed {
4479 configuration_changes.insert(module_id.clone());
4480 changed_pending_reload.push(module_id.clone());
4481 }
4482 if enabled_changed {
4483 enabled_changes.insert(module_id.clone());
4484 }
4485 if !configuration_changed && !enabled_changed {
4486 unchanged = unchanged.saturating_add(1);
4487 }
4488 }
4489
4490 if preview {
4496 return Ok(SupervisorRescanResult {
4497 added,
4498 removed,
4499 changed_pending_reload,
4500 enabled_changes: enabled_changes.iter().cloned().collect(),
4501 unchanged,
4502 preview: true,
4503 restart_required: Vec::new(),
4507 capability_warnings: Vec::new(),
4508 });
4509 }
4510
4511 for module_id in &removed {
4512 let module = ¤t
4513 .get(module_id)
4514 .expect("removed module came from current supervisor state")
4515 .0;
4516 module.retire().await.map_err(|err| {
4517 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4518 })?;
4519 self.supervisor.record_rescan_removal(module_id);
4553 self.supervisor.retire(module_id);
4554 self.route_outages.forget(module_id);
4555 }
4556
4557 for module_id in configured.keys() {
4558 let Some((module, _, _, _)) = current.get(module_id) else {
4559 continue;
4560 };
4561 let configured_module = configured
4562 .get(module_id)
4563 .expect("configured module id came from configured map");
4564 if configuration_changes.contains(module_id) {
4565 module
4566 .update_configuration(
4567 configured_module.module_spec(),
4568 configured_module.health,
4569 configured_module.drain_timeout_ms,
4570 )
4571 .await
4572 .map_err(|err| {
4573 format!(
4574 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4575 )
4576 })?;
4577 }
4578 if enabled_changes.contains(module_id) {
4579 self.route_outages.mark_operator_action(module_id);
4582 module
4583 .set_enabled(configured_module.enabled)
4584 .await
4585 .map_err(|err| {
4586 self.route_outages.operator_action_ended_unrefused(module_id);
4587 format!(
4588 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4589 configured_module.enabled
4590 )
4591 })?;
4592 }
4593 }
4594
4595 for module_id in &added {
4596 let configured_module = configured
4597 .get(module_id)
4598 .expect("added module id came from configured map");
4599 supervisor
4600 .supervise_configured_with_health(
4601 configured_module.module_spec(),
4602 configured_module.enabled,
4603 configured_module.health,
4604 configured_module.drain_timeout_ms,
4605 configured_module.restart,
4606 )
4607 .map_err(|err| {
4608 format!("failed to add module_id '{module_id}' during rescan: {err}")
4609 })?;
4610 }
4611
4612 Ok(SupervisorRescanResult {
4613 added,
4614 removed,
4615 changed_pending_reload,
4616 enabled_changes: enabled_changes.iter().cloned().collect(),
4617 unchanged,
4618 preview: false,
4619 restart_required: Vec::new(),
4622 capability_warnings: Vec::new(),
4623 })
4624 }
4625
4626 async fn handle_supervisor_set_enabled(
4627 &self,
4628 frame: Frame,
4629 module_id: String,
4630 enabled: bool,
4631 ) -> Result<Vec<Frame>, RouterError> {
4632 let operation_lock = self.supervisor.operation_lock();
4633 let _operation_guard = operation_lock.lock().await;
4634 let Some(module) = self.supervisor.get(&module_id) else {
4635 return Ok(vec![control_error_frame(
4636 &frame,
4637 "unknown_module",
4638 format!("module_id '{module_id}' is not supervised"),
4639 )?]);
4640 };
4641
4642 self.route_outages.mark_operator_action(&module_id);
4645 let applied = match module.set_enabled(enabled).await {
4646 Ok(applied) => applied,
4647 Err(err) => {
4648 self.route_outages
4649 .operator_action_ended_unrefused(&module_id);
4650 return Ok(vec![control_error_frame(
4651 &frame,
4652 "target_unavailable",
4653 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4654 )?]);
4655 }
4656 };
4657 if !applied {
4658 self.route_outages
4661 .operator_action_ended_unrefused(&module_id);
4662 }
4663
4664 self.capability_evaluator.wake_deadline_loop();
4665 self.refresh_capability_requirements();
4666 let response = ClientControlResponse::SupervisorAck { module_id, applied };
4667 Ok(vec![control_response_body_frame(
4668 &frame,
4669 &response,
4670 "ClientControlResponse::SupervisorAck",
4671 )?])
4672 }
4673
4674 async fn handle_supervisor_health_probe(
4675 &self,
4676 frame: Frame,
4677 module_id: String,
4678 ) -> Result<Vec<Frame>, RouterError> {
4679 self.refresh_capability_requirements();
4680 let Some(registration) = self
4681 .registry
4682 .get_module(&module_id)
4683 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4684 else {
4685 return Ok(vec![control_error_frame(
4686 &frame,
4687 "unknown_module",
4688 format!("module_id '{module_id}' is not registered"),
4689 )?]);
4690 };
4691
4692 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4705 {
4706 return Ok(vec![control_error_frame(
4707 &frame,
4708 "health_not_advertised",
4709 format!("module_id '{module_id}' did not advertise health.check"),
4710 )?]);
4711 }
4712
4713 let deadline = Instant::now() + self.health_probe_timeout;
4714 let pending = match self.forwarding.begin_module_control_rpc_for(
4715 &module_id,
4716 MODULE_CONTROL_OP_HEALTH_CHECK,
4717 deadline,
4718 ) {
4719 Ok(pending) => pending,
4720 Err(err) => {
4721 return Ok(vec![control_error_frame(
4722 &frame,
4723 forwarding_error_code(&err),
4724 err.to_string(),
4725 )?])
4726 }
4727 };
4728
4729 let PendingModuleControlRpc {
4730 endpoint,
4731 module_sink,
4732 negotiated_ver,
4733 corr: probe_corr,
4734 receiver,
4735 } = pending;
4736 let mut guard =
4737 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4738 let probe_body =
4739 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4740 RouterError::backend(
4741 0,
4742 frame.header.corr,
4743 format!("failed to encode health.check request: {err}"),
4744 )
4745 })?;
4746 let probe_frame = Frame::build_with_version(
4747 negotiated_ver,
4748 FrameType::Request,
4749 control_flags(),
4750 0,
4751 0,
4752 probe_corr,
4753 probe_body,
4754 )
4755 .map_err(RouterError::FrameBuild)?;
4756
4757 if let Err(err) = module_sink.send(probe_frame).await {
4758 return Ok(vec![control_error_frame(
4759 &frame,
4760 "target_unavailable",
4761 err.to_string(),
4762 )?]);
4763 }
4764
4765 match timeout_at(deadline, receiver).await {
4766 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4767 guard.disarm();
4768 let Some(report) = response.health_report() else {
4769 return Ok(vec![control_error_frame(
4770 &frame,
4771 "invalid_control_body",
4772 "health.check RPC returned a non-health response",
4773 )?]);
4774 };
4775 let HealthReport {
4780 status,
4781 detail,
4782 metrics,
4783 } = report;
4784 let capability_detail = self
4785 .capability_evaluator
4786 .required_problem_detail(&module_id);
4787 let response = ClientControlResponse::SupervisorHealthProbe {
4788 module_id,
4789 status,
4790 detail: append_capability_problem_detail(detail, capability_detail),
4791 metrics,
4792 };
4793 Ok(vec![control_response_body_frame(
4794 &frame,
4795 &response,
4796 "ClientControlResponse::SupervisorHealthProbe",
4797 )?])
4798 }
4799 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4800 guard.disarm();
4801 Ok(vec![control_error_body_frame(&frame, body)?])
4802 }
4803 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4804 guard.disarm();
4805 Ok(vec![control_error_frame(
4806 &frame,
4807 "target_unavailable",
4808 message,
4809 )?])
4810 }
4811 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4812 guard.disarm();
4813 Ok(vec![control_error_frame(
4814 &frame,
4815 "invalid_control_body",
4816 message,
4817 )?])
4818 }
4819 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4820 guard.disarm();
4821 Ok(vec![control_error_frame(
4822 &frame,
4823 "invalid_control_body",
4824 format!("expected module-control op '{expected}', got '{actual}'"),
4825 )?])
4826 }
4827 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4828 guard.disarm();
4829 Ok(vec![control_error_frame(
4830 &frame,
4831 "module_timeout",
4832 format!(
4833 "module_id '{module_id}' answered health.check after {:?}",
4834 self.health_probe_timeout
4835 ),
4836 )?])
4837 }
4838 Ok(Err(_)) => Ok(vec![control_error_frame(
4839 &frame,
4840 "target_unavailable",
4841 "health.check waiter was canceled before the module responded",
4842 )?]),
4843 Err(_) => Ok(vec![control_error_frame(
4844 &frame,
4845 "module_timeout",
4846 format!(
4847 "module_id '{module_id}' did not answer health.check within {:?}",
4848 self.health_probe_timeout
4849 ),
4850 )?]),
4851 }
4852 }
4853
4854 fn supervisor_status(
4855 &self,
4856 module_id: &str,
4857 corr: u64,
4858 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4859 self.supervisor
4860 .get(module_id)
4861 .map(|module| {
4862 let warming = module.is_warming_for_control("status").map_err(|err| {
4863 RouterError::backend(
4864 0,
4865 corr,
4866 format!(
4867 "failed to read supervisor warming state for module_id '{module_id}': {err}"
4868 ),
4869 )
4870 })?;
4871 module.status_for_control("status").map_err(|err| {
4872 RouterError::backend(
4873 0,
4874 corr,
4875 format!(
4876 "failed to read supervisor status for module_id '{module_id}': {err}"
4877 ),
4878 )
4879 }).map(|status| (status, warming))
4880 })
4881 .transpose()
4882 }
4883
4884 fn guard_module_control_op(
4885 &self,
4886 frame: &Frame,
4887 module_id: &str,
4888 op: &str,
4889 ) -> Result<Option<Frame>, RouterError> {
4890 if self.module_grants_op(module_id, op, frame.header.corr)? {
4891 return Ok(None);
4892 }
4893
4894 Ok(Some(control_error_frame(
4895 frame,
4896 "op_not_allowed",
4897 format!("module_id '{module_id}' did not grant control op '{op}'"),
4898 )?))
4899 }
4900
4901 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4902 let Some(registration) = self
4903 .registry
4904 .get_module(module_id)
4905 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4906 else {
4907 return Ok(false);
4908 };
4909 Ok(module_registration_grants_op(®istration.control_ops, op))
4910 }
4911
4912 fn handle_status_update(
4913 &self,
4914 endpoint: ModuleEndpointId,
4915 frame: Frame,
4916 ) -> Result<Vec<Frame>, RouterError> {
4917 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4918 Ok(update) => update,
4919 Err(err) => {
4920 if is_known_module_push_op(&frame.body) {
4925 return Ok(vec![control_error_frame(
4926 &frame,
4927 "invalid_control_body",
4928 format!("malformed module control push body: {err}"),
4929 )?]);
4930 }
4931 return Ok(Vec::new());
4932 }
4933 };
4934
4935 match update {
4936 ModuleControlPush::RouteStatus {
4937 route_channel,
4938 route_epoch,
4939 status,
4940 } => {
4941 self.forwarding
4942 .cache_status(endpoint, route_channel, route_epoch, status)
4943 .map_err(RouterError::Forwarding)?;
4944 }
4945 }
4946 Ok(Vec::new())
4947 }
4948
4949 fn handle_route_poll(
4950 &self,
4951 ctx: &RouteCtx,
4952 frame: Frame,
4953 route_channel: u16,
4954 route_epoch: u32,
4955 kind: PollKind,
4956 ) -> Result<Vec<Frame>, RouterError> {
4957 let snapshot = self
4958 .forwarding
4959 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4960 .map_err(RouterError::Forwarding)?;
4961 let response = match (kind, snapshot) {
4962 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4963 ClientControlResponse::RoutePoll {
4964 route_channel,
4965 route_epoch,
4966 status,
4967 live: None,
4968 }
4969 }
4970 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4971 route_channel,
4972 route_epoch,
4973 status: None,
4974 live: None,
4975 },
4976 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
4977 let live = self
5003 .process_liveness
5004 .as_ref()
5005 .and_then(|source| source.process_live(&module_id))
5006 .unwrap_or(true);
5007 ClientControlResponse::RoutePoll {
5008 route_channel,
5009 route_epoch,
5010 status: None,
5011 live: Some(live),
5012 }
5013 }
5014 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5015 route_channel,
5016 route_epoch,
5017 status: None,
5018 live: Some(false),
5019 },
5020 };
5021
5022 Ok(vec![control_response_body_frame(
5023 &frame,
5024 &response,
5025 "ClientControlResponse::RoutePoll",
5026 )?])
5027 }
5028
5029 pub(crate) fn observe_module_control_completion(
5030 &self,
5031 completion: ModuleControlRpcCompletion,
5032 ) -> bool {
5033 match completion {
5034 ModuleControlRpcCompletion::Unknown => false,
5035 ModuleControlRpcCompletion::Settled => true,
5036 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5037 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5038 info!(
5039 module_id = %module_id,
5040 latency_ms,
5041 "late health.check answer proves the module is alive"
5042 );
5043 match self
5044 .supervisor
5045 .record_late_health_answer(&module_id, latency_ms)
5046 {
5047 Ok(true) => {}
5048 Ok(false) => debug!(
5049 module_id = %module_id,
5050 latency_ms,
5051 "late health.check answer has no active supervisor snapshot"
5052 ),
5053 Err(err) => warn!(
5054 module_id = %module_id,
5055 latency_ms,
5056 error = %err,
5057 "failed to record late health.check answer"
5058 ),
5059 }
5060 true
5061 }
5062 }
5063 }
5064
5065 fn refuse_to_end_module_connection_for_a_client(
5086 &self,
5087 module_connection_id: ConnectionId,
5088 corr: u64,
5089 err: ForwardingError,
5090 ) -> Result<(), RouterError> {
5091 if let ForwardingError::ConnectionClosing { connection_id } = err {
5092 if connection_id != module_connection_id {
5093 warn!(
5094 module_connection_id = module_connection_id.get(),
5095 client_connection_id = connection_id.get(),
5096 corr,
5097 "dropping a route.bind response for a closing client; the module connection keeps serving"
5098 );
5099 return Ok(());
5100 }
5101 }
5102 Err(RouterError::Forwarding(err))
5103 }
5104
5105 fn handle_module_relay_response(
5106 &self,
5107 connection_id: ConnectionId,
5108 frame: Frame,
5109 ) -> Result<Vec<Frame>, RouterError> {
5110 let mut secondary_error = None;
5111 let outcome = match frame.header.ty {
5112 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5113 Ok(probe) if probe.op == "route.bind" => {
5114 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5115 Ok(ModuleControlResponse::RouteBindAck {}) => {
5116 RouteBindRelayOutcome::Accepted
5117 }
5118 Ok(other) => {
5119 let message =
5120 format!("route.bind response carried unexpected body: {other:?}");
5121 secondary_error = Some(control_error_frame(
5122 &frame,
5123 "invalid_control_body",
5124 message.clone(),
5125 )?);
5126 RouteBindRelayOutcome::ModuleGone(message)
5127 }
5128 Err(err) => {
5129 let message = format!("malformed route.bind response body: {err}");
5130 secondary_error = Some(control_error_frame(
5131 &frame,
5132 "invalid_control_body",
5133 message.clone(),
5134 )?);
5135 RouteBindRelayOutcome::ModuleGone(message)
5136 }
5137 }
5138 }
5139 Ok(probe) => {
5140 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5141 {
5142 Ok(response) => ModuleControlRpcOutcome::Response(response),
5143 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5144 "malformed {} response body: {err}",
5145 probe.op
5146 )),
5147 };
5148 let completion = self
5149 .forwarding
5150 .complete_module_control_rpc(
5151 connection_id,
5152 frame.header.corr,
5153 Some(&probe.op),
5154 outcome,
5155 )
5156 .map_err(RouterError::Forwarding)?;
5157 if !self.observe_module_control_completion(completion) {
5158 debug!(
5159 connection_id = connection_id.get(),
5160 corr = frame.header.corr,
5161 op = %probe.op,
5162 "dropping late or unknown module-control RPC response"
5163 );
5164 }
5165 return Ok(Vec::new());
5166 }
5167 Err(err) => {
5168 if let Some(expected_op) = self
5169 .forwarding
5170 .pending_module_control_op(connection_id, frame.header.corr)
5171 .map_err(RouterError::Forwarding)?
5172 {
5173 let completion = self
5174 .forwarding
5175 .complete_module_control_rpc(
5176 connection_id,
5177 frame.header.corr,
5178 None,
5179 ModuleControlRpcOutcome::MalformedResponse(format!(
5180 "malformed {expected_op} response body: {err}"
5181 )),
5182 )
5183 .map_err(RouterError::Forwarding)?;
5184 if !self.observe_module_control_completion(completion) {
5185 debug!(
5186 connection_id = connection_id.get(),
5187 corr = frame.header.corr,
5188 "dropping late malformed module-control RPC response"
5189 );
5190 }
5191 return Ok(Vec::new());
5192 }
5193 let message = format!("malformed route.bind response body: {err}");
5194 secondary_error = Some(control_error_frame(
5195 &frame,
5196 "invalid_control_body",
5197 message.clone(),
5198 )?);
5199 RouteBindRelayOutcome::ModuleGone(message)
5200 }
5201 },
5202 FrameType::Error => {
5203 if self
5204 .forwarding
5205 .pending_module_control_op(connection_id, frame.header.corr)
5206 .map_err(RouterError::Forwarding)?
5207 .is_some()
5208 {
5209 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5210 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5211 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5212 "malformed module-control ERROR body: {err}"
5213 )),
5214 };
5215 let completion = self
5216 .forwarding
5217 .complete_module_control_rpc(
5218 connection_id,
5219 frame.header.corr,
5220 None,
5221 outcome,
5222 )
5223 .map_err(RouterError::Forwarding)?;
5224 if !self.observe_module_control_completion(completion) {
5225 debug!(
5226 connection_id = connection_id.get(),
5227 corr = frame.header.corr,
5228 "dropping late or unknown module-control RPC error"
5229 );
5230 }
5231 return Ok(Vec::new());
5232 }
5233 match serde_json::from_slice::<ErrorBody>(&frame.body) {
5234 Ok(body) => RouteBindRelayOutcome::Rejected(body),
5235 Err(err) => {
5236 let message = format!("malformed route.bind ERROR body: {err}");
5237 secondary_error = Some(control_error_frame(
5238 &frame,
5239 "invalid_control_body",
5240 message.clone(),
5241 )?);
5242 RouteBindRelayOutcome::ModuleGone(message)
5243 }
5244 }
5245 }
5246 ty => {
5247 return Ok(vec![control_error_frame(
5248 &frame,
5249 "unsupported_control_frame",
5250 format!("unsupported module channel-0 frame {ty:?}"),
5251 )?])
5252 }
5253 };
5254
5255 let settled =
5256 self.forwarding
5257 .complete_pending_relay(connection_id, frame.header.corr, outcome);
5258 let completion = match settled {
5259 Ok(completion) => completion,
5260 Err(err) => {
5261 self.refuse_to_end_module_connection_for_a_client(
5262 connection_id,
5263 frame.header.corr,
5264 err,
5265 )?;
5266 return Ok(secondary_error.into_iter().collect());
5267 }
5268 };
5269 if let Some(target) = completion.abandoned.as_ref() {
5270 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5271 }
5272 if !completion.settled {
5273 debug!(
5274 connection_id = connection_id.get(),
5275 corr = frame.header.corr,
5276 frame_type = ?frame.header.ty,
5277 "dropping late or unknown route.bind relay response"
5278 );
5279 }
5280 Ok(secondary_error.into_iter().collect())
5281 }
5282
5283 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5284 debug!(connection_id = connection_id.get(), "handling GOODBYE");
5285 let registrations = self
5286 .deregister_connection(connection_id)
5287 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5288 let released_routes = self
5289 .forwarding
5290 .cleanup_connection(connection_id)
5291 .map_err(RouterError::Forwarding)?;
5292 self.emit_route_goodbyes(released_routes);
5293 if !registrations.is_empty() {
5295 crate::supervise::notify_registration_release();
5296 }
5297 Ok(Vec::new())
5298 }
5299}
5300
5301impl Default for ControlHandler {
5302 fn default() -> Self {
5303 Self::new(Arc::new(Registry::default()))
5304 }
5305}
5306
5307impl crate::supervise::SwapPromotionObserver for ControlHandler {
5308 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5309 self.apply_registration_capabilities(registration);
5310 }
5311}
5312
5313fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5314 CapabilityRequirementStatus {
5315 consumer: status.consumer,
5316 capability: status.capability,
5317 need: match status.need {
5318 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5319 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5320 },
5321 verdict: status.verdict.as_str().to_string(),
5322 episode_seq: status.episode_seq,
5323 config_satisfiable: status.config_satisfiable,
5324 runtime_available: status.runtime_available,
5325 detail: status.detail,
5326 }
5327}
5328
5329fn append_capability_problem_detail(
5330 detail: Option<String>,
5331 capability_detail: Option<String>,
5332) -> Option<String> {
5333 match (detail, capability_detail) {
5334 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5335 (Some(detail), None) => Some(detail),
5336 (None, Some(capability_detail)) => Some(capability_detail),
5337 (None, None) => None,
5338 }
5339}
5340
5341fn subc_ops() -> Vec<String> {
5342 SUBC_CONTROL_OPS
5343 .iter()
5344 .map(|op| (*op).to_string())
5345 .collect()
5346}
5347
5348fn module_subc_ops() -> Vec<String> {
5349 SUBC_CONTROL_OPS
5350 .iter()
5351 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5352 .map(|op| (*op).to_string())
5353 .collect()
5354}
5355
5356#[cfg(test)]
5357fn module_baseline_control_ops() -> Vec<String> {
5358 MODULE_BASELINE_CONTROL_OPS
5359 .iter()
5360 .map(|op| (*op).to_string())
5361 .collect()
5362}
5363
5364fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5365 let mut seen = HashSet::new();
5366 let mut effective = Vec::new();
5367 for op in MODULE_BASELINE_CONTROL_OPS {
5368 if seen.insert((*op).to_string()) {
5369 effective.push((*op).to_string());
5370 }
5371 }
5372 for op in declared.unwrap_or_default() {
5373 if seen.insert(op.clone()) {
5374 effective.push(op);
5375 }
5376 }
5377 effective
5378}
5379
5380fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5381 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5382}
5383
5384fn target_module_id(target: &RouteTarget) -> &str {
5385 match target {
5386 RouteTarget::ToolProvider { module_id }
5387 | RouteTarget::ManagementSurface { module_id }
5388 | RouteTarget::InternalService { module_id, .. } => module_id,
5389 }
5390}
5391
5392fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5393 roles.iter().any(|role| match (target, role) {
5394 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5395 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5396 (
5397 RouteTarget::InternalService { service_id, .. },
5398 ProviderRole::InternalService {
5399 service_id: provided,
5400 ..
5401 },
5402 ) => service_id == provided,
5403 _ => false,
5404 })
5405}
5406
5407fn is_routable_role(role: &ProviderRole) -> bool {
5408 matches!(
5409 role,
5410 ProviderRole::ToolProvider { .. }
5411 | ProviderRole::ManagementSurface { .. }
5412 | ProviderRole::InternalService { .. }
5413 )
5414}
5415
5416#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5417enum ControlRequestBodyError {
5418 UnknownOp,
5419 InvalidBody,
5420}
5421
5422#[derive(Debug, Deserialize)]
5423struct ControlOpProbe {
5424 op: String,
5425}
5426
5427const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5430
5431fn is_known_module_push_op(body: &[u8]) -> bool {
5432 serde_json::from_slice::<ControlOpProbe>(body)
5433 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5434 .unwrap_or(false)
5435}
5436
5437fn is_known_module_request_op(body: &[u8]) -> bool {
5438 serde_json::from_slice::<ControlOpProbe>(body)
5439 .map(|probe| is_module_to_subc_op(&probe.op))
5440 .unwrap_or(false)
5441}
5442
5443fn is_module_to_subc_op(op: &str) -> bool {
5444 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5445}
5446
5447fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5448 debug!(
5449 op = %op,
5450 connection_id = connection_id.get(),
5451 corr,
5452 "control dispatch"
5453 );
5454}
5455
5456fn log_slow_control_dispatch(
5457 dispatch_started_at: Option<StdInstant>,
5458 op: &'static str,
5459 connection_id: ConnectionId,
5460 corr: u64,
5461) {
5462 let Some(dispatch_started_at) = dispatch_started_at else {
5463 return;
5464 };
5465 let elapsed = dispatch_started_at.elapsed();
5466 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5467 warn!(
5468 op = %op,
5469 connection_id = connection_id.get(),
5470 corr,
5471 elapsed_ms = elapsed.as_millis() as u64,
5472 "slow control dispatch"
5473 );
5474 }
5475}
5476
5477fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5478 match request {
5479 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5480 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5481 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5482 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5483 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5484 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5485 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5486 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5487 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5488 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5489 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5490 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5491 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5492 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5493 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5494 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5495 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5496 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5497 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5498 }
5499}
5500
5501fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5502 match request {
5503 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5504 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5505 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
5506 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
5507 }
5508}
5509
5510fn parse_client_control_request(
5511 body: &[u8],
5512) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5513 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5514 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5515 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5516 ControlRequestBodyError::InvalidBody
5517 }
5518 Ok(_) => ControlRequestBodyError::UnknownOp,
5519 Err(_) => ControlRequestBodyError::InvalidBody,
5520 };
5521 (err, classification)
5522 })
5523}
5524
5525fn parse_module_control_request_from_module(
5526 body: &[u8],
5527) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5528 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5529 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5530 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
5531 Ok(_) => ControlRequestBodyError::UnknownOp,
5532 Err(_) => ControlRequestBodyError::InvalidBody,
5533 };
5534 (err, classification)
5535 })
5536}
5537
5538#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5539enum ProviderRoleKind {
5540 ToolProvider,
5541 PipelineStage,
5542 ManagementSurface,
5543 InternalService,
5544}
5545
5546fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5547 match role {
5548 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5549 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5550 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5551 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5552 }
5553}
5554
5555fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5556 roles.iter().map(provider_role_kind).collect()
5557}
5558
5559const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
5562
5563#[derive(Debug, Default, PartialEq, Eq)]
5565struct ScopeOutcomeCounts {
5566 created: usize,
5567 replaced: usize,
5568 updated: usize,
5569 unchanged: usize,
5570 refused: usize,
5571}
5572
5573impl ScopeOutcomeCounts {
5574 fn of(results: &[ScopeRecordResult]) -> Self {
5575 let mut counts = Self::default();
5576 for result in results {
5577 let slot = match result.outcome {
5578 ScopeRecordOutcome::Created => &mut counts.created,
5579 ScopeRecordOutcome::Replaced => &mut counts.replaced,
5580 ScopeRecordOutcome::Updated => &mut counts.updated,
5581 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
5582 ScopeRecordOutcome::Refused => &mut counts.refused,
5583 };
5584 *slot += 1;
5585 }
5586 counts
5587 }
5588}
5589
5590#[cfg(test)]
5591mod scope_outcome_count_tests {
5592 use super::*;
5593
5594 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
5595 ScopeRecordResult {
5596 scope_ref: "r".to_string(),
5597 scope_epoch: 1,
5598 outcome,
5599 code: None,
5600 message: None,
5601 version: None,
5602 parent_state: None,
5603 }
5604 }
5605
5606 #[test]
5609 fn every_outcome_is_counted_in_its_own_field() {
5610 let results = [
5611 result(ScopeRecordOutcome::Created),
5612 result(ScopeRecordOutcome::Created),
5613 result(ScopeRecordOutcome::Replaced),
5614 result(ScopeRecordOutcome::Updated),
5615 result(ScopeRecordOutcome::Unchanged),
5616 result(ScopeRecordOutcome::Refused),
5617 result(ScopeRecordOutcome::Refused),
5618 result(ScopeRecordOutcome::Refused),
5619 ];
5620 assert_eq!(
5621 ScopeOutcomeCounts::of(&results),
5622 ScopeOutcomeCounts {
5623 created: 2,
5624 replaced: 1,
5625 updated: 1,
5626 unchanged: 1,
5627 refused: 3,
5628 }
5629 );
5630 }
5631}
5632
5633fn capability_census_trigger(
5637 old: Option<&CapabilityDeclarations>,
5638 new: Option<&CapabilityDeclarations>,
5639) -> bool {
5640 let old_provides = old
5641 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5642 .unwrap_or_default();
5643 let old_denies = old
5644 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5645 .unwrap_or_default();
5646 let new = new.cloned().unwrap_or(CapabilityDeclarations {
5647 provides: Vec::new(),
5648 requires: Vec::new(),
5649 must_never_reach: Vec::new(),
5650 });
5651
5652 new.provides
5653 .iter()
5654 .any(|capability| !old_provides.contains(capability))
5655 || new
5656 .must_never_reach
5657 .iter()
5658 .any(|capability| !old_denies.contains(capability))
5659}
5660
5661fn denied_capability<'a>(
5664 opening_manifest: &'a ModuleManifest,
5665 target_manifest: &ModuleManifest,
5666) -> Option<&'a str> {
5667 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5668 let target_capabilities = target_manifest.capabilities.as_ref()?;
5669 opening_capabilities
5670 .must_never_reach
5671 .iter()
5672 .find(|denied| {
5673 target_capabilities
5674 .provides
5675 .iter()
5676 .any(|provided| provided == *denied)
5677 })
5678 .map(String::as_str)
5679}
5680
5681fn catalog_update_frozen_field_message(
5682 registered: &ModuleManifest,
5683 provides: &[ProviderRole],
5684) -> Option<String> {
5685 let old_has_provides = !registered.provides.is_empty();
5686 let new_has_provides = !provides.is_empty();
5687 if old_has_provides != new_has_provides {
5688 return Some(format!(
5689 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5690 registered.module_id
5691 ));
5692 }
5693
5694 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
5695 return Some(format!(
5696 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5697 registered.module_id
5698 ));
5699 }
5700
5701 let registered_concurrency = manifest_concurrency(registered);
5702 let mut candidate = registered.clone();
5703 candidate.provides = provides.to_vec();
5704 let candidate_concurrency = manifest_concurrency(&candidate);
5705 if candidate_concurrency != registered_concurrency {
5706 return Some(format!(
5707 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5708 registered.module_id, registered_concurrency, candidate_concurrency
5709 ));
5710 }
5711
5712 None
5715}
5716
5717fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5718 manifest.provides.iter().any(is_routable_role)
5719}
5720
5721fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5727 manifest
5728 .provides
5729 .iter()
5730 .find_map(|provider| match provider {
5731 ProviderRole::ToolProvider { concurrency, .. }
5732 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5733 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5734 })
5735 .unwrap_or(Concurrency::ModuleManaged)
5736}
5737
5738fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5745 let has_management_surface = manifest
5746 .provides
5747 .iter()
5748 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5749 if !has_management_surface {
5750 return false;
5751 }
5752 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5753 return false;
5754 };
5755 let Some(provides) = raw
5756 .get("manifest")
5757 .and_then(|manifest| manifest.get("provides"))
5758 .and_then(serde_json::Value::as_array)
5759 else {
5760 return false;
5761 };
5762 provides.iter().any(|role| {
5767 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5768 && role.get("concurrency").is_none()
5769 })
5770}
5771
5772fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5773 if peer_version != PROTOCOL_VERSION {
5774 return Err(format!(
5775 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5776 ));
5777 }
5778 Ok(PROTOCOL_VERSION)
5779}
5780
5781fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5782 Frame::build_with_version(
5783 response_version(frame),
5784 FrameType::Pong,
5785 frame.header.flags,
5786 0,
5787 0,
5788 frame.header.corr,
5789 Vec::new(),
5790 )
5791 .map_err(RouterError::FrameBuild)
5792}
5793
5794fn control_error_frame(
5795 frame: &Frame,
5796 code: &'static str,
5797 message: impl Into<String>,
5798) -> Result<Frame, RouterError> {
5799 control_error_body_frame(
5800 frame,
5801 ErrorBody {
5802 code: code.to_string(),
5803 message: message.into(),
5804 detail: None,
5805 },
5806 )
5807}
5808
5809fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5810 let body = serde_json::to_vec(&error).map_err(|err| {
5811 RouterError::backend(
5812 0,
5813 frame.header.corr,
5814 format!("failed to encode control ERROR: {err}"),
5815 )
5816 })?;
5817
5818 Frame::build_with_version(
5819 response_version(frame),
5820 FrameType::Error,
5821 control_flags(),
5822 0,
5823 0,
5824 frame.header.corr,
5825 body,
5826 )
5827 .map_err(RouterError::FrameBuild)
5828}
5829
5830fn control_response_body_frame<T: Serialize>(
5831 frame: &Frame,
5832 reply: &T,
5833 label: &'static str,
5834) -> Result<Frame, RouterError> {
5835 let body = serde_json::to_vec(reply).map_err(|err| {
5836 RouterError::backend(
5837 0,
5838 frame.header.corr,
5839 format!("failed to encode {label}: {err}"),
5840 )
5841 })?;
5842
5843 Frame::build_with_version(
5844 response_version(frame),
5845 FrameType::Response,
5846 control_flags(),
5847 0,
5848 0,
5849 frame.header.corr,
5850 body,
5851 )
5852 .map_err(RouterError::FrameBuild)
5853}
5854
5855fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5885 match err {
5886 ForwardingError::NoModuleConnection => "target_unavailable",
5887 ForwardingError::ModuleReloading { .. } => "module_reloading",
5888 ForwardingError::ClientRouteChannelExhausted { .. }
5889 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5890 ForwardingError::StaleModuleEndpoint
5891 | ForwardingError::UnknownReservation { .. }
5892 | ForwardingError::ConnectionClosing { .. }
5893 | ForwardingError::ClientEgressClosed { .. }
5894 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5895 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5898 ForwardingError::RelayCorrelationExhausted
5899 | ForwardingError::RouteOpenBuild(_)
5900 | ForwardingError::Poisoned => "forwarding_error",
5901 }
5902}
5903
5904fn response_version(frame: &Frame) -> u8 {
5905 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5906 frame.header.ver
5907 } else {
5908 PROTOCOL_VERSION
5909 }
5910}
5911
5912fn control_flags() -> Flags {
5913 Flags::new(false, Priority::Passive, false)
5914}
5915
5916fn send_goodbye_target_best_effort(
5921 counters: &DaemonCounters,
5922 target: &GoodbyeTarget,
5923 context: &'static str,
5924) {
5925 let Ok(frame) = Frame::build_with_version(
5926 target.negotiated_ver,
5927 FrameType::Goodbye,
5928 control_flags(),
5929 target.channel,
5930 target.epoch,
5931 0,
5932 Vec::new(),
5933 ) else {
5934 return;
5935 };
5936 crate::forwarding::send_module_route_goodbye(
5937 counters,
5938 &target.sink,
5939 frame,
5940 target.module_id.as_deref(),
5941 context,
5942 );
5943}
5944
5945pub(crate) fn send_route_control_pushes(
5946 forwarding: &ForwardingTable,
5947 routes: Vec<EndpointRoute>,
5948 push: ClientControlPush,
5949) {
5950 let body = match serde_json::to_vec(&push) {
5951 Ok(body) => body,
5952 Err(err) => {
5953 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5954 return;
5955 }
5956 };
5957 let mut targets = Vec::new();
5958 for route in routes {
5959 let target = route.goodbye_target;
5960 if let Some(existing) = targets
5961 .iter()
5962 .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5963 {
5964 debug_assert_eq!(
5965 existing.negotiated_ver, target.negotiated_ver,
5966 "one connection cannot negotiate multiple frame versions"
5967 );
5968 continue;
5969 }
5970 targets.push(target);
5971 }
5972 for target in targets {
5973 let frame = match Frame::build_with_version(
5974 target.negotiated_ver,
5975 FrameType::Push,
5976 control_flags(),
5977 0,
5978 0,
5979 0,
5980 body.clone(),
5981 ) {
5982 Ok(frame) => frame,
5983 Err(err) => {
5984 warn!(
5985 route_channel = target.channel,
5986 error = %err,
5987 "failed to build route lifecycle control PUSH frame"
5988 );
5989 continue;
5990 }
5991 };
5992 if let Err(err) = target.sink.try_send(frame) {
5993 if target.close_on_delivery_failure() {
5994 warn!(
5995 target_connection_id = target.connection_id.get(),
5996 route_channel = target.channel,
5997 error = %err,
5998 "route lifecycle control PUSH was not delivered to client; closing target connection"
5999 );
6000 let _ = forwarding.escalate_client_delivery_failure(
6001 target.connection_id,
6002 target.channel,
6003 target.epoch,
6004 CloseReason::new(
6005 "route_lifecycle_push_delivery_failed",
6006 format!(
6007 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6008 target.channel
6009 ),
6010 ),
6011 crate::forwarding::UndeliveredFrame {
6012 module_id: target.module_id.as_deref(),
6013 sink: &target.sink,
6014 },
6015 );
6016 }
6017 }
6018 }
6019}
6020
6021#[cfg(test)]
6022mod tests {
6023 use std::{
6024 collections::BTreeMap,
6025 fmt,
6026 path::PathBuf,
6027 sync::{Arc, Mutex},
6028 time::Duration,
6029 };
6030 use subc_test_support::TestTempDir;
6031
6032 use serde_json::{json, Value};
6033 use subc_protocol::{
6034 manifest::{
6035 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6036 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6037 },
6038 session::HealthStatus,
6039 FrameType,
6040 };
6041
6042 use super::*;
6043 use crate::{
6044 forwarding::{DataRoute, DataRouteState},
6045 registry::ChannelState,
6046 router::FrameSink,
6047 stderr_tail::DEFAULT_MAX_LINE_BYTES,
6048 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6049 RouteCtx, Router,
6050 };
6051 use tokio::{
6052 sync::mpsc,
6053 time::{sleep, Instant},
6054 };
6055 use tracing::{
6056 field::{Field, Visit},
6057 Event, Subscriber,
6058 };
6059 use tracing_subscriber::{layer::Context, prelude::*, Layer};
6060
6061 fn fake_aft_stub_path() -> PathBuf {
6080 let mut path = std::env::current_exe().expect("current_exe available in tests");
6081 path.pop(); path.pop(); path.push(if cfg!(windows) {
6084 "fake-aft-stub.exe"
6085 } else {
6086 "fake-aft-stub"
6087 });
6088 assert!(
6089 path.exists(),
6090 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6091 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6092 path.display()
6093 );
6094 path
6095 }
6096
6097 fn client_retries(code: &str) -> bool {
6102 subc_protocol::error_codes::is_retryable_route_open(code)
6103 }
6104
6105 #[test]
6116 fn retryability_of_forwarding_codes_matches_the_failure() {
6117 let transient = [
6120 ForwardingError::NoModuleConnection,
6121 ForwardingError::ModuleReloading {
6122 module_id: "m".into(),
6123 },
6124 ForwardingError::StaleModuleEndpoint,
6125 ForwardingError::UnknownReservation {
6126 client_channel: 1,
6127 module_channel: 1,
6128 },
6129 ForwardingError::ConnectionClosing {
6130 connection_id: ConnectionId::new(1),
6131 },
6132 ForwardingError::ClientEgressClosed {
6133 connection_id: ConnectionId::new(1),
6134 },
6135 ForwardingError::ModuleEgressUnavailable {
6136 connection_id: ConnectionId::new(1),
6137 },
6138 ];
6139 for err in transient {
6140 let code = forwarding_error_code(&err);
6141 assert!(
6142 client_retries(code),
6143 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6144 );
6145 }
6146
6147 let permanent = [
6152 ForwardingError::ClientRouteChannelExhausted {
6153 connection_id: ConnectionId::new(1),
6154 },
6155 ForwardingError::ModuleRouteChannelExhausted {
6156 endpoint: ModuleEndpointId {
6157 connection_id: ConnectionId::new(1),
6158 generation: 1,
6159 },
6160 },
6161 ForwardingError::RelayCorrelationExhausted,
6162 ForwardingError::RouteOpenBuild("x".into()),
6163 ForwardingError::Poisoned,
6164 ];
6165 for err in permanent {
6166 let code = forwarding_error_code(&err);
6167 assert!(
6168 !client_retries(code),
6169 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6170 );
6171 }
6172 }
6173
6174 #[tokio::test]
6202 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6203 let handler = ControlHandler::default();
6204 let frame =
6205 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6206
6207 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6211 assert_eq!(
6212 stamped,
6213 Principal::Direct,
6214 "a caller that proved nothing must not be stamped as a supervised module"
6215 );
6216
6217 let forged = handler
6222 .route_open_principal(
6223 &frame,
6224 Some(ConsumerIdentity {
6225 module_id: "aft".to_string(),
6226 launch_nonce: "not-a-real-nonce".to_string(),
6227 }),
6228 )
6229 .unwrap();
6230 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6231 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6232 }
6233
6234 #[test]
6251 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6252 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6253 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6254 let ClientControlRequest::RouteOpen {
6255 consumer_identity, ..
6256 } = parsed
6257 else {
6258 panic!("route.open body must parse as RouteOpen");
6259 };
6260 assert_eq!(
6261 consumer_identity,
6262 Some(ConsumerIdentity {
6263 module_id: "aft".to_string(),
6264 launch_nonce: "n".to_string(),
6265 }),
6266 "the wire field name must reach the value route_open_principal reads"
6267 );
6268 }
6269
6270 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6271 ModuleManifest::builder(module_id, "0.1.0")
6272 .protocol_ver(protocol_ver)
6273 .provides(vec![ProviderRole::ToolProvider {
6274 tools: vec![Tool {
6275 name: "read".to_string(),
6276 description: None,
6277 execution_mode: ExecutionMode::Pure,
6278 schema: json!({"type": "object"}),
6279 }],
6280 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6281 concurrency: Concurrency::ModuleManaged,
6282 emits_push: true,
6283 sub_supervises: true,
6284 }])
6285 .build()
6286 }
6287
6288 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6289 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6290 }
6291
6292 fn hello_frame_with_control_ops(
6293 module_id: &str,
6294 protocol_ver: u8,
6295 corr: u64,
6296 control_ops: Option<Vec<String>>,
6297 ) -> Frame {
6298 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6299 }
6300
6301 fn hello_frame_with_nonce(
6302 module_id: &str,
6303 protocol_ver: u8,
6304 corr: u64,
6305 launch_nonce: Option<&str>,
6306 ) -> Frame {
6307 hello_frame_full(
6308 module_id,
6309 protocol_ver,
6310 corr,
6311 None,
6312 launch_nonce.map(ToOwned::to_owned),
6313 )
6314 }
6315
6316 fn hello_frame_full(
6317 module_id: &str,
6318 protocol_ver: u8,
6319 corr: u64,
6320 control_ops: Option<Vec<String>>,
6321 launch_nonce: Option<String>,
6322 ) -> Frame {
6323 let body = serde_json::to_vec(&ModuleHelloBody {
6324 manifest: manifest(module_id, protocol_ver),
6325 protocol_ver,
6326 control_ops,
6327 launch_nonce,
6328 })
6329 .unwrap();
6330 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6331 }
6332
6333 fn non_routable_hello_frame_with_control_ops(
6334 module_id: &str,
6335 corr: u64,
6336 control_ops: Option<Vec<String>>,
6337 ) -> Frame {
6338 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6339 manifest.provides.clear();
6340 let body = serde_json::to_vec(&ModuleHelloBody {
6341 manifest,
6342 protocol_ver: PROTOCOL_VERSION,
6343 control_ops,
6344 launch_nonce: None,
6345 })
6346 .unwrap();
6347 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6348 }
6349
6350 fn capability_grammar_hello_frame(
6351 capabilities: Value,
6352 runtime_computed: Option<Value>,
6353 corr: u64,
6354 ) -> Frame {
6355 let mut body = serde_json::to_value(ModuleHelloBody {
6356 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6357 protocol_ver: PROTOCOL_VERSION,
6358 control_ops: None,
6359 launch_nonce: None,
6360 })
6361 .expect("HELLO body serializes");
6362 body["manifest"]["capabilities"] = capabilities;
6363 if let Some(runtime_computed) = runtime_computed {
6364 body["runtime_computed"] = runtime_computed;
6365 }
6366 Frame::build(
6367 FrameType::Hello,
6368 control_flags(),
6369 0,
6370 0,
6371 corr,
6372 serde_json::to_vec(&body).expect("HELLO body reserializes"),
6373 )
6374 .expect("HELLO frame builds")
6375 }
6376
6377 fn channel_request(channel: u16, corr: u64) -> Frame {
6378 Frame::build(
6379 FrameType::Request,
6380 Flags::new(true, Priority::Interactive, false),
6381 channel,
6382 0,
6383 corr,
6384 b"opaque".to_vec(),
6385 )
6386 .unwrap()
6387 }
6388
6389 fn route_ctx(
6390 connection_id: ConnectionId,
6391 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
6392 let (tx, rx) = mpsc::channel(8);
6393 (
6394 RouteCtx {
6395 connection_id,
6396 egress: FrameSink::new(tx),
6397 },
6398 rx,
6399 )
6400 }
6401
6402 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
6403 serde_json::from_slice(&frame.body).unwrap()
6404 }
6405
6406 async fn hello_via_sink(
6411 handler: &ControlHandler,
6412 ctx: &RouteCtx,
6413 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6414 hello: Frame,
6415 ) -> Frame {
6416 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
6417 assert!(
6418 replies.is_empty(),
6419 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
6420 );
6421 let ack = rx
6422 .try_recv()
6423 .expect("HELLO_ACK is queued on the module sink")
6424 .frame;
6425 assert_eq!(ack.header.ty, FrameType::HelloAck);
6426 ack
6427 }
6428
6429 fn parse_error(frame: &Frame) -> Value {
6430 serde_json::from_slice(&frame.body).unwrap()
6431 }
6432
6433 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
6434 serde_json::from_slice(&frame.body).unwrap()
6435 }
6436
6437 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
6438 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
6439 route_channel,
6440 route_epoch: 0,
6441 kind,
6442 })
6443 .unwrap();
6444 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6445 }
6446
6447 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
6448 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
6449 module_id: module_id.to_string(),
6450 })
6451 .unwrap();
6452 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6453 }
6454
6455 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
6456 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
6457 }
6458
6459 fn route_open_frame_with_consumer_capabilities(
6460 corr: u64,
6461 module_id: &str,
6462 project_root: TestTempDir,
6463 consumer_capabilities: Option<Vec<String>>,
6464 ) -> Frame {
6465 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6466 target: RouteTarget::ToolProvider {
6467 module_id: module_id.to_string(),
6468 },
6469 identity: BindIdentity::new(
6470 project_root.path().to_path_buf(),
6471 "unit".to_string(),
6472 "session".to_string(),
6473 ),
6474 consumer_identity: None,
6475 consumer_capabilities,
6476 admission_facts: None,
6477 scope: None,
6478 })
6479 .unwrap();
6480 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6481 }
6482
6483 fn route_open_frame_with_admission_facts(
6484 corr: u64,
6485 module_id: &str,
6486 project_root: TestTempDir,
6487 consumer_identity: Option<subc_control::ConsumerIdentity>,
6488 facts: Option<Value>,
6489 ) -> Frame {
6490 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6491 target: RouteTarget::ToolProvider {
6492 module_id: module_id.to_string(),
6493 },
6494 identity: BindIdentity::new(
6495 project_root.path().to_path_buf(),
6496 "unit".to_string(),
6497 format!("session-{corr}"),
6498 ),
6499 consumer_identity,
6500 consumer_capabilities: None,
6501 admission_facts: facts,
6502 scope: None,
6503 })
6504 .unwrap();
6505 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6506 }
6507
6508 #[derive(Clone, Default)]
6509 struct EventCapture {
6510 events: Arc<Mutex<Vec<CapturedEvent>>>,
6511 }
6512
6513 #[derive(Clone, Debug)]
6514 struct CapturedEvent {
6515 target: String,
6516 level: tracing::Level,
6517 fields: BTreeMap<String, String>,
6518 }
6519
6520 impl EventCapture {
6521 fn events(&self) -> Vec<CapturedEvent> {
6522 self.events.lock().unwrap().clone()
6523 }
6524 }
6525
6526 impl<S> Layer<S> for EventCapture
6527 where
6528 S: Subscriber,
6529 {
6530 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6531 let mut visitor = EventFieldVisitor::default();
6532 event.record(&mut visitor);
6533 self.events.lock().unwrap().push(CapturedEvent {
6534 target: event.metadata().target().to_string(),
6535 level: *event.metadata().level(),
6536 fields: visitor.fields,
6537 });
6538 }
6539 }
6540
6541 #[derive(Default)]
6542 struct EventFieldVisitor {
6543 fields: BTreeMap<String, String>,
6544 }
6545
6546 impl Visit for EventFieldVisitor {
6547 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6548 self.fields
6549 .insert(field.name().to_string(), format!("{value:?}"));
6550 }
6551 }
6552
6553 fn health_response(corr: u64, status: HealthStatus) -> Frame {
6554 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6555 status,
6556 detail: Some("warming".to_string()),
6557 metrics: Some(json!({"queue_depth": 3})),
6558 })
6559 .unwrap();
6560 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6561 }
6562
6563 fn route_bind_ack(corr: u64) -> Frame {
6564 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6565 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6566 }
6567
6568 fn unique_project_root(label: &str) -> TestTempDir {
6569 TestTempDir::new(label)
6570 }
6571
6572 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6573 match parse_route_poll(frame) {
6574 ClientControlResponse::RoutePoll {
6575 status: None,
6576 live: Some(live),
6577 ..
6578 } => assert_eq!(live, expected_live),
6579 other => panic!("unexpected route.poll response: {other:?}"),
6580 }
6581 }
6582
6583 fn bind_liveness_route(
6584 registry: &Registry,
6585 forwarding: &ForwardingTable,
6586 module_id: &str,
6587 ) -> (RouteCtx, u16, u32) {
6588 let module_connection = ConnectionId::new(101);
6589 let client_connection = ConnectionId::new(202);
6590 let registration = registry
6591 .register_with_control_ops(
6592 manifest(module_id, PROTOCOL_VERSION),
6593 PROTOCOL_VERSION,
6594 module_connection,
6595 module_baseline_control_ops(),
6596 )
6597 .unwrap();
6598 let (module_tx, _module_rx) = mpsc::channel(8);
6599 let endpoint = forwarding
6600 .register_module_connection(
6601 module_connection,
6602 module_id.to_string(),
6603 PROTOCOL_VERSION,
6604 manifest_concurrency(®istration.manifest),
6605 FrameSink::new(module_tx),
6606 )
6607 .unwrap();
6608 let (client_ctx, _client_rx) = route_ctx(client_connection);
6609 let pending = forwarding
6610 .begin_route_bind_relay_for_test(
6611 client_connection,
6612 client_ctx.egress.clone(),
6613 1,
6614 module_id,
6615 )
6616 .unwrap();
6617 assert_eq!(pending.endpoint, endpoint);
6618 let route_channel = pending.client_channel;
6619 let route_epoch = pending.client_epoch;
6620 forwarding
6621 .complete_pending_relay(
6622 module_connection,
6623 pending.corr,
6624 RouteBindRelayOutcome::Accepted,
6625 )
6626 .unwrap();
6627 (client_ctx, route_channel, route_epoch)
6628 }
6629
6630 struct FakeProcessLiveness {
6631 live: Option<bool>,
6632 }
6633
6634 impl ModuleProcessLiveness for FakeProcessLiveness {
6635 fn process_live(&self, _module_id: &str) -> Option<bool> {
6636 self.live
6637 }
6638 }
6639
6640 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6641 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6642 {
6643 let registry = Arc::new(Registry::default());
6644 let supervisor_handle = SupervisorHandle::new();
6645 let supervisor = Supervisor::new(
6646 Arc::clone(®istry),
6647 RestartPolicy::new(1, Duration::from_millis(10)),
6648 )
6649 .with_handle(supervisor_handle.clone());
6650 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6651 let module = supervisor
6652 .spawn(ModuleSpec {
6653 launch_nonce_env: true,
6654 module_id: "stderr-tail-wire".to_string(),
6655 program: fake_aft_stub_path(),
6656 args: Vec::new(),
6657 env: vec![
6658 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6659 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6660 ],
6661 reserved: false,
6662 reserved_prefixes: Vec::new(),
6663 protocol: ModuleProtocol::Subc,
6664 overlap: Default::default(),
6665 })
6666 .unwrap();
6667
6668 let deadline = Instant::now() + Duration::from_secs(5);
6669 loop {
6670 let tail = module.stderr_tail(None, None);
6671 if tail
6672 .entries
6673 .iter()
6674 .any(|entry| matches!(entry, TailEntry::ProcessStart))
6675 && tail.entries.iter().any(|entry| {
6676 matches!(
6677 entry,
6678 TailEntry::Line {
6679 truncated: true,
6680 ..
6681 }
6682 )
6683 })
6684 {
6685 break;
6686 }
6687 assert!(
6688 Instant::now() < deadline,
6689 "module did not produce a truncated line and restart boundary: {tail:?}"
6690 );
6691 sleep(Duration::from_millis(10)).await;
6692 }
6693
6694 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6695 let request = ClientControlRequest::SupervisorStderrTail {
6696 module_id: "stderr-tail-wire".to_string(),
6697 max_lines: None,
6698 max_bytes: None,
6699 };
6700 let frame = Frame::build(
6701 FrameType::Request,
6702 control_flags(),
6703 0,
6704 0,
6705 1,
6706 serde_json::to_vec(&request).unwrap(),
6707 )
6708 .unwrap();
6709 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6710 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6711 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6712 serde_json::from_slice(&responses[0].body).unwrap()
6713 else {
6714 panic!("expected supervisor.stderr_tail response");
6715 };
6716
6717 assert!(
6718 tail.entries
6719 .iter()
6720 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6721 "the control response lost the restart boundary"
6722 );
6723 let Some(StderrTailEntry::Line {
6724 text,
6725 truncated,
6726 at_ms,
6727 }) = tail.entries.iter().find(|entry| {
6728 matches!(
6729 entry,
6730 StderrTailEntry::Line {
6731 truncated: true,
6732 ..
6733 }
6734 )
6735 })
6736 else {
6737 panic!("the control response lost the truncated line");
6738 };
6739 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6740 assert!(*truncated);
6741 assert!(
6742 at_ms.is_some(),
6743 "the control response lost the line's capture time"
6744 );
6745 }
6746
6747 #[tokio::test(flavor = "current_thread")]
6752 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6753 let dir = TestTempDir::new("terminals-off-worker");
6754 let journal_path = dir.join("terminals.jsonl");
6755 let registry = Arc::new(Registry::default());
6756 let supervisor_handle = SupervisorHandle::new();
6757 let supervisor =
6758 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6759 .with_handle(supervisor_handle.clone())
6760 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6761 let module = supervisor
6762 .spawn(ModuleSpec {
6763 launch_nonce_env: true,
6764 module_id: "terminal-off-worker".to_string(),
6765 program: fake_aft_stub_path(),
6766 args: Vec::new(),
6767 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6768 reserved: false,
6769 reserved_prefixes: Vec::new(),
6770 protocol: ModuleProtocol::Subc,
6771 overlap: Default::default(),
6772 })
6773 .unwrap();
6774 let deadline = Instant::now() + Duration::from_secs(5);
6775 while module.terminal_history().entries.len() != 2 {
6776 assert!(Instant::now() < deadline, "module did not record two exits");
6777 sleep(Duration::from_millis(10)).await;
6778 }
6779
6780 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6781 let handler =
6782 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
6783 let frame = Frame::build(
6784 FrameType::Request,
6785 control_flags(),
6786 0,
6787 0,
6788 1,
6789 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6790 module_id: "terminal-off-worker".to_string(),
6791 })
6792 .unwrap(),
6793 )
6794 .unwrap();
6795 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6796 let spawned_at = std::time::Instant::now();
6797 let read = tokio::spawn({
6798 let handler = Arc::clone(&handler);
6799 async move { handler.handle_control_frame(&ctx, frame).await }
6800 });
6801 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6804 .await
6805 .unwrap()
6806 .expect("the history read reached its pause");
6807 let elapsed = spawned_at.elapsed();
6808 assert!(
6809 elapsed < Duration::from_secs(2) && !read.is_finished(),
6810 "this task could not run while the history read was paused \
6811 (resumed after {elapsed:?}, read finished: {})",
6812 read.is_finished()
6813 );
6814
6815 drop(release);
6816 let responses = read.await.unwrap().unwrap();
6817 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6818 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6819 panic!("expected supervisor.terminals response");
6820 };
6821 assert_eq!(terminals.entries.len(), 2);
6822 assert_eq!(terminals.journal_skipped_lines, 0);
6823 assert_eq!(terminals.journal_read_errors, 0);
6824 }
6825
6826 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6827 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6828 let registry = Arc::new(Registry::default());
6829 let supervisor_handle = SupervisorHandle::new();
6830 let supervisor =
6831 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6832 .with_handle(supervisor_handle.clone());
6833 let module = supervisor
6834 .spawn(ModuleSpec {
6835 launch_nonce_env: true,
6836 module_id: "terminal-golden".to_string(),
6837 program: fake_aft_stub_path(),
6838 args: Vec::new(),
6839 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6840 reserved: false,
6841 reserved_prefixes: Vec::new(),
6842 protocol: ModuleProtocol::Subc,
6843 overlap: Default::default(),
6844 })
6845 .unwrap();
6846
6847 let deadline = Instant::now() + Duration::from_secs(5);
6848 while module.terminal_history().entries.len() != 2 {
6849 assert!(
6850 Instant::now() < deadline,
6851 "module did not retain two terminal exits: {:?}",
6852 module.terminal_history()
6853 );
6854 sleep(Duration::from_millis(10)).await;
6855 }
6856
6857 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6858 let request = ClientControlRequest::SupervisorTerminals {
6859 module_id: "terminal-golden".to_string(),
6860 };
6861 let frame = Frame::build(
6862 FrameType::Request,
6863 control_flags(),
6864 0,
6865 0,
6866 1,
6867 serde_json::to_vec(&request).unwrap(),
6868 )
6869 .unwrap();
6870 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6871 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6872 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6873 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6874 panic!("expected supervisor.terminals response");
6875 };
6876 assert_eq!(terminals.entries.len(), 2);
6877 assert_eq!(terminals.dropped, 0);
6878
6879 let mut rendered = serde_json::to_value(response).unwrap();
6880 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6883 for (index, entry) in rendered["entries"]
6884 .as_array_mut()
6885 .expect("terminal response entries array")
6886 .iter_mut()
6887 .enumerate()
6888 {
6889 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6890 }
6891
6892 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6893 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6894 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6895 if std::env::var_os("UPDATE_GOLDEN").is_some() {
6896 std::fs::write(&golden_path, &serialized).unwrap();
6897 }
6898 let expected: Value =
6899 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6900 assert_eq!(rendered, expected);
6901 }
6902
6903 #[test]
6904 fn hello_registers_manifest_and_returns_ack() {
6905 let registry = Arc::new(Registry::default());
6906 let handler = ControlHandler::new(Arc::clone(®istry));
6907 let conn = ConnectionId::new(1);
6908
6909 let responses = handler
6910 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6911 .unwrap();
6912
6913 assert_eq!(responses.len(), 1);
6914 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6915 assert_eq!(responses[0].header.channel, 0);
6916 assert_eq!(responses[0].header.corr, 7);
6917 let ack = parse_ack(&responses[0]);
6918 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6919 assert!(ack
6920 .subc_capabilities
6921 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6922 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6923 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6924 assert!(ack
6925 .subc_ops
6926 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6927 assert!(ack
6928 .subc_ops
6929 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6930
6931 let registration = registry.get_module("aft").unwrap().unwrap();
6932 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6933 assert_eq!(registration.state, ChannelState::Active);
6934 assert_eq!(registration.connection_id, conn);
6935 assert_eq!(registration.control_ops, module_baseline_control_ops());
6936 }
6937
6938 #[test]
6939 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
6940 let invalid_identifiers = [
6941 ("case_change", "credentials-Provider/v1"),
6942 ("leading_zero", "credentials-provider/v01"),
6943 ("trailing_hyphen", "credentials-provider-/v1"),
6944 ("consecutive_hyphens", "credentials--provider/v1"),
6945 ("uppercase", "Credentials-provider/v1"),
6946 ("missing_v", "credentials-provider/1"),
6947 ("whitespace", "credentials provider/v1"),
6948 ("zero_version", "credentials-provider/v0"),
6949 ("out_of_range_version", "credentials-provider/v4294967296"),
6950 (
6951 "overlength_name",
6952 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
6953 ),
6954 ];
6955 let mut cases = invalid_identifiers
6956 .into_iter()
6957 .map(|(name, identifier)| {
6958 (
6959 format!("identifier_{name}"),
6960 "capabilities.provides[0]".to_string(),
6961 identifier.to_string(),
6962 json!({ "provides": [identifier] }),
6963 None,
6964 )
6965 })
6966 .collect::<Vec<_>>();
6967 cases.extend([
6968 (
6969 "unknown_need".to_string(),
6970 "capabilities.requires[0].need".to_string(),
6971 "deferred".to_string(),
6972 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
6973 None,
6974 ),
6975 (
6976 "duplicate_provides".to_string(),
6977 "capabilities.provides[1]".to_string(),
6978 "credentials-provider/v1".to_string(),
6979 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
6980 None,
6981 ),
6982 (
6983 "duplicate_must_never_reach".to_string(),
6984 "capabilities.must_never_reach[1]".to_string(),
6985 "credentials-provider/v1".to_string(),
6986 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
6987 None,
6988 ),
6989 (
6990 "duplicate_requires_same_need".to_string(),
6991 "capabilities.requires[1]".to_string(),
6992 "credentials-provider/v1".to_string(),
6993 json!({ "requires": [
6994 { "capability": "credentials-provider/v1", "need": "required" },
6995 { "capability": "credentials-provider/v1", "need": "required" }
6996 ] }),
6997 None,
6998 ),
6999 (
7000 "duplicate_requires_conflicting_need".to_string(),
7001 "capabilities.requires[1]".to_string(),
7002 "credentials-provider/v1".to_string(),
7003 json!({ "requires": [
7004 { "capability": "credentials-provider/v1", "need": "required" },
7005 { "capability": "credentials-provider/v1", "need": "optional" }
7006 ] }),
7007 None,
7008 ),
7009 (
7010 "capabilities_root_pointer".to_string(),
7011 "runtime_computed[0]".to_string(),
7012 "/capabilities".to_string(),
7013 json!({}),
7014 Some(json!(["/capabilities"])),
7015 ),
7016 (
7017 "capabilities_descendant_pointer".to_string(),
7018 "runtime_computed[0]".to_string(),
7019 "/capabilities/provides".to_string(),
7020 json!({}),
7021 Some(json!(["/capabilities/provides"])),
7022 ),
7023 (
7024 "malformed_pointer_without_leading_slash".to_string(),
7025 "runtime_computed[0]".to_string(),
7026 "capabilities".to_string(),
7027 json!({}),
7028 Some(json!(["capabilities"])),
7029 ),
7030 (
7031 "malformed_pointer_escape".to_string(),
7032 "runtime_computed[0]".to_string(),
7033 "/roles/~2/tools".to_string(),
7034 json!({}),
7035 Some(json!(["/roles/~2/tools"])),
7036 ),
7037 (
7038 "unknown_capabilities_field".to_string(),
7039 "capabilities.future".to_string(),
7040 "<array>".to_string(),
7041 json!({ "future": [] }),
7042 None,
7043 ),
7044 ]);
7045
7046 for (index, (name, field, value, capabilities, runtime_computed)) in
7047 cases.into_iter().enumerate()
7048 {
7049 let registry = Arc::new(Registry::default());
7050 let handler = ControlHandler::new(Arc::clone(®istry));
7051 let response = handler
7052 .handle_control(
7053 ConnectionId::new((index + 1) as u64),
7054 capability_grammar_hello_frame(
7055 capabilities,
7056 runtime_computed,
7057 index as u64 + 1,
7058 ),
7059 )
7060 .expect("invalid HELLO returns a refusal");
7061
7062 assert_eq!(response.len(), 1, "{name} must emit one refusal");
7063 let error = parse_error(&response[0]);
7064 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7065 let message = error["message"]
7066 .as_str()
7067 .expect("error message is a string");
7068 assert!(
7069 message.contains(&field),
7070 "{name}: field missing from {message}"
7071 );
7072 assert!(
7073 message.contains(&value),
7074 "{name}: value missing from {message}"
7075 );
7076 assert_eq!(
7077 registry
7078 .active_registration_count()
7079 .expect("registry reads"),
7080 0,
7081 "{name}: refused HELLO must not create a catalog entry"
7082 );
7083 }
7084 }
7085
7086 #[test]
7087 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7088 let registry = Arc::new(Registry::default());
7089 let handler = ControlHandler::new(Arc::clone(®istry));
7090 let capabilities = json!({
7091 "provides": ["credentials-provider/v1"],
7092 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7093 "must_never_reach": ["federation-transport/v1"]
7094 });
7095 let response = handler
7096 .handle_control(
7097 ConnectionId::new(99),
7098 capability_grammar_hello_frame(
7099 capabilities.clone(),
7100 Some(json!(["/roles/0/tools"])),
7101 99,
7102 ),
7103 )
7104 .expect("valid HELLO registers");
7105 assert_eq!(response[0].header.ty, FrameType::HelloAck);
7106
7107 let request = Frame::build(
7108 FrameType::Request,
7109 control_flags(),
7110 0,
7111 0,
7112 100,
7113 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7114 .expect("catalog request serializes"),
7115 )
7116 .expect("catalog request frame builds");
7117 let response = handler
7118 .handle_catalog_list(request, None)
7119 .expect("catalog list succeeds");
7120 let ClientControlResponse::CatalogList { modules, .. } =
7121 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7122 else {
7123 panic!("catalog request must return catalog.list");
7124 };
7125 assert_eq!(modules.len(), 1);
7126 assert_eq!(
7127 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7128 capabilities
7129 );
7130 }
7131
7132 #[test]
7133 fn catalog_list_mirrors_management_operation_description() {
7134 let registry = Arc::new(Registry::default());
7135 let handler = ControlHandler::new(Arc::clone(®istry));
7136 let description = "List managed records and return their identifiers and metadata.";
7137 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7138 manifest.provides = vec![ProviderRole::ManagementSurface {
7139 operations: vec![ManagementOperation {
7140 name: "records.list".to_string(),
7141 kind: ManagementOperationKind::Query,
7142 description: Some(description.to_string()),
7143 }],
7144 config_schema: json!({"type": "object"}),
7145 observability: vec![ObservabilitySurface {
7146 name: "records.stats".to_string(),
7147 kind: ObservabilityKind::Snapshot,
7148 }],
7149 identity_scope: vec![IdentityScope::Project],
7150 concurrency: Concurrency::ModuleManaged,
7151 }];
7152 registry
7153 .register_with_control_ops(
7154 manifest,
7155 PROTOCOL_VERSION,
7156 ConnectionId::new(99),
7157 Vec::new(),
7158 )
7159 .expect("described management manifest registers");
7160
7161 let request = Frame::build(
7162 FrameType::Request,
7163 control_flags(),
7164 0,
7165 0,
7166 100,
7167 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7168 .expect("catalog request serializes"),
7169 )
7170 .expect("catalog request frame builds");
7171 let response = handler
7172 .handle_catalog_list(request, None)
7173 .expect("catalog list succeeds");
7174 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7175 assert_eq!(
7176 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7177 "catalog.list must preserve the declared operation description verbatim"
7178 );
7179 }
7180
7181 #[test]
7182 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7183 let registry = Arc::new(Registry::default());
7184 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7185 [("vault".to_string(), true), ("squatter".to_string(), true)],
7186 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7187 );
7188 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7189 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7190 provides: vec!["credentials-provider/v1".to_string()],
7191 requires: Vec::new(),
7192 must_never_reach: Vec::new(),
7193 });
7194 let frame = Frame::build(
7195 FrameType::Hello,
7196 control_flags(),
7197 0,
7198 0,
7199 77,
7200 serde_json::to_vec(&ModuleHelloBody {
7201 manifest: squatter,
7202 protocol_ver: PROTOCOL_VERSION,
7203 control_ops: None,
7204 launch_nonce: None,
7205 })
7206 .expect("HELLO serializes"),
7207 )
7208 .expect("HELLO frame builds");
7209 let response = handler
7210 .handle_control(ConnectionId::new(77), frame)
7211 .expect("reserved claim receives a typed refusal");
7212 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7213 assert_eq!(
7214 registry
7215 .active_registration_count()
7216 .expect("registry reads"),
7217 0,
7218 "a reserved capability refusal must not leave a catalog entry"
7219 );
7220 }
7221
7222 #[test]
7223 fn server_describe_surfaces_required_capability_verdict_fields() {
7224 let registry = Arc::new(Registry::default());
7225 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7226 [
7227 ("consumer".to_string(), true),
7228 ("provider".to_string(), false),
7229 ],
7230 BTreeMap::new(),
7231 );
7232 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
7233 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7234 provides: Vec::new(),
7235 requires: vec![subc_protocol::manifest::CapabilityRequirement {
7236 capability: "credentials-provider/v1".to_string(),
7237 need: subc_protocol::manifest::CapabilityNeed::Required,
7238 }],
7239 must_never_reach: Vec::new(),
7240 });
7241 let hello = Frame::build(
7242 FrameType::Hello,
7243 control_flags(),
7244 0,
7245 0,
7246 78,
7247 serde_json::to_vec(&ModuleHelloBody {
7248 manifest: consumer,
7249 protocol_ver: PROTOCOL_VERSION,
7250 control_ops: None,
7251 launch_nonce: None,
7252 })
7253 .expect("HELLO serializes"),
7254 )
7255 .expect("HELLO frame builds");
7256 handler
7257 .handle_control(ConnectionId::new(78), hello)
7258 .expect("consumer registers");
7259 let describe = Frame::build(
7260 FrameType::Request,
7261 control_flags(),
7262 0,
7263 0,
7264 79,
7265 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
7266 .expect("request serializes"),
7267 )
7268 .expect("describe frame builds");
7269 let response = handler
7270 .handle_server_describe(describe)
7271 .expect("server.describe succeeds");
7272 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
7273 let requirement = &rendered["capability_requirements"][0];
7274 assert_eq!(requirement["consumer"], "consumer");
7275 assert_eq!(requirement["verdict"], "never_provided");
7276 assert_eq!(requirement["episode_seq"], 1);
7277 assert_eq!(requirement["config_satisfiable"], false);
7278 assert_eq!(requirement["runtime_available"], false);
7279 assert!(requirement["detail"]
7280 .as_str()
7281 .expect("detail string")
7282 .contains("credentials-provider/v1"));
7283 }
7284
7285 #[test]
7286 fn catalog_list_omits_capabilities_for_legacy_manifest() {
7287 let registry = Arc::new(Registry::default());
7288 let handler = ControlHandler::new(Arc::clone(®istry));
7289 let hello = handler
7290 .handle_control(
7291 ConnectionId::new(101),
7292 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
7293 )
7294 .expect("legacy HELLO registers");
7295 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
7296
7297 let request = Frame::build(
7298 FrameType::Request,
7299 control_flags(),
7300 0,
7301 0,
7302 102,
7303 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7304 .expect("catalog request serializes"),
7305 )
7306 .expect("catalog request frame builds");
7307 let response = handler
7308 .handle_catalog_list(request, None)
7309 .expect("catalog list succeeds");
7310 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7311 assert!(
7312 body["modules"][0].get("capabilities").is_none(),
7313 "legacy manifest must retain an absent capabilities field on catalog.list"
7314 );
7315 }
7316
7317 #[test]
7318 fn hello_ack_omits_storage_when_no_storage_config() {
7319 let registry = Arc::new(Registry::default());
7320 let handler = ControlHandler::new(Arc::clone(®istry));
7321 let responses = handler
7322 .handle_control(
7323 ConnectionId::new(1),
7324 hello_frame("aft", PROTOCOL_VERSION, 7),
7325 )
7326 .unwrap();
7327 let ack = parse_ack(&responses[0]);
7328 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
7329 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
7330 }
7331
7332 #[tokio::test]
7333 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
7334 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
7335 let registry = Arc::new(Registry::default());
7336 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
7337 let responses = handler
7338 .handle_control(
7339 ConnectionId::new(1),
7340 hello_frame("aft", PROTOCOL_VERSION, 7),
7341 )
7342 .unwrap();
7343 let ack = parse_ack(&responses[0]);
7344 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
7345
7346 let described = handler
7347 .handle_control_frame(
7348 &route_ctx(ConnectionId::new(2)).0,
7349 Frame::build(
7350 FrameType::Request,
7351 control_flags(),
7352 0,
7353 0,
7354 9,
7355 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
7356 )
7357 .unwrap(),
7358 )
7359 .await
7360 .unwrap();
7361 let ClientControlResponse::ServerDescribe { machine_id, .. } =
7362 serde_json::from_slice(&described[0].body).unwrap()
7363 else {
7364 panic!("server.describe answered with another shape");
7365 };
7366 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
7367 }
7368
7369 #[test]
7370 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
7371 let registry = Arc::new(Registry::default());
7374 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
7375 crate::daemon_config::StorageConfig::Sqlite {
7376 data_home: std::path::PathBuf::from("/data"),
7377 },
7378 ));
7379
7380 let responses = handler
7381 .handle_control(
7382 ConnectionId::new(1),
7383 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
7384 )
7385 .unwrap();
7386 let ack = parse_ack(&responses[0]);
7387 assert_eq!(
7388 ack.storage,
7389 Some(serde_json::json!({
7390 "module_id": "alfonso-routing",
7391 "storage_namespace": "default",
7392 "isolation": { "kind": "module" },
7393 "backend": {
7394 "backend": "sqlite",
7395 "path": "/data/cortexkit/alfonso-routing/store.db"
7396 }
7397 })),
7398 "the delivered descriptor is the module's own sqlite store path"
7399 );
7400 }
7401
7402 #[test]
7403 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
7404 let registry = Arc::new(Registry::default());
7405 let handler = ControlHandler::new(Arc::clone(®istry));
7406 let conn = ConnectionId::new(1);
7407 let responses = handler
7408 .handle_control(
7409 conn,
7410 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7411 )
7412 .unwrap();
7413 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7414 let registration = registry.get_module("aft").unwrap().unwrap();
7415 assert_eq!(registration.control_ops, module_baseline_control_ops());
7416
7417 let frame =
7418 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
7419 assert!(handler
7420 .guard_module_control_op(&frame, "aft", "route.bind")
7421 .unwrap()
7422 .is_none());
7423 let error = handler
7424 .guard_module_control_op(&frame, "aft", "test.synthetic")
7425 .unwrap()
7426 .expect("synthetic ungranted op should be rejected");
7427 assert_eq!(error.header.ty, FrameType::Error);
7428 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
7429 }
7430
7431 #[test]
7432 fn hello_control_ops_some_adds_optional_grants() {
7433 let registry = Arc::new(Registry::default());
7434 let handler = ControlHandler::new(Arc::clone(®istry));
7435 handler
7436 .handle_control(
7437 ConnectionId::new(1),
7438 hello_frame_with_control_ops(
7439 "aft",
7440 PROTOCOL_VERSION,
7441 7,
7442 Some(vec![
7443 "future.synthetic".to_string(),
7444 "route.bind".to_string(),
7445 ]),
7446 ),
7447 )
7448 .unwrap();
7449 let registration = registry.get_module("aft").unwrap().unwrap();
7450 assert_eq!(
7451 registration.control_ops,
7452 vec![
7453 "route.bind".to_string(),
7454 "route.status".to_string(),
7455 "future.synthetic".to_string(),
7456 ]
7457 );
7458 let frame =
7459 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
7460 assert!(handler
7461 .guard_module_control_op(&frame, "aft", "future.synthetic")
7462 .unwrap()
7463 .is_none());
7464 }
7465
7466 #[tokio::test]
7467 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
7468 let registry = Arc::new(Registry::default());
7469 let forwarding = Arc::new(ForwardingTable::default());
7470 let handler =
7471 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7472 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
7473 hello_via_sink(
7474 &handler,
7475 &module_ctx,
7476 &mut module_rx,
7477 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7478 )
7479 .await;
7480
7481 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
7482 let responses = handler
7483 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
7484 .await
7485 .unwrap();
7486 assert_eq!(responses.len(), 1);
7487 assert_eq!(responses[0].header.ty, FrameType::Error);
7488 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
7489 assert!(module_rx.try_recv().is_err());
7490 }
7491
7492 #[tokio::test]
7493 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
7494 let registry = Arc::new(Registry::default());
7495 let forwarding = Arc::new(ForwardingTable::default());
7496 let handler =
7497 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7498 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
7499 hello_via_sink(
7500 &handler,
7501 &module_ctx,
7502 &mut module_rx,
7503 hello_frame_with_control_ops(
7504 "aft",
7505 PROTOCOL_VERSION,
7506 7,
7507 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
7508 ),
7509 )
7510 .await;
7511
7512 let project_root = unique_project_root("demux");
7513 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
7514 let route_handler = handler.clone();
7515 let route_task = tokio::spawn(async move {
7516 route_handler
7517 .handle_control_frame(
7518 &route_client_ctx,
7519 route_open_frame(100, "aft", project_root),
7520 )
7521 .await
7522 .unwrap()
7523 });
7524 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7525 .await
7526 .unwrap()
7527 .unwrap();
7528 assert!(matches!(
7529 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
7530 ModuleControlRequest::RouteBind { .. }
7531 ));
7532
7533 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
7534 let health_handler = handler.clone();
7535 let health_task = tokio::spawn(async move {
7536 health_handler
7537 .handle_control_frame(
7538 &health_client_ctx,
7539 supervisor_health_probe_frame(101, "aft"),
7540 )
7541 .await
7542 .unwrap()
7543 });
7544 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7545 .await
7546 .unwrap()
7547 .unwrap();
7548 assert_eq!(
7549 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
7550 ModuleControlRequest::HealthCheck {}
7551 );
7552
7553 handler
7554 .handle_control_frame(
7555 &module_ctx,
7556 health_response(health_frame.header.corr, HealthStatus::Degraded),
7557 )
7558 .await
7559 .unwrap();
7560 let health_response = health_task.await.unwrap();
7561 assert_eq!(health_response.len(), 1);
7562 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
7563 ClientControlResponse::SupervisorHealthProbe {
7564 module_id,
7565 status,
7566 detail,
7567 metrics,
7568 } => {
7569 assert_eq!(module_id, "aft");
7570 assert_eq!(status, HealthStatus::Degraded);
7571 assert_eq!(detail.as_deref(), Some("warming"));
7572 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
7573 }
7574 other => panic!("unexpected health response: {other:?}"),
7575 }
7576
7577 handler
7578 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7579 .await
7580 .unwrap();
7581 let route_response = route_task.await.unwrap();
7582 assert!(route_response.is_empty());
7583 let published = route_client_rx.recv().await.unwrap();
7584 assert!(matches!(
7585 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7586 ClientControlResponse::RouteOpen { .. }
7587 ));
7588 }
7589
7590 async fn relay_route_open(
7595 handler: &ControlHandler,
7596 client_connection: ConnectionId,
7597 client_egress: &FrameSink,
7598 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7599 corr: u64,
7600 module_id: &str,
7601 project_root_label: &str,
7602 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7603 let ctx = RouteCtx {
7604 connection_id: client_connection,
7605 egress: client_egress.clone(),
7606 };
7607 let handler = handler.clone();
7608 let project_root = unique_project_root(project_root_label);
7609 let module_id = module_id.to_string();
7610 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7611 let task = tokio::spawn(async move {
7612 let _guard = tracing::dispatcher::set_default(&dispatch);
7613 handler
7614 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
7615 .await
7616 .unwrap()
7617 });
7618 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
7619 .await
7620 .expect("module receives the relayed route.bind")
7621 .expect("module egress is open");
7622 (task, bind.frame)
7623 }
7624
7625 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
7626 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7627 ModuleControlRequest::RouteBind {
7628 route_channel,
7629 epoch,
7630 ..
7631 } => (route_channel, epoch),
7632 other => panic!("expected a route.bind request, got {other:?}"),
7633 }
7634 }
7635
7636 fn published_route(frame: &Frame) -> (u16, u32) {
7637 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7638 ClientControlResponse::RouteOpen {
7639 route_channel,
7640 route_epoch,
7641 } => (route_channel, route_epoch),
7642 other => panic!("expected a route.open response, got {other:?}"),
7643 }
7644 }
7645
7646 #[tokio::test]
7666 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7667 let registry = Arc::new(Registry::default());
7668 let forwarding = Arc::new(ForwardingTable::default());
7669 let handler =
7670 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7671
7672 let module_connection = ConnectionId::new(30);
7673 let (module_ctx, mut module_rx) = route_ctx(module_connection);
7674 hello_via_sink(
7675 &handler,
7676 &module_ctx,
7677 &mut module_rx,
7678 hello_frame("aft", PROTOCOL_VERSION, 7),
7679 )
7680 .await;
7681
7682 let dying_client = ConnectionId::new(31);
7683 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7684
7685 let (first_task, first_bind) = relay_route_open(
7688 &handler,
7689 dying_client,
7690 &dying_ctx.egress,
7691 &mut module_rx,
7692 100,
7693 "aft",
7694 "closing-first",
7695 )
7696 .await;
7697 handler
7698 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7699 .await
7700 .unwrap();
7701 assert!(first_task.await.unwrap().is_empty());
7702 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7703
7704 let (second_task, second_bind) = relay_route_open(
7706 &handler,
7707 dying_client,
7708 &dying_ctx.egress,
7709 &mut module_rx,
7710 101,
7711 "aft",
7712 "closing-second",
7713 )
7714 .await;
7715 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7716
7717 assert!(forwarding
7720 .escalate_client_delivery_failure(
7721 dying_client,
7722 first_channel,
7723 first_epoch,
7724 CloseReason::new(
7725 "module_to_client_delivery_failed",
7726 "client egress refused a module frame",
7727 ),
7728 crate::forwarding::UndeliveredFrame {
7729 module_id: None,
7730 sink: &dying_ctx.egress,
7731 },
7732 )
7733 .unwrap());
7734 assert!(!dying_ctx.egress.is_closed());
7735
7736 let ack = handler
7738 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7739 .await;
7740 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7741 if module_loop_error.is_some() {
7742 handler.cleanup_connection(module_connection).unwrap();
7746 }
7747 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7751 .await
7752 .ok()
7753 .flatten();
7754
7755 assert!(
7757 registry
7758 .get_module_by_connection(module_connection)
7759 .unwrap()
7760 .is_some(),
7761 "one client's closing connection ended the shared module connection: \
7762 {module_loop_error:?}"
7763 );
7764 let cotenant = ConnectionId::new(32);
7766 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7767 let (cotenant_task, cotenant_bind) = relay_route_open(
7768 &handler,
7769 cotenant,
7770 &cotenant_ctx.egress,
7771 &mut module_rx,
7772 102,
7773 "aft",
7774 "closing-cotenant",
7775 )
7776 .await;
7777 handler
7778 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7779 .await
7780 .unwrap();
7781 assert!(cotenant_task.await.unwrap().is_empty());
7782 let (cotenant_channel, cotenant_epoch) =
7783 published_route(&cotenant_rx.recv().await.unwrap());
7784 assert!(matches!(
7785 forwarding
7786 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7787 .unwrap(),
7788 DataRoute::Client(DataRouteState::Bound(_))
7789 ));
7790
7791 let goodbye = post_ack_module_frame
7794 .expect("module receives a GOODBYE for the abandoned route channel");
7795 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7796 assert_eq!(goodbye.header.channel, abandoned_channel);
7797 assert_eq!(goodbye.header.epoch, abandoned_epoch);
7798
7799 assert!(dying_rx.try_recv().is_err());
7803 let second_response = second_task.await.unwrap();
7804 assert_eq!(second_response.len(), 1);
7805 assert_eq!(
7806 parse_error(&second_response[0])["code"],
7807 "target_unavailable"
7808 );
7809 }
7810
7811 #[test]
7818 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7819 let handler = ControlHandler::default();
7820 let module_connection = ConnectionId::new(30);
7821 let client_connection = ConnectionId::new(31);
7822
7823 handler
7824 .refuse_to_end_module_connection_for_a_client(
7825 module_connection,
7826 77,
7827 ForwardingError::ConnectionClosing {
7828 connection_id: client_connection,
7829 },
7830 )
7831 .expect("a closing client must never end the module connection");
7832
7833 assert!(matches!(
7834 handler.refuse_to_end_module_connection_for_a_client(
7835 module_connection,
7836 78,
7837 ForwardingError::ConnectionClosing {
7838 connection_id: module_connection,
7839 },
7840 ),
7841 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7842 connection_id
7843 })) if connection_id == module_connection
7844 ));
7845 assert!(matches!(
7846 handler.refuse_to_end_module_connection_for_a_client(
7847 module_connection,
7848 79,
7849 ForwardingError::Poisoned,
7850 ),
7851 Err(RouterError::Forwarding(ForwardingError::Poisoned))
7852 ));
7853 assert!(matches!(
7854 handler.refuse_to_end_module_connection_for_a_client(
7855 module_connection,
7856 80,
7857 ForwardingError::StaleModuleEndpoint,
7858 ),
7859 Err(RouterError::Forwarding(
7860 ForwardingError::StaleModuleEndpoint
7861 ))
7862 ));
7863 }
7864
7865 #[tokio::test]
7878 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7879 let registry = Arc::new(Registry::default());
7880 let forwarding = Arc::new(ForwardingTable::default());
7881 let supervisor = SupervisorHandle::new();
7882 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7883 let handler =
7884 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7885 .with_supervisor(supervisor);
7886
7887 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7888 hello_via_sink(
7889 &handler,
7890 &target_ctx,
7891 &mut target_rx,
7892 hello_frame("target", PROTOCOL_VERSION, 1),
7893 )
7894 .await;
7895
7896 let wrong_nonce = handler
7900 .handle_control_frame(
7901 &route_ctx(ConnectionId::new(91)).0,
7902 route_open_frame_with_admission_facts(
7903 20,
7904 "target",
7905 unique_project_root("admission-facts"),
7906 Some(subc_control::ConsumerIdentity {
7907 module_id: "fed".to_string(),
7908 launch_nonce: "not-the-real-nonce".to_string(),
7909 }),
7910 None,
7911 ),
7912 )
7913 .await
7914 .unwrap();
7915 assert_eq!(
7916 parse_error(&wrong_nonce[0])["code"],
7917 "bad_consumer_identity",
7918 "a mismatched launch nonce must be refused, not stamped Reserved"
7919 );
7920
7921 let never_spawned = handler
7925 .handle_control_frame(
7926 &route_ctx(ConnectionId::new(92)).0,
7927 route_open_frame_with_admission_facts(
7928 21,
7929 "target",
7930 unique_project_root("admission-facts"),
7931 Some(subc_control::ConsumerIdentity {
7932 module_id: "never-spawned".to_string(),
7933 launch_nonce: "any-nonce".to_string(),
7934 }),
7935 None,
7936 ),
7937 )
7938 .await
7939 .unwrap();
7940 assert_eq!(
7941 parse_error(&never_spawned[0])["code"],
7942 "bad_consumer_identity",
7943 "an unspawned module_id must be refused rather than accepted for lack of a record"
7944 );
7945 }
7946
7947 #[tokio::test]
7970 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
7971 let registry = Arc::new(Registry::default());
7972 let forwarding = Arc::new(ForwardingTable::default());
7973 let supervisor = SupervisorHandle::new();
7974 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7975 let handler =
7976 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7977 .with_supervisor(supervisor);
7978
7979 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
7980 hello_via_sink(
7981 &handler,
7982 &target_ctx,
7983 &mut target_rx,
7984 hello_frame("target", PROTOCOL_VERSION, 1),
7985 )
7986 .await;
7987
7988 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
7989 let route_handler = handler.clone();
7990 let route_task = tokio::spawn(async move {
7991 route_handler
7992 .handle_control_frame(
7993 &client_ctx,
7994 route_open_frame_with_admission_facts(
7995 30,
7996 "target",
7997 unique_project_root("admission-facts"),
7998 Some(subc_control::ConsumerIdentity {
7999 module_id: "fed".to_string(),
8000 launch_nonce: "fed-nonce".to_string(),
8001 }),
8002 None,
8003 ),
8004 )
8005 .await
8006 .unwrap()
8007 });
8008
8009 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8016 .await
8017 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
8018 .expect("module control channel closed before route.bind");
8019 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8020 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
8021 panic!("expected route.bind")
8022 };
8023 assert_eq!(
8024 principal,
8025 Some(Principal::Reserved {
8026 module_id: "fed".to_string()
8027 }),
8028 "a correctly attested consumer must be stamped Reserved for its own id"
8029 );
8030
8031 handler
8032 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8033 .await
8034 .unwrap();
8035 assert!(route_task.await.unwrap().is_empty());
8036 assert!(
8037 matches!(
8038 serde_json::from_slice::<ClientControlResponse>(
8039 &client_rx.recv().await.unwrap().body
8040 )
8041 .unwrap(),
8042 ClientControlResponse::RouteOpen { .. }
8043 ),
8044 "the route must actually open, not merely avoid an error"
8045 );
8046 }
8047
8048 #[tokio::test(start_paused = true)]
8049 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
8050 let registry = Arc::new(Registry::default());
8051 let forwarding = Arc::new(ForwardingTable::default());
8052 let supervisor = SupervisorHandle::new();
8053 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8054 let handler =
8055 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8056 .with_supervisor(supervisor);
8057
8058 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
8059 hello_via_sink(
8060 &handler,
8061 &target_ctx,
8062 &mut target_rx,
8063 hello_frame("target", PROTOCOL_VERSION, 1),
8064 )
8065 .await;
8066
8067 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
8068 let direct_handler = handler.clone();
8069 let direct_open = tokio::spawn(async move {
8070 direct_handler
8071 .handle_control_frame(
8072 &direct_ctx,
8073 route_open_frame(2, "target", unique_project_root("route-census-direct")),
8074 )
8075 .await
8076 .unwrap()
8077 });
8078 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8079 .await
8080 .expect("no direct route.bind within 5s")
8081 .expect("target control channel closed before direct route.bind");
8082 handler
8083 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
8084 .await
8085 .unwrap();
8086 assert!(direct_open.await.unwrap().is_empty());
8087 let _ = direct_rx.recv().await.unwrap();
8088
8089 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
8090 let reserved_handler = handler.clone();
8091 let reserved_open = tokio::spawn(async move {
8092 reserved_handler
8093 .handle_control_frame(
8094 &reserved_ctx,
8095 route_open_frame_with_admission_facts(
8096 3,
8097 "target",
8098 unique_project_root("admission-facts"),
8099 Some(ConsumerIdentity {
8100 module_id: "fed".to_string(),
8101 launch_nonce: "fed-nonce".to_string(),
8102 }),
8103 None,
8104 ),
8105 )
8106 .await
8107 .unwrap()
8108 });
8109 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8110 .await
8111 .expect("no reserved route.bind within 5s")
8112 .expect("target control channel closed before reserved route.bind");
8113 handler
8114 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
8115 .await
8116 .unwrap();
8117 assert!(reserved_open.await.unwrap().is_empty());
8118 let _ = reserved_rx.recv().await.unwrap();
8119
8120 forwarding
8121 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
8122 .unwrap();
8123 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
8124 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
8125 module_id: Some("target".to_string()),
8126 })
8127 .unwrap();
8128 let census_frame =
8129 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
8130 let response = handler
8131 .handle_control_frame(&census_ctx, census_frame)
8132 .await
8133 .unwrap()
8134 .pop()
8135 .unwrap();
8136 let actual: Value = serde_json::from_slice(&response.body).unwrap();
8137 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
8138 assert!(matches!(
8139 decoded,
8140 ClientControlResponse::SupervisorRoutes { .. }
8141 ));
8142 let routes = actual["modules"][0]["routes"].as_array().unwrap();
8143 assert_eq!(routes.len(), 2);
8144 assert!(routes.iter().all(|route| route["draining"] == true));
8145 assert!(
8148 routes.iter().all(|route| route["drain_reason"] == "reload"),
8149 "draining routes must name the drain's reason: {routes:?}"
8150 );
8151 assert!(routes.iter().any(|route| {
8152 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
8153 }));
8154 assert!(routes.iter().any(|route| {
8155 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
8156 }));
8157
8158 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8159 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
8160 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8161 std::fs::write(
8162 &golden_path,
8163 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
8164 )
8165 .unwrap();
8166 }
8167 let expected: Value =
8168 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
8169 assert_eq!(actual, expected);
8170 }
8171
8172 async fn query_live_roots(
8173 handler: &ControlHandler,
8174 module_ctx: &RouteCtx,
8175 ) -> ModuleControlResponseToModule {
8176 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
8177 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
8178 let response = handler
8179 .handle_control_frame(module_ctx, frame)
8180 .await
8181 .unwrap()
8182 .pop()
8183 .unwrap();
8184 serde_json::from_slice(&response.body).unwrap()
8185 }
8186
8187 #[tokio::test(start_paused = true)]
8188 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
8189 let registry = Arc::new(Registry::default());
8190 let forwarding = Arc::new(ForwardingTable::default());
8191 let handler = ControlHandler::with_forwarding(registry, forwarding);
8192 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
8193 hello_via_sink(
8194 &handler,
8195 &target_ctx,
8196 &mut target_rx,
8197 hello_frame("target", PROTOCOL_VERSION, 1),
8198 )
8199 .await;
8200 let root = unique_project_root("live-roots-known");
8201 let path = ProjectRootId::from_path_allowing_missing(root.path())
8202 .unwrap()
8203 .as_path()
8204 .to_path_buf();
8205 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
8206 let open_handler = handler.clone();
8207 let opened = tokio::spawn(async move {
8208 open_handler
8209 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
8210 .await
8211 .unwrap()
8212 });
8213 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8214 .await
8215 .unwrap()
8216 .unwrap();
8217 handler
8218 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
8219 .await
8220 .unwrap();
8221 assert!(opened.await.unwrap().is_empty());
8222 let _ = client_rx.recv().await.unwrap();
8223
8224 let root = unique_project_root("live-roots-pending");
8225 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
8226 .unwrap()
8227 .as_path()
8228 .to_path_buf();
8229 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
8230 let open_handler = handler.clone();
8231 let pending = tokio::spawn(async move {
8232 open_handler
8233 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
8234 .await
8235 .unwrap()
8236 });
8237 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8238 .await
8239 .unwrap()
8240 .unwrap();
8241 let actual = query_live_roots(&handler, &target_ctx).await;
8242 let ModuleControlResponseToModule::LiveRoots {
8243 roots,
8244 unknown_root_bindings,
8245 total_bindings,
8246 } = actual
8247 else {
8248 panic!("expected live roots")
8249 };
8250 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
8251 assert_eq!(unknown_root_bindings, 0);
8252 assert_eq!(
8253 roots.len(),
8254 2,
8255 "root-known arm must retain each canonical root"
8256 );
8257 assert_eq!(
8258 total_bindings,
8259 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8260 );
8261 let counts = roots
8262 .iter()
8263 .map(|root| (root.project_root.clone(), root.bound, root.pending))
8264 .collect::<Vec<_>>();
8265 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
8266 expected.sort_by(|a, b| a.0.cmp(&b.0));
8267 assert_eq!(
8268 counts, expected,
8269 "roots must sort by path and count pending separately"
8270 );
8271 handler
8272 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
8273 .await
8274 .unwrap();
8275 assert!(pending.await.unwrap().is_empty());
8276 }
8277
8278 #[tokio::test(start_paused = true)]
8279 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
8280 let forwarding = Arc::new(ForwardingTable::default());
8281 let handler =
8282 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8283 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
8284 hello_via_sink(
8285 &handler,
8286 &target_ctx,
8287 &mut target_rx,
8288 hello_frame("target", PROTOCOL_VERSION, 1),
8289 )
8290 .await;
8291 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
8292 let pending = forwarding
8293 .begin_route_bind_relay_for_test(
8294 client_ctx.connection_id,
8295 client_ctx.egress.clone(),
8296 2,
8297 "target",
8298 )
8299 .unwrap();
8300 forwarding
8301 .complete_pending_relay(
8302 target_ctx.connection_id,
8303 pending.corr,
8304 RouteBindRelayOutcome::Accepted,
8305 )
8306 .unwrap();
8307 let actual = query_live_roots(&handler, &target_ctx).await;
8308 let ModuleControlResponseToModule::LiveRoots {
8309 roots,
8310 unknown_root_bindings,
8311 total_bindings,
8312 } = actual
8313 else {
8314 panic!("expected live roots")
8315 };
8316 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
8317 assert_eq!(
8318 unknown_root_bindings, 1,
8319 "unknown-root arm must not read as no bindings"
8320 );
8321 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
8322 assert_eq!(
8323 total_bindings,
8324 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8325 );
8326 }
8327
8328 #[tokio::test(start_paused = true)]
8336 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
8337 let forwarding = Arc::new(ForwardingTable::default());
8338 let handler =
8339 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8340 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
8341 let replies = handler
8342 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
8343 .await
8344 .unwrap();
8345 let queued_by_hello = module_rx.len();
8346
8347 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
8348 let open_handler = handler.clone();
8349 let open = tokio::spawn(async move {
8350 open_handler
8351 .handle_control_frame(
8352 &client_ctx,
8353 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
8354 )
8355 .await
8356 .unwrap()
8357 });
8358 let mut spins = 0;
8360 while module_rx.len() == queued_by_hello {
8361 spins += 1;
8362 assert!(spins < 10_000, "route.open never queued a route.bind");
8363 tokio::task::yield_now().await;
8364 }
8365
8366 for reply in replies {
8368 module_ctx.egress.send(reply).await.unwrap();
8369 }
8370
8371 let first = module_rx.recv().await.unwrap().frame;
8372 assert_eq!(
8373 first.header.ty,
8374 FrameType::HelloAck,
8375 "the first frame a registering module reads must be its HELLO_ACK"
8376 );
8377 assert_eq!(first.header.corr, 7);
8378 let second = module_rx.recv().await.unwrap().frame;
8379 assert_eq!(second.header.ty, FrameType::Request);
8380 assert!(
8381 matches!(
8382 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
8383 ModuleControlRequest::RouteBind { .. }
8384 ),
8385 "the route.bind follows the ack"
8386 );
8387 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
8388
8389 handler
8390 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
8391 .await
8392 .unwrap();
8393 assert!(open.await.unwrap().is_empty());
8394 let _ = client_rx.recv().await.unwrap();
8395 }
8396
8397 #[tokio::test(start_paused = true)]
8398 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
8399 let handler = ControlHandler::with_forwarding(
8400 Arc::new(Registry::default()),
8401 Arc::new(ForwardingTable::default()),
8402 );
8403 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
8404 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
8405 hello_via_sink(
8406 &handler,
8407 &first_ctx,
8408 &mut first_rx,
8409 hello_frame("first", PROTOCOL_VERSION, 1),
8410 )
8411 .await;
8412 hello_via_sink(
8413 &handler,
8414 &second_ctx,
8415 &mut second_rx,
8416 hello_frame("second", PROTOCOL_VERSION, 2),
8417 )
8418 .await;
8419 let root = unique_project_root("second-only");
8420 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
8421 let cloned = handler.clone();
8422 let open = tokio::spawn(async move {
8423 cloned
8424 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
8425 .await
8426 .unwrap()
8427 });
8428 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
8429 .await
8430 .unwrap()
8431 .unwrap();
8432 let first = query_live_roots(&handler, &first_ctx).await;
8433 let second = query_live_roots(&handler, &second_ctx).await;
8434 assert!(
8435 matches!(
8436 first,
8437 ModuleControlResponseToModule::LiveRoots {
8438 total_bindings: 0,
8439 ..
8440 }
8441 ),
8442 "cross-module scope must not expose another module's roots"
8443 );
8444 assert!(
8445 matches!(
8446 second,
8447 ModuleControlResponseToModule::LiveRoots {
8448 total_bindings: 1,
8449 ..
8450 }
8451 ),
8452 "second module must see its pending route"
8453 );
8454 handler
8455 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
8456 .await
8457 .unwrap();
8458 assert!(open.await.unwrap().is_empty());
8459 }
8460
8461 #[tokio::test(start_paused = true)]
8462 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
8463 let handler = ControlHandler::with_forwarding(
8464 Arc::new(Registry::default()),
8465 Arc::new(ForwardingTable::default()),
8466 );
8467 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
8468 hello_via_sink(
8469 &handler,
8470 &target_ctx,
8471 &mut target_rx,
8472 hello_frame("target", PROTOCOL_VERSION, 1),
8473 )
8474 .await;
8475 let actual = query_live_roots(&handler, &target_ctx).await;
8476 let ModuleControlResponseToModule::LiveRoots {
8477 roots,
8478 unknown_root_bindings,
8479 total_bindings,
8480 } = actual
8481 else {
8482 panic!("expected live roots")
8483 };
8484 assert!(roots.is_empty());
8485 assert_eq!(unknown_root_bindings, 0);
8486 assert_eq!(total_bindings, 0);
8487 assert_eq!(
8488 total_bindings,
8489 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8490 );
8491 }
8492
8493 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
8500 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8501 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
8502 let text = std::fs::read_to_string(&path)
8503 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
8504 let vectors: Vec<(String, Value)> = text
8505 .lines()
8506 .filter(|line| !line.trim().is_empty())
8507 .map(|line| {
8508 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
8509 let id = entry["corpus_id"]
8510 .as_str()
8511 .expect("every vector carries a corpus_id")
8512 .to_string();
8513 (id, entry["package"].clone())
8514 })
8515 .collect();
8516 assert_eq!(
8520 vectors.len(),
8521 3,
8522 "vendored fed corpus changed size; re-sync from subc-federation"
8523 );
8524
8525 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
8540 let richest = vectors
8541 .iter()
8542 .filter_map(|(_, package)| package.as_object())
8543 .map(|object| {
8544 object
8545 .keys()
8546 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
8547 .count()
8548 })
8549 .max()
8550 .unwrap_or(0);
8551 assert!(
8552 richest >= 2,
8553 "vendored corpus no longer carries a package with unmodelled fields, \
8554 so the relay test can no longer distinguish a verbatim relay from a lossy one"
8555 );
8556
8557 vectors
8558 }
8559
8560 #[tokio::test]
8568 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
8569 for (corpus_id, package) in fed_admission_facts_vectors() {
8570 let registry = Arc::new(Registry::default());
8571 let forwarding = Arc::new(ForwardingTable::default());
8572 let supervisor = SupervisorHandle::new();
8573 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8574 let handler =
8575 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8576 .with_supervisor(supervisor)
8577 .with_admission_facts_config(
8578 Some("fed".to_string()),
8579 Some(vec!["target".to_string()]),
8580 );
8581
8582 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8583 hello_via_sink(
8584 &handler,
8585 &target_ctx,
8586 &mut target_rx,
8587 hello_frame("target", PROTOCOL_VERSION, 1),
8588 )
8589 .await;
8590
8591 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8592 let route_handler = handler.clone();
8593 let expected = package.clone();
8594 let route_task = tokio::spawn(async move {
8595 route_handler
8596 .handle_control_frame(
8597 &client_ctx,
8598 route_open_frame_with_admission_facts(
8599 20,
8600 "target",
8601 unique_project_root("admission-facts"),
8602 Some(subc_control::ConsumerIdentity {
8603 module_id: "fed".to_string(),
8604 launch_nonce: "fed-nonce".to_string(),
8605 }),
8606 Some(package),
8607 ),
8608 )
8609 .await
8610 .unwrap()
8611 });
8612
8613 let bind_frame = target_rx.recv().await.unwrap();
8614 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8615 let ModuleControlRequest::RouteBind {
8616 admission_facts, ..
8617 } = bind
8618 else {
8619 panic!("{corpus_id}: expected route.bind")
8620 };
8621 assert_eq!(
8622 admission_facts,
8623 Some(expected),
8624 "{corpus_id}: relay must not add, drop or reshape any field"
8625 );
8626
8627 handler
8628 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8629 .await
8630 .unwrap();
8631 route_task.await.unwrap();
8632 }
8633 }
8634
8635 #[tokio::test]
8636 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8637 let registry = Arc::new(Registry::default());
8638 let forwarding = Arc::new(ForwardingTable::default());
8639 let supervisor = SupervisorHandle::new();
8640 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8641 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8642 let handler =
8643 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8644 .with_supervisor(supervisor)
8645 .with_admission_facts_config(
8646 Some("fed".to_string()),
8647 Some(vec!["target".to_string()]),
8648 );
8649
8650 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8651 hello_via_sink(
8652 &handler,
8653 &target_ctx,
8654 &mut target_rx,
8655 hello_frame("target", PROTOCOL_VERSION, 1),
8656 )
8657 .await;
8658 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8659 hello_via_sink(
8660 &handler,
8661 &other_ctx,
8662 &mut other_rx,
8663 hello_frame("other", PROTOCOL_VERSION, 2),
8664 )
8665 .await;
8666
8667 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8668 let expected_facts = facts.clone();
8669 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8670 let route_handler = handler.clone();
8671 let route_task = tokio::spawn(async move {
8672 route_handler
8673 .handle_control_frame(
8674 &client_ctx,
8675 route_open_frame_with_admission_facts(
8676 10,
8677 "target",
8678 unique_project_root("admission-facts"),
8679 Some(subc_control::ConsumerIdentity {
8680 module_id: "fed".to_string(),
8681 launch_nonce: "fed-nonce".to_string(),
8682 }),
8683 Some(facts.clone()),
8684 ),
8685 )
8686 .await
8687 .unwrap()
8688 });
8689 let bind_frame = target_rx.recv().await.unwrap();
8690 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8691 let ModuleControlRequest::RouteBind {
8692 admission_facts, ..
8693 } = bind
8694 else {
8695 panic!("expected route.bind")
8696 };
8697 assert_eq!(admission_facts, Some(expected_facts));
8698 handler
8699 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8700 .await
8701 .unwrap();
8702 assert!(route_task.await.unwrap().is_empty());
8703 assert!(matches!(
8704 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8705 .unwrap(),
8706 ClientControlResponse::RouteOpen { .. }
8707 ));
8708
8709 let direct = handler
8710 .handle_control_frame(
8711 &route_ctx(ConnectionId::new(73)).0,
8712 route_open_frame_with_admission_facts(
8713 11,
8714 "target",
8715 unique_project_root("admission-facts"),
8716 None,
8717 Some(json!({"x": 1})),
8718 ),
8719 )
8720 .await
8721 .unwrap();
8722 assert_eq!(
8723 parse_error(&direct[0])["code"],
8724 "admission_facts_not_permitted"
8725 );
8726
8727 let different_reserved = handler
8728 .handle_control_frame(
8729 &route_ctx(ConnectionId::new(77)).0,
8730 route_open_frame_with_admission_facts(
8731 15,
8732 "target",
8733 unique_project_root("admission-facts"),
8734 Some(subc_control::ConsumerIdentity {
8735 module_id: "other".to_string(),
8736 launch_nonce: "other-nonce".to_string(),
8737 }),
8738 Some(json!({"x": 1})),
8739 ),
8740 )
8741 .await
8742 .unwrap();
8743 assert_eq!(
8744 parse_error(&different_reserved[0])["code"],
8745 "admission_facts_not_permitted"
8746 );
8747
8748 let other_target = handler
8749 .handle_control_frame(
8750 &route_ctx(ConnectionId::new(74)).0,
8751 route_open_frame_with_admission_facts(
8752 12,
8753 "other",
8754 unique_project_root("admission-facts"),
8755 Some(subc_control::ConsumerIdentity {
8756 module_id: "fed".to_string(),
8757 launch_nonce: "fed-nonce".to_string(),
8758 }),
8759 Some(json!({"x": 1})),
8760 ),
8761 )
8762 .await
8763 .unwrap();
8764 assert_eq!(
8765 parse_error(&other_target[0])["code"],
8766 "admission_facts_target_not_allowed"
8767 );
8768
8769 let nonexistent = handler
8770 .handle_control_frame(
8771 &route_ctx(ConnectionId::new(75)).0,
8772 route_open_frame_with_admission_facts(
8773 13,
8774 "missing",
8775 unique_project_root("admission-facts"),
8776 None,
8777 Some(json!({"x": 1})),
8778 ),
8779 )
8780 .await
8781 .unwrap();
8782 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8783
8784 let described = handler
8785 .handle_control_frame(
8786 &route_ctx(ConnectionId::new(76)).0,
8787 Frame::build(
8788 FrameType::Request,
8789 control_flags(),
8790 0,
8791 0,
8792 14,
8793 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8794 )
8795 .unwrap(),
8796 )
8797 .await
8798 .unwrap();
8799 let ClientControlResponse::ServerDescribe { capabilities, .. } =
8800 serde_json::from_slice(&described[0].body).unwrap()
8801 else {
8802 panic!("expected server.describe response")
8803 };
8804 assert!(capabilities
8805 .iter()
8806 .any(|cap| cap == "admission_facts_relay_v1"));
8807 }
8808
8809 #[tokio::test]
8810 async fn admission_facts_without_configured_carrier_are_rejected() {
8811 let registry = Arc::new(Registry::default());
8812 let forwarding = Arc::new(ForwardingTable::default());
8813 let handler = ControlHandler::with_forwarding(registry, forwarding);
8814 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8815 hello_via_sink(
8816 &handler,
8817 &target_ctx,
8818 &mut target_rx,
8819 hello_frame("target", PROTOCOL_VERSION, 1),
8820 )
8821 .await;
8822
8823 let responses = handler
8824 .handle_control_frame(
8825 &route_ctx(ConnectionId::new(79)).0,
8826 route_open_frame_with_admission_facts(
8827 16,
8828 "target",
8829 unique_project_root("admission-facts"),
8830 None,
8831 Some(json!({"x": 1})),
8832 ),
8833 )
8834 .await
8835 .unwrap();
8836 assert_eq!(
8837 parse_error(&responses[0])["code"],
8838 "admission_facts_not_permitted"
8839 );
8840 }
8841
8842 #[tokio::test]
8843 async fn route_open_relays_consumer_capabilities_verbatim() {
8844 let registry = Arc::new(Registry::default());
8845 let forwarding = Arc::new(ForwardingTable::default());
8846 let handler =
8847 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8848 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8849 hello_via_sink(
8850 &handler,
8851 &module_ctx,
8852 &mut module_rx,
8853 hello_frame("aft", PROTOCOL_VERSION, 7),
8854 )
8855 .await;
8856
8857 let expected = vec!["elicitation".to_string(), "roots".to_string()];
8858 let expected_for_request = expected.clone();
8859 let project_root = unique_project_root("consumer-capabilities-present");
8860 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8861 let route_handler = handler.clone();
8862 let route_task = tokio::spawn(async move {
8863 route_handler
8864 .handle_control_frame(
8865 &client_ctx,
8866 route_open_frame_with_consumer_capabilities(
8867 401,
8868 "aft",
8869 project_root,
8870 Some(expected_for_request),
8871 ),
8872 )
8873 .await
8874 .unwrap()
8875 });
8876 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8877 .await
8878 .unwrap()
8879 .unwrap();
8880 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8881 let ModuleControlRequest::RouteBind {
8882 consumer_capabilities,
8883 ..
8884 } = bind
8885 else {
8886 panic!("expected route.bind request, got {bind:?}");
8887 };
8888 assert_eq!(consumer_capabilities, Some(expected.clone()));
8889
8890 handler
8891 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8892 .await
8893 .unwrap();
8894 let route_response = route_task.await.unwrap();
8895 assert!(route_response.is_empty());
8896 let published = client_rx.recv().await.unwrap();
8897 assert!(matches!(
8898 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8899 ClientControlResponse::RouteOpen { .. }
8900 ));
8901 }
8902
8903 #[tokio::test]
8904 async fn route_open_without_consumer_capabilities_relays_none() {
8905 let registry = Arc::new(Registry::default());
8906 let forwarding = Arc::new(ForwardingTable::default());
8907 let handler =
8908 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8909 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8910 hello_via_sink(
8911 &handler,
8912 &module_ctx,
8913 &mut module_rx,
8914 hello_frame("aft", PROTOCOL_VERSION, 7),
8915 )
8916 .await;
8917
8918 let project_root = unique_project_root("consumer-capabilities-absent");
8919 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8920 let route_handler = handler.clone();
8921 let route_task = tokio::spawn(async move {
8922 route_handler
8923 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
8924 .await
8925 .unwrap()
8926 });
8927 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8928 .await
8929 .unwrap()
8930 .unwrap();
8931 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8932 let ModuleControlRequest::RouteBind {
8933 consumer_capabilities,
8934 ..
8935 } = bind
8936 else {
8937 panic!("expected route.bind request, got {bind:?}");
8938 };
8939 assert_eq!(consumer_capabilities, None);
8940
8941 handler
8942 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8943 .await
8944 .unwrap();
8945 let route_response = route_task.await.unwrap();
8946 assert!(route_response.is_empty());
8947 let published = client_rx.recv().await.unwrap();
8948 assert!(matches!(
8949 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8950 ClientControlResponse::RouteOpen { .. }
8951 ));
8952 }
8953
8954 #[tokio::test]
8955 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
8956 let registry = Arc::new(Registry::default());
8957 let forwarding = Arc::new(ForwardingTable::default());
8958 let handler =
8959 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8960 .with_health_probe_timeout(Duration::from_secs(5));
8961 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
8962 hello_via_sink(
8963 &handler,
8964 &module_ctx,
8965 &mut module_rx,
8966 non_routable_hello_frame_with_control_ops(
8967 "mcp",
8968 300,
8969 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8970 ),
8971 )
8972 .await;
8973 assert!(registry
8974 .get_module("mcp")
8975 .unwrap()
8976 .unwrap()
8977 .manifest
8978 .provides
8979 .is_empty());
8980
8981 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
8982 let route_response = handler
8983 .handle_control_frame(
8984 &route_client_ctx,
8985 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
8986 )
8987 .await
8988 .unwrap();
8989 assert_eq!(route_response[0].header.ty, FrameType::Error);
8990 assert_eq!(
8991 parse_error(&route_response[0])["code"],
8992 "target_unavailable"
8993 );
8994 assert!(parse_error(&route_response[0])["message"]
8995 .as_str()
8996 .unwrap()
8997 .contains("does not provide the requested target"));
8998 assert!(module_rx.try_recv().is_err());
8999
9000 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
9001 let health_handler = handler.clone();
9002 let health_task = tokio::spawn(async move {
9003 health_handler
9004 .handle_control_frame(
9005 &health_client_ctx,
9006 supervisor_health_probe_frame(302, "mcp"),
9007 )
9008 .await
9009 .unwrap()
9010 });
9011 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9012 .await
9013 .unwrap()
9014 .unwrap();
9015 assert_eq!(
9016 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9017 ModuleControlRequest::HealthCheck {}
9018 );
9019 handler
9020 .handle_control_frame(
9021 &module_ctx,
9022 health_response(health_frame.header.corr, HealthStatus::Ok),
9023 )
9024 .await
9025 .unwrap();
9026 let health_response = health_task.await.unwrap();
9027 assert_eq!(health_response[0].header.ty, FrameType::Response);
9028 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9029 ClientControlResponse::SupervisorHealthProbe {
9030 module_id, status, ..
9031 } => {
9032 assert_eq!(module_id, "mcp");
9033 assert_eq!(status, HealthStatus::Ok);
9034 }
9035 other => panic!("unexpected health response: {other:?}"),
9036 }
9037
9038 forwarding
9043 .cleanup_connection(module_ctx.connection_id)
9044 .unwrap();
9045 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
9046 let cleanup_response = tokio::time::timeout(
9047 Duration::from_millis(200),
9048 handler.handle_control_frame(
9049 &cleanup_probe_ctx,
9050 supervisor_health_probe_frame(303, "mcp"),
9051 ),
9052 )
9053 .await
9054 .expect("probe should fail immediately when the control lane is gone")
9055 .unwrap();
9056 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
9057 assert_eq!(
9058 parse_error(&cleanup_response[0])["code"],
9059 "target_unavailable"
9060 );
9061 assert!(parse_error(&cleanup_response[0])["message"]
9062 .as_str()
9063 .unwrap()
9064 .contains("no module connection"));
9065
9066 handler
9067 .cleanup_connection(module_ctx.connection_id)
9068 .unwrap();
9069 }
9070
9071 #[tokio::test]
9072 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
9073 let registry = Arc::new(Registry::default());
9074 let supervisor_handle = SupervisorHandle::new();
9075 let supervisor =
9076 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9077 .with_handle(supervisor_handle.clone())
9078 .with_connection_file_path(
9079 std::env::temp_dir()
9080 .join(format!("subc-route-open-warming-{}", std::process::id())),
9081 );
9082 let module = supervisor
9083 .supervise_configured(
9084 ModuleSpec {
9085 launch_nonce_env: true,
9086 module_id: "warming".to_string(),
9087 program: fake_aft_stub_path(),
9088 args: Vec::new(),
9089 env: Vec::new(),
9090 reserved: false,
9091 reserved_prefixes: Vec::new(),
9092 protocol: ModuleProtocol::Subc,
9093 overlap: Default::default(),
9094 },
9095 true,
9096 )
9097 .unwrap();
9098 assert_eq!(module.state().unwrap(), ModuleState::Running);
9099
9100 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9101 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
9102 let response = handler
9103 .handle_control_frame(
9104 &ctx,
9105 route_open_frame(304, "warming", unique_project_root("warming")),
9106 )
9107 .await
9108 .unwrap();
9109 module.stop().await.unwrap();
9110
9111 assert_eq!(response[0].header.ty, FrameType::Error);
9112 let error = parse_error(&response[0]);
9113 assert_eq!(error["code"], "module_warming");
9114 assert!(error["message"]
9115 .as_str()
9116 .unwrap()
9117 .contains("state=running, enabled=true, live=false"));
9118 }
9119
9120 #[test]
9121 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
9122 let handler = ControlHandler::new(Arc::new(Registry::default()));
9123 let capture = EventCapture::default();
9124 let _subscriber =
9125 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9126 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
9127 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
9128 let pending = (0..limit).collect::<Vec<_>>();
9129 let response = handler
9130 .route_open_capacity_refusal(
9131 &ctx,
9132 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
9133 "busy",
9134 pending.len(),
9135 limit,
9136 )
9137 .unwrap();
9138 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9139 let event = capture
9140 .events()
9141 .into_iter()
9142 .find(|event| {
9143 event.target == "control"
9144 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
9145 })
9146 .expect("connection admission refusal event");
9147 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9148 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9149 }
9150
9151 #[test]
9152 fn route_open_target_cap_logs_admission_reason_and_capacity() {
9153 let handler = ControlHandler::new(Arc::new(Registry::default()));
9154 let capture = EventCapture::default();
9155 let _subscriber =
9156 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9157 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
9158 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
9159 let guards = (0..limit)
9160 .map(|_| {
9161 handler
9162 .route_bind_concurrency
9163 .try_admit("busy", limit)
9164 .unwrap()
9165 })
9166 .collect::<Vec<_>>();
9167 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
9168 Err(in_flight) => in_flight,
9169 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
9170 };
9171 let response = handler
9172 .route_open_target_capacity_refusal(
9173 &ctx,
9174 &route_open_frame(397, "busy", unique_project_root("target-cap")),
9175 "busy",
9176 in_flight,
9177 )
9178 .unwrap();
9179 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9180 let event = capture
9181 .events()
9182 .into_iter()
9183 .find(|event| {
9184 event.target == "control"
9185 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
9186 })
9187 .expect("target admission refusal event");
9188 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9189 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9190 drop(guards);
9191 }
9192
9193 #[tokio::test]
9198 async fn route_open_refusal_names_the_check_that_refused() {
9199 let handler = ControlHandler::new(Arc::new(Registry::default()));
9200 let capture = EventCapture::default();
9201 let _subscriber =
9202 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9203 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9204 let response = handler
9205 .handle_control_frame(
9206 &ctx,
9207 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
9208 )
9209 .await
9210 .unwrap();
9211
9212 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9213 let event = capture
9214 .events()
9215 .into_iter()
9216 .find(|event| {
9217 event.target == "control"
9218 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9219 })
9220 .expect("route.open refusal event");
9221 assert_eq!(
9222 event.fields.get("reason"),
9223 Some(&"\"not_registered\"".to_string())
9224 );
9225 }
9226
9227 #[tokio::test]
9228 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
9229 let registry = Arc::new(Registry::default());
9230 let supervisor_handle = SupervisorHandle::new();
9231 let supervisor =
9232 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9233 .with_handle(supervisor_handle.clone())
9234 .with_connection_file_path(std::env::temp_dir().join(format!(
9235 "subc-route-open-refusal-info-{}",
9236 std::process::id()
9237 )));
9238 let module = supervisor
9239 .supervise_configured(
9240 ModuleSpec {
9241 launch_nonce_env: true,
9242 module_id: "warming".to_string(),
9243 program: fake_aft_stub_path(),
9244 args: Vec::new(),
9245 env: Vec::new(),
9246 reserved: false,
9247 reserved_prefixes: Vec::new(),
9248 protocol: ModuleProtocol::Subc,
9249 overlap: Default::default(),
9250 },
9251 true,
9252 )
9253 .unwrap();
9254 assert_eq!(module.state().unwrap(), ModuleState::Running);
9255
9256 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9257 assert!(handler
9258 .counters()
9259 .snapshot()
9260 .get("route_open_refused_by_code")
9261 .is_none());
9262 let capture = EventCapture::default();
9263 let _subscriber =
9264 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9265 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
9266 let response = handler
9267 .handle_control_frame(
9268 &ctx,
9269 route_open_frame(394, "warming", unique_project_root("refusal-info")),
9270 )
9271 .await
9272 .unwrap();
9273 module.stop().await.unwrap();
9274
9275 assert_eq!(parse_error(&response[0])["code"], "module_warming");
9276 let event = capture
9277 .events()
9278 .into_iter()
9279 .find(|event| {
9280 event.target == "control"
9281 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
9282 })
9283 .expect("route.open refusal event");
9284 assert_eq!(
9285 event.fields.get("module_id"),
9286 Some(&"\"warming\"".to_string())
9287 );
9288 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
9289 assert_eq!(
9290 event.fields.get("reason"),
9291 Some(&"\"supervised_not_registered\"".to_string())
9292 );
9293 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
9294 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
9295 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
9296 assert_eq!(
9297 handler.counters().snapshot()["route_open_refused_by_code"],
9298 json!({ "module_warming": 1 })
9299 );
9300 }
9301
9302 const OUTAGE_START: &str = "route.open refusing module: not serving";
9303 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
9304
9305 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
9306 capture
9307 .events()
9308 .into_iter()
9309 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
9310 .collect()
9311 }
9312
9313 fn supervise_stub(
9314 registry: &Arc<Registry>,
9315 module_id: &str,
9316 enabled: bool,
9317 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
9318 let supervisor_handle = SupervisorHandle::new();
9319 let supervisor =
9320 Supervisor::new(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
9321 .with_handle(supervisor_handle.clone())
9322 .with_connection_file_path(std::env::temp_dir().join(format!(
9323 "subc-route-outage-{module_id}-{}",
9324 std::process::id()
9325 )));
9326 let module = supervisor
9327 .supervise_configured(
9328 ModuleSpec {
9329 launch_nonce_env: true,
9330 module_id: module_id.to_string(),
9331 program: fake_aft_stub_path(),
9332 args: Vec::new(),
9333 env: Vec::new(),
9334 reserved: false,
9335 reserved_prefixes: Vec::new(),
9336 protocol: ModuleProtocol::Subc,
9337 overlap: Default::default(),
9338 },
9339 enabled,
9340 )
9341 .unwrap();
9342 (supervisor_handle, module)
9343 }
9344
9345 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
9346 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
9347 module_id: module_id.to_string(),
9348 drain_timeout_ms: Some(50),
9349 })
9350 .unwrap();
9351 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9352 }
9353
9354 #[test]
9358 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
9359 let registry = Arc::new(Registry::default());
9360 let forwarding = Arc::new(ForwardingTable::default());
9361 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9362 let second = ControlHandler::with_forwarding(registry, forwarding);
9363 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
9364 }
9365
9366 #[tokio::test(flavor = "current_thread")]
9370 async fn route_open_unknown_module_refusals_add_no_outage_state() {
9371 let handler = ControlHandler::new(Arc::new(Registry::default()));
9372 let capture = EventCapture::default();
9373 let _subscriber =
9374 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9375 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
9376 for corr in 0..8 {
9377 let response = handler
9378 .handle_control_frame(
9379 &ctx,
9380 route_open_frame(
9381 380 + corr,
9382 &format!("nobody-{corr}"),
9383 unique_project_root("outage-unknown"),
9384 ),
9385 )
9386 .await
9387 .unwrap();
9388 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9389 }
9390
9391 assert_eq!(handler.route_outages.tracked_module_count(), 0);
9392 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
9393 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
9394 }
9395
9396 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9401 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
9402 let registry = Arc::new(Registry::default());
9403 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
9404 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9405 let capture = EventCapture::default();
9406 let _subscriber =
9407 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9408 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
9409 handler.route_outages.record_accepted("outage-restart");
9412
9413 let response = handler
9414 .handle_control_frame(
9415 &ctx,
9416 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
9417 )
9418 .await
9419 .unwrap();
9420 assert_eq!(response[0].header.ty, FrameType::Error);
9421 let starts = outage_lines(&capture, OUTAGE_START);
9422 assert_eq!(starts.len(), 1, "{starts:?}");
9423 assert_eq!(starts[0].level, tracing::Level::WARN);
9424 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
9425 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
9426 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
9427 handler.route_outages.record_accepted("outage-restart");
9428 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
9429
9430 let restart = handler
9431 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
9432 .await
9433 .unwrap();
9434 assert_eq!(
9435 restart[0].header.ty,
9436 FrameType::Response,
9437 "{:?}",
9438 parse_error(&restart[0])
9439 );
9440 handler
9441 .handle_control_frame(
9442 &ctx,
9443 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
9444 )
9445 .await
9446 .unwrap();
9447 module.stop().await.unwrap();
9448
9449 let starts = outage_lines(&capture, OUTAGE_START);
9450 assert_eq!(starts.len(), 2, "{starts:?}");
9451 assert_eq!(starts[1].level, tracing::Level::INFO);
9452 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
9453 }
9454
9455 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9459 async fn failed_operator_restart_leaves_no_operator_mark() {
9460 let registry = Arc::new(Registry::default());
9461 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
9462 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9463 let capture = EventCapture::default();
9464 let _subscriber =
9465 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9466 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
9467 handler.route_outages.record_accepted("outage-disabled");
9468
9469 let restart = handler
9470 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
9471 .await
9472 .unwrap();
9473 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
9474 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
9475
9476 handler
9477 .handle_control_frame(
9478 &ctx,
9479 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
9480 )
9481 .await
9482 .unwrap();
9483 let starts = outage_lines(&capture, OUTAGE_START);
9484 assert_eq!(starts.len(), 1, "{starts:?}");
9485 assert_eq!(starts[0].level, tracing::Level::WARN);
9486 }
9487
9488 #[tokio::test(flavor = "current_thread")]
9489 async fn route_open_unknown_module_escapes_target_module_id() {
9490 let handler = ControlHandler::new(Arc::new(Registry::default()));
9491 let capture = EventCapture::default();
9492 let _subscriber =
9493 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9494 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
9495 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9496 let response = handler
9497 .handle_control_frame(
9498 &ctx,
9499 route_open_frame(
9500 395,
9501 hostile_module_id,
9502 unique_project_root("hostile-target-module-id"),
9503 ),
9504 )
9505 .await
9506 .unwrap();
9507
9508 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9509 let event = capture
9510 .events()
9511 .into_iter()
9512 .find(|event| {
9513 event.target == "control"
9514 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9515 })
9516 .expect("route.open unknown-module refusal event");
9517 let logged = event.fields.get("module_id").expect("module_id field");
9518 assert!(!logged.bytes().any(|byte| byte < 0x20));
9519 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9520 }
9521
9522 #[tokio::test(flavor = "current_thread")]
9523 async fn route_open_module_rejection_uses_daemon_counter_key() {
9524 let registry = Arc::new(Registry::default());
9525 let forwarding = Arc::new(ForwardingTable::default());
9526 let handler =
9527 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9528 let module_connection = ConnectionId::new(95);
9529 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9530 hello_via_sink(
9531 &handler,
9532 &module_ctx,
9533 &mut module_rx,
9534 hello_frame("aft", PROTOCOL_VERSION, 395),
9535 )
9536 .await;
9537
9538 let client_connection = ConnectionId::new(96);
9539 let (client_ctx, _client_rx) = route_ctx(client_connection);
9540 let capture = EventCapture::default();
9541 let _subscriber =
9542 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9543 let (route_task, bind) = relay_route_open(
9544 &handler,
9545 client_connection,
9546 &client_ctx.egress,
9547 &mut module_rx,
9548 396,
9549 "aft",
9550 "hostile-module-code",
9551 )
9552 .await;
9553 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
9554 let rejection = Frame::build(
9555 FrameType::Error,
9556 control_flags(),
9557 0,
9558 0,
9559 bind.header.corr,
9560 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
9561 )
9562 .unwrap();
9563 handler
9564 .handle_control_frame(&module_ctx, rejection)
9565 .await
9566 .unwrap();
9567
9568 let response = route_task.await.unwrap();
9569 assert_eq!(parse_error(&response[0])["code"], hostile_code);
9570 let counters = handler.counters().snapshot();
9571 assert_eq!(
9572 counters["route_open_refused_by_code"],
9573 json!({ "module_rejected": 1 })
9574 );
9575 assert!(counters["route_open_refused_by_code"]
9576 .get(hostile_code)
9577 .is_none());
9578
9579 let event = capture
9580 .events()
9581 .into_iter()
9582 .find(|event| {
9583 event.target == "control"
9584 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
9585 })
9586 .expect("route.open module-rejection refusal event");
9587 let logged = event.fields.get("module_code").expect("module_code field");
9588 assert!(!logged.bytes().any(|byte| byte < 0x20));
9589 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9590 }
9591
9592 #[tokio::test]
9593 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
9594 let registry = Arc::new(Registry::default());
9595 let supervisor_handle = SupervisorHandle::new();
9596 let missing_program = std::env::temp_dir().join(format!(
9597 "subc-route-open-missing-program-{}",
9598 std::process::id()
9599 ));
9600 let supervisor =
9601 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9602 .with_handle(supervisor_handle.clone());
9603 let module = supervisor
9604 .supervise_configured(
9605 ModuleSpec {
9606 launch_nonce_env: true,
9607 module_id: "failed".to_string(),
9608 program: missing_program,
9609 args: Vec::new(),
9610 env: Vec::new(),
9611 reserved: false,
9612 reserved_prefixes: Vec::new(),
9613 protocol: ModuleProtocol::Subc,
9614 overlap: Default::default(),
9615 },
9616 true,
9617 )
9618 .unwrap();
9619 assert_eq!(module.state().unwrap(), ModuleState::Failed);
9620
9621 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9622 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
9623 let response = handler
9624 .handle_control_frame(
9625 &ctx,
9626 route_open_frame(305, "failed", unique_project_root("failed")),
9627 )
9628 .await
9629 .unwrap();
9630
9631 assert_eq!(response[0].header.ty, FrameType::Error);
9632 let error = parse_error(&response[0]);
9633 assert_eq!(error["code"], "target_unavailable");
9634 assert!(error["message"]
9635 .as_str()
9636 .unwrap()
9637 .contains("state=failed, enabled=true, live=false"));
9638 }
9639
9640 #[tokio::test]
9641 async fn route_open_role_mismatch_remains_target_unavailable() {
9642 let registry = Arc::new(Registry::default());
9643 let handler = ControlHandler::new(Arc::clone(®istry));
9644 handler
9645 .handle_control(
9646 ConnectionId::new(41),
9647 non_routable_hello_frame_with_control_ops("health-only", 306, None),
9648 )
9649 .unwrap();
9650
9651 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
9652 let response = handler
9653 .handle_control_frame(
9654 &ctx,
9655 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
9656 )
9657 .await
9658 .unwrap();
9659
9660 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9661 assert!(parse_error(&response[0])["message"]
9662 .as_str()
9663 .unwrap()
9664 .contains("does not provide the requested target"));
9665 }
9666
9667 #[tokio::test]
9668 async fn route_open_inactive_registration_remains_target_unavailable() {
9669 let registry = Arc::new(Registry::default());
9670 let handler = ControlHandler::new(Arc::clone(®istry));
9671 handler
9672 .handle_control(
9673 ConnectionId::new(43),
9674 hello_frame("inactive", PROTOCOL_VERSION, 308),
9675 )
9676 .unwrap();
9677 assert!(registry
9678 .set_module_state_for_test("inactive", ChannelState::Closed)
9679 .unwrap());
9680
9681 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
9682 let response = handler
9683 .handle_control_frame(
9684 &ctx,
9685 route_open_frame(309, "inactive", unique_project_root("inactive")),
9686 )
9687 .await
9688 .unwrap();
9689
9690 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9691 assert!(parse_error(&response[0])["message"]
9692 .as_str()
9693 .unwrap()
9694 .contains("is not active"));
9695 }
9696
9697 #[tokio::test]
9698 async fn late_health_reply_is_recorded_through_the_module_response_path() {
9699 let registry = Arc::new(Registry::default());
9700 let forwarding = Arc::new(ForwardingTable::default());
9701 let supervisor_handle = SupervisorHandle::new();
9702 let supervisor = Supervisor::new(Arc::clone(®istry), crate::RestartPolicy::default())
9703 .with_forwarding(Arc::clone(&forwarding))
9704 .with_handle(supervisor_handle.clone());
9705 let module = supervisor
9706 .supervise_configured(
9707 crate::ModuleSpec {
9708 launch_nonce_env: true,
9709 module_id: "late-health-response".to_string(),
9710 program: PathBuf::from("disabled-module"),
9711 args: Vec::new(),
9712 env: Vec::new(),
9713 reserved: false,
9714 reserved_prefixes: Vec::new(),
9715 protocol: ModuleProtocol::Subc,
9716 overlap: Default::default(),
9717 },
9718 false,
9719 )
9720 .unwrap();
9721 let handler =
9722 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9723 .with_supervisor(supervisor_handle);
9724 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
9725 handler
9726 .handle_control_frame(
9727 &module_ctx,
9728 hello_frame_with_control_ops(
9729 "late-health-response",
9730 PROTOCOL_VERSION,
9731 7,
9732 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9733 ),
9734 )
9735 .await
9736 .unwrap();
9737 let probe_started_at = Instant::now() - Duration::from_millis(80);
9738 let pending = forwarding
9739 .begin_health_probe_rpc_for(
9740 "late-health-response",
9741 MODULE_CONTROL_OP_HEALTH_CHECK,
9742 probe_started_at,
9743 Instant::now() - Duration::from_millis(1),
9744 )
9745 .unwrap();
9746 assert!(forwarding
9747 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
9748 .unwrap());
9749
9750 let responses = handler
9751 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
9752 .await
9753 .unwrap();
9754
9755 assert!(responses.is_empty());
9756 let health = module.status().unwrap().health;
9757 assert_eq!(health.late_answer_count, 1);
9758 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
9759 }
9760
9761 #[tokio::test]
9762 async fn health_probe_timeout_and_module_death_are_typed() {
9763 let registry = Arc::new(Registry::default());
9764 let forwarding = Arc::new(ForwardingTable::default());
9765 let handler =
9766 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9767 .with_health_probe_timeout(Duration::from_millis(50));
9768 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9769 hello_via_sink(
9770 &handler,
9771 &module_ctx,
9772 &mut module_rx,
9773 hello_frame_with_control_ops(
9774 "aft",
9775 PROTOCOL_VERSION,
9776 7,
9777 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9778 ),
9779 )
9780 .await;
9781
9782 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9783 let responses = handler
9784 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
9785 .await
9786 .unwrap();
9787 assert_eq!(responses[0].header.ty, FrameType::Error);
9788 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
9789 let _ = module_rx.try_recv();
9790
9791 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
9792 let health_handler = handler.clone();
9793 let death_task = tokio::spawn(async move {
9794 health_handler
9795 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
9796 .await
9797 .unwrap()
9798 });
9799 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9800 .await
9801 .unwrap()
9802 .unwrap();
9803 handler
9804 .cleanup_connection(module_ctx.connection_id)
9805 .unwrap();
9806 let responses = death_task.await.unwrap();
9807 assert_eq!(responses[0].header.ty, FrameType::Error);
9808 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
9809 }
9810
9811 #[test]
9812 fn hello_requires_exact_protocol_version() {
9813 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
9814 let registry = Arc::new(Registry::default());
9815 let handler = ControlHandler::new(Arc::clone(®istry));
9816 let responses = handler
9817 .handle_control(
9818 ConnectionId::new(connection),
9819 hello_frame("aft", offered, 9),
9820 )
9821 .unwrap();
9822
9823 assert_eq!(responses.len(), 1);
9824 assert_eq!(responses[0].header.ty, FrameType::Error);
9825 let error = parse_error(&responses[0]);
9826 assert_eq!(error["code"], "version_unsupported");
9827 assert!(registry.get_module("aft").unwrap().is_none());
9828 assert_eq!(registry.active_registration_count().unwrap(), 0);
9829 }
9830 }
9831
9832 #[test]
9833 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
9834 let registry = Arc::new(Registry::default());
9835 let forwarding = Arc::new(ForwardingTable::default());
9836 let handler =
9837 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9838 let module_connection = ConnectionId::new(301);
9839 let registration = registry
9840 .register_with_control_ops(
9841 manifest("aft-push", PROTOCOL_VERSION),
9842 PROTOCOL_VERSION,
9843 module_connection,
9844 module_baseline_control_ops(),
9845 )
9846 .unwrap();
9847 let (module_tx, _module_rx) = mpsc::channel(8);
9848 let endpoint = forwarding
9849 .register_module_connection(
9850 module_connection,
9851 "aft-push".to_string(),
9852 PROTOCOL_VERSION,
9853 manifest_concurrency(®istration.manifest),
9854 FrameSink::new(module_tx),
9855 )
9856 .unwrap();
9857
9858 let unknown = Frame::build(
9860 FrameType::Push,
9861 control_flags(),
9862 0,
9863 0,
9864 5,
9865 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
9866 )
9867 .unwrap();
9868 let out = handler.handle_status_update(endpoint, unknown).unwrap();
9869 assert!(
9870 out.is_empty(),
9871 "unknown push op must be ignored, got {out:?}"
9872 );
9873
9874 let malformed = Frame::build(
9876 FrameType::Push,
9877 control_flags(),
9878 0,
9879 0,
9880 6,
9881 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
9882 )
9883 .unwrap();
9884 let out = handler.handle_status_update(endpoint, malformed).unwrap();
9885 assert_eq!(out.len(), 1);
9886 assert_eq!(out[0].header.ty, FrameType::Error);
9887 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
9888 }
9889
9890 #[test]
9891 fn hello_rejected_when_connection_already_owns_client_routes() {
9892 let registry = Arc::new(Registry::default());
9893 let forwarding = Arc::new(ForwardingTable::default());
9894 let handler =
9895 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9896 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
9898 let client_connection = ConnectionId::new(202);
9899
9900 let responses = handler
9903 .handle_control(
9904 client_connection,
9905 hello_frame("aft-second", PROTOCOL_VERSION, 9),
9906 )
9907 .unwrap();
9908 assert_eq!(responses[0].header.ty, FrameType::Error);
9909 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
9910 assert!(registry.get_module("aft-second").unwrap().is_none());
9911 }
9912
9913 #[test]
9914 fn reserved_module_hello_requires_matching_launch_nonce() {
9915 let registry = Arc::new(Registry::default());
9916 let supervisor = SupervisorHandle::new();
9917 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9920 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9921
9922 let no_nonce = handler
9924 .handle_control(
9925 ConnectionId::new(1),
9926 hello_frame("vault", PROTOCOL_VERSION, 1),
9927 )
9928 .unwrap();
9929 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
9930 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
9931 assert!(registry.get_module("vault").unwrap().is_none());
9932
9933 let wrong = handler
9935 .handle_control(
9936 ConnectionId::new(2),
9937 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
9938 )
9939 .unwrap();
9940 assert_eq!(wrong[0].header.ty, FrameType::Error);
9941 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
9942 assert!(registry.get_module("vault").unwrap().is_none());
9943
9944 let ok = handler
9946 .handle_control(
9947 ConnectionId::new(3),
9948 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
9949 )
9950 .unwrap();
9951 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
9952 assert!(registry.get_module("vault").unwrap().is_some());
9953 }
9954
9955 #[test]
9956 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
9957 let registry = Arc::new(Registry::default());
9958 let supervisor = SupervisorHandle::new();
9959 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9960 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9961 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9962
9963 let squat = handler
9964 .handle_control(
9965 ConnectionId::new(1),
9966 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
9967 )
9968 .unwrap();
9969 assert_eq!(squat[0].header.ty, FrameType::Error);
9970 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
9971 assert!(parse_error(&squat[0])["message"]
9972 .as_str()
9973 .unwrap()
9974 .contains("fed:"));
9975
9976 let accepted_peer = handler
9977 .handle_control(
9978 ConnectionId::new(2),
9979 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
9980 )
9981 .unwrap();
9982 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
9983
9984 let accepted_short = handler
9985 .handle_control(
9986 ConnectionId::new(3),
9987 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
9988 )
9989 .unwrap();
9990 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
9991
9992 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
9993 let response = handler
9994 .handle_control(
9995 ConnectionId::new(conn),
9996 hello_frame(module_id, PROTOCOL_VERSION, conn),
9997 )
9998 .unwrap();
9999 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
10000 }
10001 }
10002
10003 #[test]
10004 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
10005 let registry = Arc::new(Registry::default());
10006 let supervisor = SupervisorHandle::new();
10007 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10008 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10009 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
10010 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10011
10012 let owner_nonce = handler
10013 .handle_control(
10014 ConnectionId::new(1),
10015 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
10016 )
10017 .unwrap();
10018 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
10019 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
10020 assert!(registry.get_module("fed:special").unwrap().is_none());
10021
10022 let exact_nonce = handler
10023 .handle_control(
10024 ConnectionId::new(2),
10025 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
10026 )
10027 .unwrap();
10028 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
10029 assert!(registry.get_module("fed:special").unwrap().is_some());
10030 }
10031
10032 #[test]
10033 fn non_reserved_module_ignores_launch_nonce() {
10034 let registry = Arc::new(Registry::default());
10035 let handler = ControlHandler::new(Arc::clone(®istry));
10038 let no_nonce = handler
10039 .handle_control(
10040 ConnectionId::new(1),
10041 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
10042 )
10043 .unwrap();
10044 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
10045 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
10046
10047 let echoed_nonce = handler
10048 .handle_control(
10049 ConnectionId::new(2),
10050 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
10051 )
10052 .unwrap();
10053 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
10054 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
10055 }
10056
10057 #[test]
10058 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
10059 let handler = ControlHandler::default();
10060 let conn = ConnectionId::new(1);
10061 let malformed = Frame::build(
10062 FrameType::Hello,
10063 control_flags(),
10064 0,
10065 0,
10066 3,
10067 b"{not json".to_vec(),
10068 )
10069 .unwrap();
10070
10071 let error = handler.handle_control(conn, malformed).unwrap();
10072 assert_eq!(error[0].header.ty, FrameType::Error);
10073 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
10074
10075 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
10076 let pong = handler.handle_control(conn, ping).unwrap();
10077 assert_eq!(pong[0].header.ty, FrameType::Pong);
10078 assert_eq!(pong[0].header.corr, 4);
10079 }
10080
10081 #[test]
10082 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
10083 let registry = Arc::new(Registry::default());
10084 let handler = ControlHandler::new(Arc::clone(®istry));
10085
10086 handler
10087 .handle_control(
10088 ConnectionId::new(1),
10089 hello_frame("aft", PROTOCOL_VERSION, 1),
10090 )
10091 .unwrap();
10092 let duplicate = handler
10093 .handle_control(
10094 ConnectionId::new(2),
10095 hello_frame("aft", PROTOCOL_VERSION, 2),
10096 )
10097 .unwrap();
10098
10099 assert_eq!(duplicate[0].header.ty, FrameType::Error);
10100 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10101 let registration = registry.get_module("aft").unwrap().unwrap();
10102 assert_eq!(registration.connection_id, ConnectionId::new(1));
10103 }
10104
10105 #[test]
10106 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
10107 let registry = Arc::new(Registry::default());
10108 let forwarding = Arc::new(ForwardingTable::default());
10109 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
10110 let handler =
10111 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10112 .with_process_liveness(process_liveness);
10113 let (ctx, route_channel, route_epoch) =
10114 bind_liveness_route(®istry, &forwarding, "aft-dead");
10115 let responses = handler
10116 .handle_route_poll(
10117 &ctx,
10118 route_poll_frame(41, PollKind::Liveness, route_channel),
10119 route_channel,
10120 route_epoch,
10121 PollKind::Liveness,
10122 )
10123 .unwrap();
10124
10125 assert_eq!(responses.len(), 1);
10126 assert_eq!(responses[0].header.ty, FrameType::Response);
10127 assert_route_poll_liveness(&responses[0], false);
10128 }
10129
10130 #[test]
10131 fn liveness_poll_without_process_source_uses_bound_route() {
10132 let registry = Arc::new(Registry::default());
10133 let forwarding = Arc::new(ForwardingTable::default());
10134 let handler =
10135 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10136 let (ctx, route_channel, route_epoch) =
10137 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
10138 let responses = handler
10139 .handle_route_poll(
10140 &ctx,
10141 route_poll_frame(42, PollKind::Liveness, route_channel),
10142 route_channel,
10143 route_epoch,
10144 PollKind::Liveness,
10145 )
10146 .unwrap();
10147
10148 assert_route_poll_liveness(&responses[0], true);
10149 }
10150
10151 #[test]
10152 fn liveness_poll_untracked_process_source_uses_bound_route() {
10153 let registry = Arc::new(Registry::default());
10154 let forwarding = Arc::new(ForwardingTable::default());
10155 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
10156 let handler =
10157 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10158 .with_process_liveness(process_liveness);
10159 let (ctx, route_channel, route_epoch) =
10160 bind_liveness_route(®istry, &forwarding, "aft-untracked");
10161 let responses = handler
10162 .handle_route_poll(
10163 &ctx,
10164 route_poll_frame(43, PollKind::Liveness, route_channel),
10165 route_channel,
10166 route_epoch,
10167 PollKind::Liveness,
10168 )
10169 .unwrap();
10170
10171 assert_route_poll_liveness(&responses[0], true);
10172 }
10173
10174 #[tokio::test]
10175 async fn unknown_op_returns_unknown_control_op() {
10176 let handler = ControlHandler::default();
10177 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
10178 let request = Frame::build(
10179 FrameType::Request,
10180 control_flags(),
10181 0,
10182 0,
10183 55,
10184 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
10185 )
10186 .unwrap();
10187
10188 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10189
10190 assert_eq!(response.len(), 1);
10191 assert_eq!(response[0].header.ty, FrameType::Error);
10192 assert_eq!(response[0].header.corr, 55);
10193 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
10194 }
10195
10196 #[tokio::test]
10197 async fn supervisor_provenance_rejects_unknown_exact_module() {
10198 let handler = ControlHandler::default();
10199 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
10200 let request = Frame::build(
10201 FrameType::Request,
10202 control_flags(),
10203 0,
10204 0,
10205 57,
10206 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
10207 )
10208 .unwrap();
10209
10210 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10211
10212 assert_eq!(response.len(), 1);
10213 assert_eq!(response[0].header.ty, FrameType::Error);
10214 assert_eq!(response[0].header.corr, 57);
10215 let error = parse_error(&response[0]);
10216 assert_eq!(error["code"], "unknown_module");
10217 assert_eq!(error["message"], "module_id 'missing' is not supervised");
10218 }
10219
10220 #[test]
10221 fn provenance_probe_override_keeps_handler_tests_deterministic() {
10222 let expected = subc_control::RunningImageAgreement::Unavailable {
10223 reason: subc_control::RunningImageUnavailableReason::HashFailed,
10224 };
10225 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
10226 assert_eq!(handler.provenance_probe_override, Some(expected));
10227 }
10228
10229 #[test]
10230 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
10231 let verdict = reload_verdict(
10232 std::path::Path::new("/bin/new"),
10233 Some(std::path::Path::new("/bin/old")),
10234 subc_control::RunningImageAgreement::Unavailable {
10235 reason: subc_control::RunningImageUnavailableReason::HashFailed,
10236 },
10237 );
10238 assert!(matches!(
10239 verdict.path,
10240 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
10241 if configured == std::path::Path::new("/bin/new")
10242 && spawned_from == std::path::Path::new("/bin/old")
10243 ));
10244 }
10245
10246 #[test]
10247 fn reload_verdict_detects_replaced_image_at_same_path() {
10248 let image = subc_control::RunningImageAgreement::Mismatch {
10249 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
10250 digest: "old".into(),
10251 },
10252 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
10253 digest: "new".into(),
10254 },
10255 };
10256 let verdict = reload_verdict(
10257 std::path::Path::new("/bin/same"),
10258 Some(std::path::Path::new("/bin/same")),
10259 image.clone(),
10260 );
10261 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10262 assert_eq!(verdict.image, image);
10263 }
10264
10265 #[test]
10266 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
10267 let image = subc_control::RunningImageAgreement::Unavailable {
10268 reason: subc_control::RunningImageUnavailableReason::NotRunning,
10269 };
10270 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
10271 assert_eq!(
10272 verdict.path,
10273 subc_control::ReloadPathAgreement::Unavailable {
10274 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
10275 }
10276 );
10277 assert_eq!(verdict.image, image);
10278
10279 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
10280 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
10281 };
10282 let verdict = reload_verdict(
10283 std::path::Path::new("/bin/same"),
10284 Some(std::path::Path::new("/bin/same")),
10285 unconfirmed.clone(),
10286 );
10287 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10288 assert_eq!(verdict.image, unconfirmed);
10289 }
10290
10291 #[test]
10292 fn reload_verdict_preserves_each_image_unavailability_reason() {
10293 use subc_control::RunningImageUnavailableReason as Reason;
10294
10295 for reason in [
10296 Reason::NotRunning,
10297 Reason::UnsupportedPlatform,
10298 Reason::RunningExecutableUnreadable,
10299 Reason::SpawnedPathUnreadable,
10300 Reason::HashFailed,
10301 Reason::ProcessIdentityUnconfirmed,
10302 Reason::Unknown("future_probe_reason".to_string()),
10303 ] {
10304 let image = subc_control::RunningImageAgreement::Unavailable {
10305 reason: reason.clone(),
10306 };
10307 let verdict = reload_verdict(
10308 std::path::Path::new("/bin/same"),
10309 Some(std::path::Path::new("/bin/same")),
10310 image.clone(),
10311 );
10312 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10313 assert_eq!(verdict.image, image, "{reason:?}");
10314 }
10315 }
10316
10317 #[tokio::test]
10318 async fn malformed_control_bodies_return_invalid_control_body() {
10319 let handler = ControlHandler::default();
10320 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
10321
10322 for (corr, body) in [
10323 (56, br#"{"route_channel":1}"#.as_slice()),
10324 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
10325 (
10326 58,
10327 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
10328 ),
10329 ] {
10330 let request = Frame::build(
10331 FrameType::Request,
10332 control_flags(),
10333 0,
10334 0,
10335 corr,
10336 body.to_vec(),
10337 )
10338 .unwrap();
10339 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10340
10341 assert_eq!(response.len(), 1);
10342 assert_eq!(response[0].header.ty, FrameType::Error);
10343 assert_eq!(response[0].header.corr, corr);
10344 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
10345 }
10346 }
10347
10348 #[tokio::test]
10349 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
10350 let registry = Arc::new(Registry::default());
10351 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
10352 let router = Router::with_control_handler(Arc::clone(&control));
10353 let connection = router.begin_connection();
10354 let (ctx, mut rx) = route_ctx(connection.id());
10355
10356 router
10357 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
10358 .await
10359 .unwrap();
10360 let response = rx.recv().await.unwrap();
10361 let ack = parse_ack(&response);
10362 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10363 let channel = 1;
10364
10365 let goodbye =
10366 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
10367 router.route_for_connection(&ctx, goodbye).await.unwrap();
10368 assert!(rx.try_recv().is_err());
10369 assert!(registry.get_module("aft").unwrap().is_none());
10370
10371 router
10372 .route_for_connection(&ctx, channel_request(channel, 13))
10373 .await
10374 .unwrap();
10375 let error_frame = rx.recv().await.unwrap();
10376 assert_eq!(error_frame.header.ty, FrameType::Error);
10377 assert_eq!(error_frame.header.channel, channel);
10378 }
10379
10380 #[tokio::test]
10381 async fn dropping_router_connection_releases_registration() {
10382 let registry = Arc::new(Registry::default());
10383 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
10384 let router = Router::with_control_handler(control);
10385 let connection = router.begin_connection();
10386 let (ctx, mut rx) = route_ctx(connection.id());
10387
10388 router
10389 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
10390 .await
10391 .unwrap();
10392 let response = rx.recv().await.unwrap();
10393 let ack = parse_ack(&response);
10394 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10395 assert!(registry.get_module("aft").unwrap().is_some());
10396
10397 drop(connection);
10398
10399 assert!(registry.get_module("aft").unwrap().is_none());
10400 assert_eq!(registry.active_registration_count().unwrap(), 0);
10401 }
10402
10403 fn capability_manifest(
10404 module_id: &str,
10405 provides: &[&str],
10406 must_never_reach: &[&str],
10407 ) -> ModuleManifest {
10408 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
10409 manifest.capabilities = Some(CapabilityDeclarations {
10410 provides: provides
10411 .iter()
10412 .map(|capability| (*capability).to_string())
10413 .collect(),
10414 requires: Vec::new(),
10415 must_never_reach: must_never_reach
10416 .iter()
10417 .map(|capability| (*capability).to_string())
10418 .collect(),
10419 });
10420 manifest
10421 }
10422
10423 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
10424 Frame::build(
10425 FrameType::Hello,
10426 control_flags(),
10427 0,
10428 0,
10429 corr,
10430 serde_json::to_vec(&ModuleHelloBody {
10431 protocol_ver: manifest.protocol_ver,
10432 manifest,
10433 control_ops: None,
10434 launch_nonce: None,
10435 })
10436 .expect("capability test HELLO serializes"),
10437 )
10438 .expect("capability test HELLO frame builds")
10439 }
10440
10441 fn catalog_update_with_capabilities_frame(
10442 corr: u64,
10443 capabilities: CapabilityDeclarations,
10444 ) -> Frame {
10445 Frame::build(
10446 FrameType::Request,
10447 control_flags(),
10448 0,
10449 0,
10450 corr,
10451 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10452 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
10453 capabilities: Some(capabilities),
10454 ready: None,
10455 })
10456 .expect("capability catalog.update serializes"),
10457 )
10458 .expect("capability catalog.update frame builds")
10459 }
10460
10461 async fn register_capability_manifest(
10462 handler: &ControlHandler,
10463 ctx: &RouteCtx,
10464 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10465 manifest: ModuleManifest,
10466 corr: u64,
10467 ) {
10468 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
10469 }
10470
10471 async fn open_route_for_capability_test(
10472 handler: &ControlHandler,
10473 target_ctx: &RouteCtx,
10474 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10475 client_connection_id: u64,
10476 corr: u64,
10477 target_module_id: &str,
10478 consumer_identity: Option<ConsumerIdentity>,
10479 ) -> (
10480 mpsc::Receiver<crate::router::OutboundFrame>,
10481 ModuleControlRequest,
10482 ) {
10483 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
10484 let route_handler = handler.clone();
10485 let target_module_id = target_module_id.to_string();
10486 let route_task = tokio::spawn(async move {
10487 route_handler
10488 .handle_control_frame(
10489 &client_ctx,
10490 route_open_frame_with_admission_facts(
10491 corr,
10492 &target_module_id,
10493 unique_project_root("admission-facts"),
10494 consumer_identity,
10495 None,
10496 ),
10497 )
10498 .await
10499 .expect("capability test route.open succeeds")
10500 });
10501 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
10502 .await
10503 .expect("capability test route.open must reach route.bind")
10504 .expect("target control receiver stays open");
10505 let bind_request: ModuleControlRequest =
10506 serde_json::from_slice(&bind.body).expect("route.bind decodes");
10507 handler
10508 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
10509 .await
10510 .expect("capability test route.bind ACK succeeds");
10511 assert!(route_task.await.expect("route.open task joins").is_empty());
10512 let opened = client_rx
10513 .recv()
10514 .await
10515 .expect("successful route.open publishes a response");
10516 assert!(matches!(
10517 serde_json::from_slice::<ClientControlResponse>(&opened.body),
10518 Ok(ClientControlResponse::RouteOpen { .. })
10519 ));
10520 (client_rx, bind_request)
10521 }
10522
10523 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
10524 assert_eq!(frame.header.ty, FrameType::Push);
10525 assert_eq!(frame.header.channel, 0);
10526 assert_eq!(
10527 serde_json::from_slice::<ClientControlPush>(&frame.body)
10528 .expect("route.closed control push decodes"),
10529 ClientControlPush::RouteClosed {
10530 module_id: target_module_id.to_string(),
10531 reason: RouteCloseReason::CapabilityDenied,
10532 drained: false,
10533 abandoned: 0,
10534 excluded_subscriptions: 0,
10535 terminal: Some(false),
10536 }
10537 );
10538 }
10539
10540 #[tokio::test]
10541 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
10542 let registry = Arc::new(Registry::default());
10543 let forwarding = Arc::new(ForwardingTable::default());
10544 let supervisor = SupervisorHandle::new();
10545 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10546 let handler =
10547 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10548 .with_supervisor(supervisor);
10549 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
10550 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
10551 register_capability_manifest(
10552 &handler,
10553 &target_ctx,
10554 &mut target_rx,
10555 capability_manifest("target", &["credentials-provider/v1"], &[]),
10556 1,
10557 )
10558 .await;
10559 register_capability_manifest(
10560 &handler,
10561 &opener_ctx,
10562 &mut opener_rx,
10563 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10564 2,
10565 )
10566 .await;
10567
10568 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
10569 let replies = handler
10570 .handle_control_frame(
10571 &client_ctx,
10572 route_open_frame_with_admission_facts(
10573 3,
10574 "target",
10575 unique_project_root("admission-facts"),
10576 Some(ConsumerIdentity {
10577 module_id: "opener".to_string(),
10578 launch_nonce: "opener-nonce".to_string(),
10579 }),
10580 None,
10581 ),
10582 )
10583 .await
10584 .expect("denied route.open returns a typed frame");
10585 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10586 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10587 assert!(
10588 target_rx.try_recv().is_err(),
10589 "forbidden route.open must not relay route.bind"
10590 );
10591 }
10592
10593 #[tokio::test]
10594 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
10595 let registry = Arc::new(Registry::default());
10596 let forwarding = Arc::new(ForwardingTable::default());
10597 let supervisor = SupervisorHandle::new();
10598 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10599 let handler =
10600 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10601 .with_supervisor(supervisor);
10602 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
10603 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
10604 register_capability_manifest(
10605 &handler,
10606 &target_ctx,
10607 &mut target_rx,
10608 capability_manifest("target", &["credentials-provider/v1"], &[]),
10609 1,
10610 )
10611 .await;
10612 register_capability_manifest(
10613 &handler,
10614 &old_opener_ctx,
10615 &mut old_opener_rx,
10616 capability_manifest("opener", &[], &[]),
10617 2,
10618 )
10619 .await;
10620 let (mut client_rx, _) = open_route_for_capability_test(
10621 &handler,
10622 &target_ctx,
10623 &mut target_rx,
10624 712,
10625 3,
10626 "target",
10627 Some(ConsumerIdentity {
10628 module_id: "opener".to_string(),
10629 launch_nonce: "opener-nonce".to_string(),
10630 }),
10631 )
10632 .await;
10633 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10634
10635 handler
10636 .cleanup_connection(old_opener_ctx.connection_id)
10637 .expect("old opener registration cleans up");
10638 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
10639 register_capability_manifest(
10640 &handler,
10641 &new_opener_ctx,
10642 &mut new_opener_rx,
10643 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10644 4,
10645 )
10646 .await;
10647
10648 assert_capability_denied_push(
10649 client_rx
10650 .try_recv()
10651 .expect("HELLO deny addition must emit route.closed")
10652 .frame,
10653 "target",
10654 );
10655 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10656 assert!(matches!(
10657 target_rx.try_recv(),
10658 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10659 ));
10660 }
10661
10662 #[tokio::test]
10663 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
10664 let registry = Arc::new(Registry::default());
10665 let forwarding = Arc::new(ForwardingTable::default());
10666 let supervisor = SupervisorHandle::new();
10667 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10668 let handler =
10669 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10670 .with_supervisor(supervisor);
10671 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
10672 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
10673 register_capability_manifest(
10674 &handler,
10675 &target_ctx,
10676 &mut target_rx,
10677 capability_manifest("target", &[], &[]),
10678 1,
10679 )
10680 .await;
10681 register_capability_manifest(
10682 &handler,
10683 &opener_ctx,
10684 &mut opener_rx,
10685 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10686 2,
10687 )
10688 .await;
10689 let (mut client_rx, _) = open_route_for_capability_test(
10690 &handler,
10691 &target_ctx,
10692 &mut target_rx,
10693 722,
10694 3,
10695 "target",
10696 Some(ConsumerIdentity {
10697 module_id: "opener".to_string(),
10698 launch_nonce: "opener-nonce".to_string(),
10699 }),
10700 )
10701 .await;
10702 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10703
10704 let replies = handler
10705 .handle_control_frame(
10706 &target_ctx,
10707 catalog_update_with_capabilities_frame(
10708 4,
10709 CapabilityDeclarations {
10710 provides: vec!["credentials-provider/v1".to_string()],
10711 requires: Vec::new(),
10712 must_never_reach: Vec::new(),
10713 },
10714 ),
10715 )
10716 .await
10717 .expect("claim catalog.update succeeds");
10718 assert!(matches!(
10719 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
10720 Ok(ModuleControlResponseToModule::CatalogUpdate {})
10721 ));
10722 assert_capability_denied_push(
10723 client_rx
10724 .try_recv()
10725 .expect("claim addition must emit route.closed")
10726 .frame,
10727 "target",
10728 );
10729 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10730 assert!(matches!(
10731 target_rx.try_recv(),
10732 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10733 ));
10734 }
10735
10736 #[tokio::test]
10737 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
10738 let registry = Arc::new(Registry::default());
10739 let forwarding = Arc::new(ForwardingTable::default());
10740 let supervisor = SupervisorHandle::new();
10741 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10742 let handler =
10743 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10744 .with_supervisor(supervisor);
10745 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
10746 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
10747 register_capability_manifest(
10748 &handler,
10749 &target_ctx,
10750 &mut target_rx,
10751 capability_manifest("target", &["credentials-provider/v1"], &[]),
10752 1,
10753 )
10754 .await;
10755 register_capability_manifest(
10756 &handler,
10757 &opener_ctx,
10758 &mut opener_rx,
10759 capability_manifest("opener", &[], &[]),
10760 2,
10761 )
10762 .await;
10763 let (mut client_rx, _) = open_route_for_capability_test(
10764 &handler,
10765 &target_ctx,
10766 &mut target_rx,
10767 732,
10768 3,
10769 "target",
10770 Some(ConsumerIdentity {
10771 module_id: "opener".to_string(),
10772 launch_nonce: "opener-nonce".to_string(),
10773 }),
10774 )
10775 .await;
10776 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10777
10778 handler
10779 .handle_control_frame(
10780 &target_ctx,
10781 catalog_update_with_capabilities_frame(
10782 4,
10783 CapabilityDeclarations {
10784 provides: Vec::new(),
10785 requires: Vec::new(),
10786 must_never_reach: Vec::new(),
10787 },
10788 ),
10789 )
10790 .await
10791 .expect("claim removal catalog.update succeeds");
10792 assert_eq!(
10793 forwarding.active_binding_count().unwrap(),
10794 1,
10795 "removing an attested target claim must leave the route census unchanged"
10796 );
10797 assert!(
10798 client_rx.try_recv().is_err(),
10799 "claim removal must not emit route.closed capability_denied"
10800 );
10801 assert!(
10802 target_rx.try_recv().is_err(),
10803 "claim removal must not send the target a route GOODBYE"
10804 );
10805 }
10806
10807 #[tokio::test]
10810 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
10811 let registry = Arc::new(Registry::default());
10812 let forwarding = Arc::new(ForwardingTable::default());
10813 let supervisor = SupervisorHandle::new();
10814 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10815 let handler =
10816 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10817 .with_supervisor(supervisor);
10818 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
10819 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
10820 register_capability_manifest(
10821 &handler,
10822 &target_ctx,
10823 &mut target_rx,
10824 capability_manifest("target", &["credentials-provider/v1"], &[]),
10825 1,
10826 )
10827 .await;
10828 register_capability_manifest(
10829 &handler,
10830 &opener_ctx,
10831 &mut opener_rx,
10832 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10833 2,
10834 )
10835 .await;
10836
10837 let (_client_rx, bind) = open_route_for_capability_test(
10838 &handler,
10839 &target_ctx,
10840 &mut target_rx,
10841 742,
10842 3,
10843 "target",
10844 None,
10845 )
10846 .await;
10847 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10848 panic!("direct scope-honesty route must bind");
10849 };
10850 assert_eq!(principal, Some(Principal::Direct));
10851 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10852 }
10853
10854 #[tokio::test]
10857 async fn must_never_reach_self_route_is_capability_forbidden() {
10858 let registry = Arc::new(Registry::default());
10859 let forwarding = Arc::new(ForwardingTable::default());
10860 let supervisor = SupervisorHandle::new();
10861 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
10862 let handler =
10863 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10864 .with_supervisor(supervisor);
10865 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
10866 register_capability_manifest(
10867 &handler,
10868 &self_ctx,
10869 &mut self_rx,
10870 capability_manifest(
10871 "self-provider",
10872 &["credentials-provider/v1"],
10873 &["credentials-provider/v1"],
10874 ),
10875 1,
10876 )
10877 .await;
10878
10879 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
10880 let replies = handler
10881 .handle_control_frame(
10882 &client_ctx,
10883 route_open_frame_with_admission_facts(
10884 2,
10885 "self-provider",
10886 unique_project_root("admission-facts"),
10887 Some(ConsumerIdentity {
10888 module_id: "self-provider".to_string(),
10889 launch_nonce: "self-nonce".to_string(),
10890 }),
10891 None,
10892 ),
10893 )
10894 .await
10895 .expect("self-route refusal returns a typed frame");
10896 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10897 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10898 assert!(
10899 self_rx.try_recv().is_err(),
10900 "self denial must not relay route.bind"
10901 );
10902 }
10903
10904 #[test]
10905 fn unsupported_channel_zero_frame_returns_error() {
10906 let handler = ControlHandler::default();
10907 let request = Frame::build(
10908 FrameType::Request,
10909 control_flags(),
10910 0,
10911 0,
10912 21,
10913 b"opaque".to_vec(),
10914 )
10915 .unwrap();
10916
10917 let response = handler
10918 .handle_control(ConnectionId::new(1), request)
10919 .unwrap();
10920
10921 assert_eq!(response[0].header.ty, FrameType::Error);
10922 assert_eq!(
10923 parse_error(&response[0])["code"],
10924 "unsupported_control_frame"
10925 );
10926 }
10927
10928 mod swap {
10933 use super::*;
10934
10935 const INCUMBENT: ConnectionId = ConnectionId::new(30);
10936 const CANDIDATE: ConnectionId = ConnectionId::new(40);
10937
10938 struct Swap {
10939 registry: Arc<Registry>,
10940 forwarding: Arc<ForwardingTable>,
10941 handler: ControlHandler,
10942 incumbent_ctx: RouteCtx,
10943 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10944 candidate_ctx: RouteCtx,
10945 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10946 }
10947
10948 async fn swap_with_incumbent() -> Swap {
10949 let registry = Arc::new(Registry::default());
10950 let forwarding = Arc::new(ForwardingTable::default());
10951 let handler =
10952 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10953 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
10954 hello_via_sink(
10955 &handler,
10956 &incumbent_ctx,
10957 &mut incumbent_rx,
10958 hello_frame("aft", PROTOCOL_VERSION, 7),
10959 )
10960 .await;
10961 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
10962 Swap {
10963 registry,
10964 forwarding,
10965 handler,
10966 incumbent_ctx,
10967 incumbent_rx,
10968 candidate_ctx,
10969 candidate_rx,
10970 }
10971 }
10972
10973 fn register_candidate(swap: &Swap, ready: Option<bool>) {
10974 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
10975 candidate_manifest.ready = ready;
10976 let registration = swap
10977 .registry
10978 .register_candidate_with_control_ops(
10979 candidate_manifest,
10980 PROTOCOL_VERSION,
10981 CANDIDATE,
10982 module_baseline_control_ops(),
10983 )
10984 .unwrap();
10985 swap.forwarding
10986 .register_candidate_module_connection(
10987 CANDIDATE,
10988 "aft".to_string(),
10989 PROTOCOL_VERSION,
10990 manifest_concurrency(®istration.manifest),
10991 swap.candidate_ctx.egress.clone(),
10992 )
10993 .unwrap();
10994 }
10995
10996 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
10997 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
10998 swap.registry.promote_candidate("aft").unwrap().unwrap();
10999 cutover.incumbent.unwrap()
11000 }
11001
11002 fn keyed_total(counters: &Value, key: &str) -> u64 {
11003 counters[key]
11004 .as_object()
11005 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
11006 .unwrap_or(0)
11007 }
11008
11009 #[tokio::test]
11015 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
11016 let mut swap = swap_with_incumbent().await;
11017 let handler = swap.handler.clone();
11018
11019 let cotenant = ConnectionId::new(31);
11021 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11022 let (cotenant_task, cotenant_bind) = relay_route_open(
11023 &handler,
11024 cotenant,
11025 &cotenant_ctx.egress,
11026 &mut swap.incumbent_rx,
11027 100,
11028 "aft",
11029 "swap-cotenant",
11030 )
11031 .await;
11032 handler
11033 .handle_control_frame(
11034 &swap.incumbent_ctx,
11035 route_bind_ack(cotenant_bind.header.corr),
11036 )
11037 .await
11038 .unwrap();
11039 assert!(cotenant_task.await.unwrap().is_empty());
11040 let (cotenant_channel, cotenant_epoch) =
11041 published_route(&cotenant_rx.recv().await.unwrap());
11042
11043 let caller = ConnectionId::new(32);
11045 let (caller_ctx, mut caller_rx) = route_ctx(caller);
11046 let (caller_task, caller_bind) = relay_route_open(
11047 &handler,
11048 caller,
11049 &caller_ctx.egress,
11050 &mut swap.incumbent_rx,
11051 101,
11052 "aft",
11053 "swap-caller",
11054 )
11055 .await;
11056 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
11057
11058 register_candidate(&swap, None);
11059 cutover(&swap);
11060
11061 let ack = handler
11063 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
11064 .await;
11065 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11066 if module_loop_error.is_some() {
11067 handler.cleanup_connection(INCUMBENT).unwrap();
11070 }
11071
11072 assert!(
11074 cotenant_rx.try_recv().is_err(),
11075 "the co-tenant route on the incumbent was torn down by one late ack: \
11076 {module_loop_error:?}"
11077 );
11078 assert!(matches!(
11079 swap.forwarding
11080 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11081 .unwrap(),
11082 DataRoute::Client(DataRouteState::Bound(_))
11083 ));
11084 assert_eq!(module_loop_error, None);
11085 assert!(swap
11086 .registry
11087 .get_module_by_connection(INCUMBENT)
11088 .unwrap()
11089 .is_some());
11090
11091 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
11093 .await
11094 .expect("the incumbent is told to drop the abandoned binding")
11095 .unwrap()
11096 .frame;
11097 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11098 assert_eq!(goodbye.header.channel, abandoned_channel);
11099 assert_eq!(goodbye.header.epoch, abandoned_epoch);
11100 assert!(swap.incumbent_rx.try_recv().is_err());
11101
11102 let response = caller_task.await.unwrap();
11104 assert_eq!(response.len(), 1);
11105 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
11106 assert!(caller_rx.try_recv().is_err());
11107
11108 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
11112 let counters = handler.counters().snapshot();
11113 assert_eq!(
11114 keyed_total(&counters, "route_open_accepted_by_principal"),
11115 1
11116 );
11117 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
11118 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
11119 }
11120
11121 #[tokio::test]
11125 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
11126 let mut swap = swap_with_incumbent().await;
11127 register_candidate(&swap, None);
11128 let incumbent = cutover(&swap);
11129 swap.forwarding
11130 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
11131 .unwrap()
11132 .expect("the incumbent is still registered");
11133
11134 let client = ConnectionId::new(33);
11135 let (client_ctx, mut client_rx) = route_ctx(client);
11136 let route_handler = swap.handler.clone();
11137 let open_ctx = RouteCtx {
11138 connection_id: client,
11139 egress: client_ctx.egress.clone(),
11140 };
11141 let mut route_task = tokio::spawn(async move {
11142 route_handler
11143 .handle_control_frame(
11144 &open_ctx,
11145 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
11146 )
11147 .await
11148 .unwrap()
11149 });
11150 let bind = tokio::select! {
11151 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
11152 response = &mut route_task => {
11153 let response = response.unwrap();
11154 panic!(
11155 "post-cutover route.open was refused instead of relayed to the candidate: {}",
11156 parse_error(&response[0])["code"]
11157 );
11158 }
11159 };
11160 swap.handler
11161 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
11162 .await
11163 .unwrap();
11164 assert!(route_task.await.unwrap().is_empty());
11165 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
11166 match swap
11167 .forwarding
11168 .lookup_data_route(client, channel, epoch)
11169 .unwrap()
11170 {
11171 DataRoute::Client(DataRouteState::Bound(route)) => {
11172 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
11173 }
11174 other => panic!("expected a bound route on the candidate, got {other:?}"),
11175 }
11176 assert!(swap.incumbent_rx.try_recv().is_err());
11177 }
11178
11179 #[tokio::test]
11184 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
11185 let swap = swap_with_incumbent().await;
11186 register_candidate(&swap, Some(false));
11187 let update = Frame::build(
11188 FrameType::Request,
11189 control_flags(),
11190 0,
11191 0,
11192 55,
11193 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11194 provides: manifest("aft", PROTOCOL_VERSION).provides,
11195 capabilities: None,
11196 ready: Some(true),
11197 })
11198 .unwrap(),
11199 )
11200 .unwrap();
11201
11202 let replies = swap
11203 .handler
11204 .handle_control_frame(&swap.candidate_ctx, update)
11205 .await
11206 .unwrap();
11207
11208 assert_eq!(replies.len(), 1);
11209 assert_eq!(
11210 replies[0].header.ty,
11211 FrameType::Response,
11212 "candidate catalog.update was refused: {:?}",
11213 serde_json::from_slice::<Value>(&replies[0].body).ok()
11214 );
11215 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
11216 assert_eq!(
11217 swap.registry
11218 .get_module("aft")
11219 .unwrap()
11220 .unwrap()
11221 .connection_id,
11222 INCUMBENT
11223 );
11224 }
11225 }
11226
11227 mod swap_admission {
11231 use super::*;
11232
11233 const INCUMBENT_NONCE: &str = "incumbent-nonce";
11234 const CANDIDATE_NONCE: &str = "candidate-nonce";
11235
11236 fn handler_with_incumbent(
11237 module_id: &str,
11238 reserved: bool,
11239 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
11240 let registry = Arc::new(Registry::default());
11241 let supervisor = SupervisorHandle::new();
11242 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
11243 if reserved {
11244 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
11245 }
11246 let handler =
11247 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
11248 let incumbent = handler
11249 .handle_control(
11250 ConnectionId::new(1),
11251 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
11252 )
11253 .unwrap();
11254 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
11255 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
11256 (registry, supervisor, handler)
11257 }
11258
11259 #[test]
11265 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
11266 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
11267
11268 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
11269 let replies = handler
11270 .handle_control(
11271 ConnectionId::new(connection),
11272 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
11273 )
11274 .unwrap();
11275 assert_eq!(replies[0].header.ty, FrameType::Error);
11276 assert_eq!(
11277 parse_error(&replies[0])["code"],
11278 "swap_token_invalid",
11279 "nonce {nonce:?}"
11280 );
11281 }
11282 assert!(registry.get_candidate("aft").unwrap().is_none());
11283 assert_eq!(
11284 registry.get_module("aft").unwrap().unwrap().connection_id,
11285 ConnectionId::new(1)
11286 );
11287
11288 let admitted = handler
11291 .handle_control(
11292 ConnectionId::new(4),
11293 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
11294 )
11295 .unwrap();
11296 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
11297 assert_eq!(
11298 registry
11299 .get_candidate("aft")
11300 .unwrap()
11301 .unwrap()
11302 .connection_id,
11303 ConnectionId::new(4)
11304 );
11305 assert_eq!(
11306 registry.get_module("aft").unwrap().unwrap().connection_id,
11307 ConnectionId::new(1),
11308 "the candidate must not take the active slot"
11309 );
11310 let replayed = handler
11311 .handle_control(
11312 ConnectionId::new(5),
11313 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
11314 )
11315 .unwrap();
11316 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
11317
11318 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
11322 let squatter = handler
11323 .handle_control(
11324 ConnectionId::new(6),
11325 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
11326 )
11327 .unwrap();
11328 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
11329 assert!(
11330 registry.get_module("aft").unwrap().is_none(),
11331 "a squatter took the active slot of an id being swapped"
11332 );
11333 }
11334
11335 #[test]
11340 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
11341 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
11342
11343 let replies = handler
11344 .handle_control(
11345 ConnectionId::new(2),
11346 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
11347 )
11348 .unwrap();
11349
11350 assert_eq!(
11351 replies[0].header.ty,
11352 FrameType::HelloAck,
11353 "reserved candidate refused: {:?}",
11354 serde_json::from_slice::<Value>(&replies[0].body).ok()
11355 );
11356 assert_eq!(
11357 registry
11358 .get_candidate("vault")
11359 .unwrap()
11360 .unwrap()
11361 .connection_id,
11362 ConnectionId::new(2)
11363 );
11364 }
11365
11366 #[test]
11369 fn without_an_open_swap_the_ordinary_gates_decide() {
11370 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
11371 supervisor.close_swap("vault");
11372
11373 let candidate = handler
11374 .handle_control(
11375 ConnectionId::new(2),
11376 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
11377 )
11378 .unwrap();
11379 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
11380 let duplicate = handler
11381 .handle_control(
11382 ConnectionId::new(3),
11383 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
11384 )
11385 .unwrap();
11386 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11387 assert!(registry.get_candidate("vault").unwrap().is_none());
11388 }
11389 }
11390
11391 mod scopes {
11395 use subc_protocol::scope::{
11396 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
11397 ScopeStatus,
11398 };
11399
11400 use super::*;
11401
11402 const OWNER: &str = "prefrontal-core";
11403
11404 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
11405 ScopeRecord {
11406 scope_ref: scope_ref.to_string(),
11407 scope_epoch,
11408 kind: ScopeKind::Head,
11409 parent: None,
11410 child_owners: Vec::new(),
11411 carriers: Vec::new(),
11412 attributes: Default::default(),
11413 }
11414 }
11415
11416 async fn call(
11417 handler: &ControlHandler,
11418 ctx: &RouteCtx,
11419 request: &ModuleControlRequestFromModule,
11420 ) -> Frame {
11421 let body = serde_json::to_vec(request).unwrap();
11422 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
11423 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
11424 assert_eq!(replies.len(), 1, "{replies:?}");
11425 replies.pop().unwrap()
11426 }
11427
11428 async fn sync(
11429 handler: &ControlHandler,
11430 ctx: &RouteCtx,
11431 generation: u64,
11432 scopes: Vec<ScopeRecord>,
11433 ) -> Result<ModuleControlResponseToModule, String> {
11434 let reply = call(
11435 handler,
11436 ctx,
11437 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
11438 )
11439 .await;
11440 match reply.header.ty {
11441 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
11442 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
11443 }
11444 }
11445
11446 async fn describe(
11447 handler: &ControlHandler,
11448 ctx: &RouteCtx,
11449 owner: &str,
11450 scope_ref: &str,
11451 ) -> ModuleControlResponseToModule {
11452 let reply = call(
11453 handler,
11454 ctx,
11455 &ModuleControlRequestFromModule::ScopeDescribe {
11456 owner: Principal::Reserved {
11457 module_id: owner.to_string(),
11458 },
11459 scope_ref: scope_ref.to_string(),
11460 },
11461 )
11462 .await;
11463 assert_eq!(
11464 reply.header.ty,
11465 FrameType::Response,
11466 "{:?}",
11467 parse_error(&reply)
11468 );
11469 serde_json::from_slice(&reply.body).unwrap()
11470 }
11471
11472 async fn module(
11474 handler: &ControlHandler,
11475 connection: u64,
11476 module_id: &str,
11477 nonce: Option<&str>,
11478 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11479 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
11480 hello_via_sink(
11481 handler,
11482 &ctx,
11483 &mut rx,
11484 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
11485 )
11486 .await;
11487 (ctx, rx)
11488 }
11489
11490 #[tokio::test]
11493 async fn a_client_connection_cannot_sync_or_describe() {
11494 let handler = ControlHandler::new(Arc::new(Registry::default()));
11495 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
11496 for request in [
11497 ModuleControlRequestFromModule::ScopeSync {
11498 generation: 1,
11499 scopes: vec![head("s", 1)],
11500 },
11501 ModuleControlRequestFromModule::ScopeDescribe {
11502 owner: Principal::Direct,
11503 scope_ref: "s".to_string(),
11504 },
11505 ] {
11506 let reply = call(&handler, &ctx, &request).await;
11507 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
11508 }
11509 assert!(
11510 !handler
11511 .scopes
11512 .read()
11513 .unwrap()
11514 .describe(
11515 &Principal::Reserved {
11516 module_id: OWNER.to_string()
11517 },
11518 "s"
11519 )
11520 .owner_synced
11521 );
11522 }
11523
11524 #[tokio::test]
11527 async fn a_module_without_a_supervised_launch_cannot_sync() {
11528 let handler = ControlHandler::new(Arc::new(Registry::default()));
11529 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
11530 assert_eq!(
11531 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
11532 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11533 );
11534 }
11535
11536 #[tokio::test]
11537 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
11538 let supervisor = SupervisorHandle::new();
11539 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
11540 let handler = ControlHandler::new(Arc::new(Registry::default()))
11541 .with_supervisor(supervisor.clone());
11542 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
11543 sync(&handler, &incumbent, 1, vec![head("s", 1)])
11544 .await
11545 .expect("the current launch syncs");
11546
11547 supervisor.open_swap(OWNER, "n2".to_string());
11550 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
11551 assert_eq!(
11552 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
11553 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11554 );
11555 sync(&handler, &incumbent, 2, vec![head("s", 1)])
11556 .await
11557 .expect("the serving owner syncs during the swap");
11558
11559 supervisor.close_swap(OWNER);
11562 assert_eq!(
11563 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
11564 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11565 );
11566 sync(&handler, &incumbent, 3, vec![head("s", 1)])
11567 .await
11568 .expect("the serving owner syncs after the rollback");
11569 handler.cleanup_connection(candidate.connection_id).unwrap();
11570
11571 supervisor.open_swap(OWNER, "n3".to_string());
11576 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
11577 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
11578 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
11579 .await
11580 .expect("the promoted launch takes authority");
11581 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
11582 panic!("unexpected reply {reply:?}");
11583 };
11584 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
11585 assert_eq!(
11586 sync(&handler, &incumbent, 4, Vec::new()).await,
11587 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11588 );
11589 }
11590
11591 #[tokio::test]
11594 async fn closing_the_authority_connection_frees_sync_authority() {
11595 let supervisor = SupervisorHandle::new();
11596 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
11597 let handler = ControlHandler::new(Arc::new(Registry::default()))
11598 .with_supervisor(supervisor.clone());
11599 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
11600 sync(&handler, &first, 10, vec![head("s", 1)])
11601 .await
11602 .unwrap();
11603 handler.cleanup_connection(first.connection_id).unwrap();
11604
11605 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
11606 sync(&handler, &second, 1, vec![head("s", 1)])
11607 .await
11608 .expect("the next connection takes the released authority");
11609 }
11610
11611 #[tokio::test]
11612 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
11613 let registry = Arc::new(Registry::default());
11614 let supervisor_handle = SupervisorHandle::new();
11615 let supervisor = Supervisor::new(Arc::clone(®istry), RestartPolicy::default())
11616 .with_handle(supervisor_handle.clone())
11617 .with_daemon_incarnation("incarnation-7".to_string());
11618 supervisor
11621 .supervise_configured(
11622 ModuleSpec {
11623 launch_nonce_env: true,
11624 module_id: OWNER.to_string(),
11625 program: PathBuf::from("/nonexistent/prefrontal-core"),
11626 args: Vec::new(),
11627 env: Vec::new(),
11628 reserved: false,
11629 reserved_prefixes: Vec::new(),
11630 protocol: ModuleProtocol::Subc,
11631 overlap: Default::default(),
11632 },
11633 false,
11634 )
11635 .unwrap();
11636 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
11637 let handler =
11638 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11639 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
11640
11641 let ModuleControlResponseToModule::ScopeDescribe {
11643 status,
11644 daemon_incarnation,
11645 owner_synced,
11646 owner_configured,
11647 scope,
11648 ..
11649 } = describe(&handler, &reader, OWNER, "s").await
11650 else {
11651 panic!("not a describe reply");
11652 };
11653 assert_eq!(status, ScopeStatus::NotLive);
11654 assert_eq!(daemon_incarnation, "incarnation-7");
11655 assert!(!owner_synced);
11656 assert!(owner_configured);
11657 assert!(scope.is_none());
11658
11659 let ModuleControlResponseToModule::ScopeDescribe {
11662 status,
11663 owner_configured,
11664 ..
11665 } = describe(&handler, &reader, "ghost", "s").await
11666 else {
11667 panic!("not a describe reply");
11668 };
11669 assert_eq!(status, ScopeStatus::NotLive);
11670 assert!(!owner_configured);
11671
11672 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
11674 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
11675 let ModuleControlResponseToModule::ScopeDescribe {
11676 status,
11677 scope_epoch,
11678 owner_synced,
11679 scope,
11680 ..
11681 } = describe(&handler, &reader, OWNER, "s").await
11682 else {
11683 panic!("not a describe reply");
11684 };
11685 assert_eq!(status, ScopeStatus::Live);
11686 assert_eq!(scope_epoch, Some(4));
11687 assert!(owner_synced);
11688 let stamp = scope.expect("a live scope carries its stamp");
11689 assert!(
11690 stamp.owner_authorized,
11691 "prefrontal-core is the default authority"
11692 );
11693 assert_eq!(stamp.kind, ScopeKind::Head);
11694 }
11695
11696 #[tokio::test]
11697 async fn scope_authority_owners_decides_owner_authorized() {
11698 let supervisor = SupervisorHandle::new();
11699 supervisor.set_spawn_nonce("broca", "b1".to_string());
11700 let handler = ControlHandler::new(Arc::new(Registry::default()))
11701 .with_supervisor(supervisor)
11702 .with_scope_authority_owners(vec!["broca".to_string()]);
11703 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
11704 let mut gated = head("s", 1);
11705 gated.attributes.agent_id = Some("agent".to_string());
11706 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
11707 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
11708 describe(&handler, &broca, "broca", "s").await
11709 else {
11710 panic!("not a describe reply");
11711 };
11712 assert!(scope.unwrap().owner_authorized);
11713 }
11714
11715 #[tokio::test]
11719 async fn scope_ops_and_the_scopes_capability_are_advertised() {
11720 let handler = ControlHandler::new(Arc::new(Registry::default()));
11721 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
11722 let ack = hello_via_sink(
11723 &handler,
11724 &ctx,
11725 &mut rx,
11726 hello_frame("m", PROTOCOL_VERSION, 1),
11727 )
11728 .await;
11729 let ack = parse_ack(&ack);
11730 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
11731 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
11732 }
11733 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
11734
11735 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
11736 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11737 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11738 let reply = handler
11739 .handle_control_frame(&client, frame)
11740 .await
11741 .unwrap()
11742 .pop()
11743 .unwrap();
11744 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11745 serde_json::from_slice(&reply.body).unwrap()
11746 else {
11747 panic!("not a server.describe reply");
11748 };
11749 assert!(
11750 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
11751 "{capabilities:?}"
11752 );
11753 }
11754
11755 const PLEXUS: &str = "plexus";
11758 const OTHER: &str = "other";
11759 const AFT: &str = "aft";
11760 const BROCA: &str = "broca";
11761 const MAGIC: &str = "magic-context";
11762
11763 fn nonce(module_id: &str) -> String {
11764 format!("nonce-{module_id}")
11765 }
11766
11767 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11768 let (tx, rx) = mpsc::channel(64);
11769 (
11770 RouteCtx {
11771 connection_id: ConnectionId::new(connection),
11772 egress: FrameSink::new(tx),
11773 },
11774 rx,
11775 )
11776 }
11777
11778 struct Rig {
11782 handler: ControlHandler,
11783 forwarding: Arc<ForwardingTable>,
11784 owner: RouteCtx,
11785 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11786 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
11787 generation: u64,
11788 next_connection: u64,
11789 _supervisor: Supervisor,
11790 }
11791
11792 async fn rig() -> Rig {
11793 let registry = Arc::new(Registry::default());
11794 let forwarding = Arc::new(ForwardingTable::default());
11795 let supervisor_handle = SupervisorHandle::new();
11796 let supervisor = Supervisor::new(Arc::clone(®istry), RestartPolicy::default())
11797 .with_handle(supervisor_handle.clone());
11798 supervisor
11799 .supervise_configured(
11800 ModuleSpec {
11801 launch_nonce_env: true,
11802 module_id: OWNER.to_string(),
11803 program: PathBuf::from("/nonexistent/prefrontal-core"),
11804 args: Vec::new(),
11805 env: Vec::new(),
11806 reserved: false,
11807 reserved_prefixes: Vec::new(),
11808 protocol: ModuleProtocol::Subc,
11809 overlap: Default::default(),
11810 },
11811 false,
11812 )
11813 .unwrap();
11814 for module_id in [OWNER, AFT, BROCA, MAGIC] {
11815 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
11816 }
11817 let handler =
11818 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11819 .with_supervisor(supervisor_handle);
11820 let (owner, mut owner_rx) = wide_ctx(1);
11821 hello_via_sink(
11822 &handler,
11823 &owner,
11824 &mut owner_rx,
11825 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
11826 )
11827 .await;
11828 let mut modules = BTreeMap::new();
11829 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
11830 let (ctx, mut rx) = wide_ctx(connection);
11831 hello_via_sink(
11832 &handler,
11833 &ctx,
11834 &mut rx,
11835 hello_frame(module_id, PROTOCOL_VERSION, connection),
11836 )
11837 .await;
11838 modules.insert(module_id.to_string(), (ctx, rx));
11839 }
11840 Rig {
11841 handler,
11842 forwarding,
11843 owner,
11844 _owner_rx: owner_rx,
11845 modules,
11846 generation: 0,
11847 next_connection: 100,
11848 _supervisor: supervisor,
11849 }
11850 }
11851
11852 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
11853 ScopeCarrier {
11854 principal: Principal::Reserved {
11855 module_id: module_id.to_string(),
11856 },
11857 targets: targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()),
11858 }
11859 }
11860
11861 fn session(scope_epoch: u64) -> ScopeRecord {
11864 let mut record = head("s", scope_epoch);
11865 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
11866 record.attributes.agent_id = Some("agent-1".to_string());
11867 record.attributes.delegates = true;
11868 record
11869 }
11870
11871 impl Rig {
11872 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
11873 self.generation += 1;
11874 sync(&self.handler, &self.owner, self.generation, scopes)
11875 .await
11876 .expect("the owner's sync is accepted");
11877 }
11878
11879 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
11880 ScopeSelector {
11881 owner: Principal::Reserved {
11882 module_id: OWNER.to_string(),
11883 },
11884 scope_ref: scope_ref.to_string(),
11885 scope_epoch,
11886 }
11887 }
11888
11889 fn open_frame(
11890 &mut self,
11891 opener: Option<&str>,
11892 target: &str,
11893 scope: Option<ScopeSelector>,
11894 ) -> (
11895 RouteCtx,
11896 mpsc::Receiver<crate::router::OutboundFrame>,
11897 Frame,
11898 ) {
11899 self.next_connection += 1;
11900 let (ctx, rx) = wide_ctx(self.next_connection);
11901 let root = unique_project_root("scoped-open");
11902 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11903 target: RouteTarget::ToolProvider {
11904 module_id: target.to_string(),
11905 },
11906 identity: BindIdentity::new(
11907 root.path().to_path_buf(),
11908 "unit".to_string(),
11909 "session".to_string(),
11910 ),
11911 consumer_identity: opener.map(|module_id| ConsumerIdentity {
11912 module_id: module_id.to_string(),
11913 launch_nonce: nonce(module_id),
11914 }),
11915 consumer_capabilities: None,
11916 admission_facts: None,
11917 scope,
11918 })
11919 .unwrap();
11920 let frame = Frame::build(
11921 FrameType::Request,
11922 control_flags(),
11923 0,
11924 0,
11925 self.next_connection,
11926 body,
11927 )
11928 .unwrap();
11929 (ctx, rx, frame)
11930 }
11931
11932 async fn refused(
11934 &mut self,
11935 opener: Option<&str>,
11936 target: &str,
11937 scope: Option<ScopeSelector>,
11938 ) -> String {
11939 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
11940 let replies = self
11941 .handler
11942 .handle_control_frame(&ctx, frame)
11943 .await
11944 .unwrap();
11945 assert_eq!(replies.len(), 1, "{replies:?}");
11946 assert_eq!(replies[0].header.ty, FrameType::Error);
11947 let (_, module_rx) = self.modules.get_mut(target).unwrap();
11948 assert!(
11949 module_rx.try_recv().is_err(),
11950 "a refused open relays nothing"
11951 );
11952 parse_error(&replies[0])["code"]
11953 .as_str()
11954 .unwrap()
11955 .to_string()
11956 }
11957
11958 async fn relayed(
11960 &mut self,
11961 opener: Option<&str>,
11962 target: &str,
11963 scope: Option<ScopeSelector>,
11964 ) -> Relayed {
11965 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
11966 let handler = self.handler.clone();
11967 let task_ctx = ctx.clone();
11968 let task = tokio::spawn(async move {
11969 handler
11970 .handle_control_frame(&task_ctx, frame)
11971 .await
11972 .unwrap()
11973 });
11974 let (_, module_rx) = self.modules.get_mut(target).unwrap();
11975 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
11976 .await
11977 .expect("the target receives the relayed route.bind")
11978 .unwrap()
11979 .frame;
11980 Relayed {
11981 target: target.to_string(),
11982 client: ctx,
11983 client_rx: rx,
11984 task,
11985 bind,
11986 }
11987 }
11988
11989 async fn ack(&self, relayed: &Relayed) {
11990 let (module, _) = &self.modules[&relayed.target];
11991 self.handler
11992 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
11993 .await
11994 .unwrap();
11995 }
11996
11997 async fn bound(
11999 &mut self,
12000 opener: Option<&str>,
12001 target: &str,
12002 scope: Option<ScopeSelector>,
12003 ) -> Bound {
12004 let relayed = self.relayed(opener, target, scope).await;
12005 self.ack(&relayed).await;
12006 let Relayed {
12007 target,
12008 client,
12009 mut client_rx,
12010 task,
12011 bind,
12012 } = relayed;
12013 assert!(
12014 task.await.unwrap().is_empty(),
12015 "the open is answered by commit"
12016 );
12017 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
12018 Bound {
12019 target,
12020 client,
12021 client_rx,
12022 channel,
12023 epoch,
12024 bind,
12025 }
12026 }
12027
12028 fn live(&self, route: &Bound) -> bool {
12029 matches!(
12030 self.forwarding
12031 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
12032 .unwrap(),
12033 DataRoute::Client(DataRouteState::Bound(_))
12034 )
12035 }
12036 }
12037
12038 struct Relayed {
12039 target: String,
12040 client: RouteCtx,
12041 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12042 task: tokio::task::JoinHandle<Vec<Frame>>,
12043 bind: Frame,
12044 }
12045
12046 struct Bound {
12047 target: String,
12048 client: RouteCtx,
12049 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12050 channel: u16,
12051 epoch: u32,
12052 bind: Frame,
12053 }
12054
12055 impl Bound {
12056 fn closed_reason(&mut self) -> RouteCloseReason {
12059 let mut reason = None;
12060 let mut goodbye = false;
12061 while let Ok(outbound) = self.client_rx.try_recv() {
12062 let frame = outbound.frame;
12063 match frame.header.ty {
12064 FrameType::Goodbye => {
12065 assert_eq!(
12066 (frame.header.channel, frame.header.epoch),
12067 (self.channel, self.epoch)
12068 );
12069 goodbye = true;
12070 }
12071 FrameType::Push => {
12072 let ClientControlPush::RouteClosed {
12073 reason: r,
12074 module_id,
12075 ..
12076 } = serde_json::from_slice(&frame.body).unwrap()
12077 else {
12078 panic!("unexpected push");
12079 };
12080 assert_eq!(module_id, self.target);
12081 reason = Some(r);
12082 }
12083 other => panic!("unexpected frame {other:?}"),
12084 }
12085 }
12086 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
12087 reason.expect("the client is told why the route closed")
12088 }
12089
12090 fn untouched(&mut self) -> bool {
12091 self.client_rx.try_recv().is_err()
12092 }
12093
12094 fn stamp(&self) -> Option<ScopeStamp> {
12095 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
12096 ModuleControlRequest::RouteBind { scope, .. } => scope,
12097 other => panic!("expected a route.bind, got {other:?}"),
12098 }
12099 }
12100 }
12101
12102 #[tokio::test]
12103 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
12104 ) {
12105 let mut rig = rig().await;
12106 let mut record = session(1);
12107 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
12108 record.child_owners = vec![Principal::Reserved {
12109 module_id: MAGIC.to_string(),
12110 }];
12111 rig.sync(vec![record]).await;
12112 let scope = || Some(rig_selector("s", Some(1)));
12113
12114 rig.bound(Some(OWNER), PLEXUS, scope()).await;
12117 rig.bound(Some(AFT), OTHER, scope()).await;
12118 rig.bound(Some(BROCA), PLEXUS, scope()).await;
12119
12120 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
12124 assert_eq!(
12125 rig.refused(opener, target, scope()).await,
12126 error_codes::SCOPE_NOT_CARRIER,
12127 "{opener:?} -> {target}"
12128 );
12129 }
12130 }
12131
12132 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12133 ScopeSelector {
12134 owner: Principal::Reserved {
12135 module_id: OWNER.to_string(),
12136 },
12137 scope_ref: scope_ref.to_string(),
12138 scope_epoch,
12139 }
12140 }
12141
12142 #[tokio::test]
12143 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
12144 let mut rig = rig().await;
12145 rig.sync(vec![session(1)]).await;
12146 for opener in [OWNER, AFT] {
12147 assert_eq!(
12148 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
12149 .await,
12150 error_codes::SCOPE_EPOCH_REQUIRED,
12151 "{opener}"
12152 );
12153 }
12154 }
12155
12156 #[tokio::test]
12157 async fn admission_separates_not_synced_not_live_and_ended() {
12158 let mut rig = rig().await;
12159 let code = rig
12161 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12162 .await;
12163 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
12164 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
12165
12166 let ghost = ScopeSelector {
12168 owner: Principal::Reserved {
12169 module_id: "ghost".to_string(),
12170 },
12171 scope_ref: "s".to_string(),
12172 scope_epoch: Some(1),
12173 };
12174 assert_eq!(
12175 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
12176 error_codes::SCOPE_NOT_LIVE
12177 );
12178
12179 rig.sync(vec![session(2)]).await;
12180 assert_eq!(
12181 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
12182 .await,
12183 error_codes::SCOPE_NOT_LIVE
12184 );
12185 for epoch in [1, 3] {
12186 assert_eq!(
12187 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
12188 .await,
12189 error_codes::SCOPE_ENDED,
12190 "epoch {epoch}"
12191 );
12192 }
12193 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
12195 .await;
12196 }
12197
12198 #[tokio::test]
12199 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
12200 let mut rig = rig().await;
12201 rig.sync(vec![session(1)]).await;
12202 let route = rig
12203 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12204 .await;
12205 let stamp = route.stamp().expect("a scoped bind carries the stamp");
12206 assert_eq!(stamp.scope_ref, "s");
12207 assert_eq!(stamp.scope_epoch, 1);
12208 assert_eq!(stamp.kind, ScopeKind::Head);
12209 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
12210 assert!(stamp.attributes.delegates);
12211 assert!(stamp.owner_authorized);
12212 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
12213 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
12214
12215 let (broca, mut broca_rx) = wide_ctx(50);
12218 hello_via_sink(
12219 &rig.handler,
12220 &broca,
12221 &mut broca_rx,
12222 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
12223 )
12224 .await;
12225 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
12226 .await
12227 .unwrap();
12228 let own = ScopeSelector {
12229 owner: Principal::Reserved {
12230 module_id: BROCA.to_string(),
12231 },
12232 scope_ref: "b".to_string(),
12233 scope_epoch: Some(1),
12234 };
12235 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
12236 assert!(!route.stamp().unwrap().owner_authorized);
12237 }
12238
12239 #[tokio::test]
12244 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
12245 let mut rig = rig().await;
12246 rig.sync(vec![session(1)]).await;
12247 let mut cotenant = rig
12248 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
12249 .await;
12250
12251 let mut changed = session(1);
12252 changed.child_owners.push(Principal::Reserved {
12253 module_id: MAGIC.to_string(),
12254 });
12255 let mut ended = None;
12256 for (code, next) in [
12257 (error_codes::SCOPE_CHANGED, vec![changed]),
12258 (error_codes::SCOPE_ENDED, Vec::new()),
12259 ] {
12260 let relayed = rig
12261 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12262 .await;
12263 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
12264 ended = Some(next.is_empty());
12265 rig.sync(next).await;
12266 rig.ack(&relayed).await;
12267 let replies = relayed.task.await.unwrap();
12268 assert_eq!(replies.len(), 1, "{replies:?}");
12269 assert_eq!(parse_error(&replies[0])["code"], code);
12270 assert_eq!(
12271 subc_protocol::error_codes::is_retryable_route_open(code),
12272 code == error_codes::SCOPE_CHANGED
12273 );
12274 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
12275 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12277 let mut goodbyes = Vec::new();
12280 while let Ok(outbound) = plexus_rx.try_recv() {
12281 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
12282 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
12283 }
12284 assert!(
12285 goodbyes.contains(&(bind_channel, bind_epoch)),
12286 "{goodbyes:?}"
12287 );
12288 assert!(rig
12289 .handler
12290 .registry
12291 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
12292 .unwrap()
12293 .is_some());
12294 }
12295 assert_eq!(ended, Some(true));
12296 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
12299 }
12300
12301 #[tokio::test]
12304 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
12305 struct Case {
12306 name: &'static str,
12307 change: fn(&mut ScopeRecord),
12308 closed: [Option<RouteCloseReason>; 4],
12311 }
12312 use RouteCloseReason::*;
12313 let cases = [
12314 Case {
12315 name: "a carrier entry removed",
12316 change: |r| {
12317 r.carriers.retain(|c| {
12318 c.principal
12319 != Principal::Reserved {
12320 module_id: AFT.to_string(),
12321 }
12322 })
12323 },
12324 closed: [None, Some(ScopeCarrierRemoved), None, None],
12325 },
12326 Case {
12327 name: "a target removed from a carrier",
12328 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
12329 closed: [None, None, None, Some(ScopeCarrierRemoved)],
12330 },
12331 Case {
12332 name: "a bare carrier narrowed to targets",
12333 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
12334 closed: [None, Some(ScopeCarrierRemoved), None, None],
12335 },
12336 Case {
12337 name: "delegates turned off",
12338 change: |r| r.attributes.delegates = false,
12339 closed: [Some(ScopeDelegationChanged); 4],
12340 },
12341 Case {
12342 name: "agent_id changed",
12343 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
12344 closed: [Some(ScopeDelegationChanged); 4],
12345 },
12346 Case {
12347 name: "a carrier added, child owners changed, the record re-sent",
12348 change: |r| {
12349 r.carriers.push(carrier(MAGIC, None));
12350 r.child_owners.push(Principal::Reserved {
12351 module_id: MAGIC.to_string(),
12352 });
12353 },
12354 closed: [None; 4],
12355 },
12356 Case {
12357 name: "a target added",
12358 change: |r| {
12359 r.carriers[1]
12360 .targets
12361 .as_mut()
12362 .unwrap()
12363 .push("third".to_string())
12364 },
12365 closed: [None; 4],
12366 },
12367 Case {
12368 name: "delegates turned on",
12369 change: |r| r.attributes.delegates = true,
12370 closed: [None; 4],
12371 },
12372 ];
12373 for case in cases {
12374 let mut rig = rig().await;
12375 rig.sync(vec![session(1)]).await;
12376 let scope = || Some(rig_selector("s", Some(1)));
12377 let mut routes = [
12378 rig.bound(Some(OWNER), PLEXUS, scope()).await,
12379 rig.bound(Some(AFT), PLEXUS, scope()).await,
12380 rig.bound(Some(BROCA), PLEXUS, scope()).await,
12381 rig.bound(Some(BROCA), OTHER, scope()).await,
12382 ];
12383 let mut record = session(1);
12384 (case.change)(&mut record);
12385 rig.sync(vec![record]).await;
12386 for (index, expected) in case.closed.iter().enumerate() {
12387 let route = &mut routes[index];
12388 match expected {
12389 Some(reason) => {
12390 assert!(!rig.live(route), "{}: route {index} still live", case.name);
12391 assert_eq!(
12392 route.closed_reason(),
12393 *reason,
12394 "{}: route {index}",
12395 case.name
12396 );
12397 }
12398 None => {
12399 assert!(rig.live(route), "{}: route {index} closed", case.name);
12400 assert!(
12401 route.untouched(),
12402 "{}: route {index} was told something",
12403 case.name
12404 );
12405 }
12406 }
12407 }
12408 }
12409 }
12410
12411 #[tokio::test]
12412 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
12413 for next in [Vec::new(), vec![session(2)]] {
12415 let mut rig = rig().await;
12416 rig.sync(vec![session(1)]).await;
12417 let mut route = rig
12418 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12419 .await;
12420 rig.sync(next).await;
12421 assert!(!rig.live(&route));
12422 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
12423 }
12424
12425 let mut rig = rig().await;
12428 let mut child = session(1);
12429 child.scope_ref = "child".to_string();
12430 child.kind = ScopeKind::Worker;
12431 child.parent = Some(ScopeParent {
12432 owner: Principal::Reserved {
12433 module_id: OWNER.to_string(),
12434 },
12435 scope_ref: "s".to_string(),
12436 scope_epoch: 1,
12437 });
12438 rig.sync(vec![session(1), child.clone()]).await;
12439 let mut child_route = rig
12440 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
12441 .await;
12442 assert_eq!(
12443 child_route.stamp().unwrap().parent_state,
12444 Some(ParentState::Linked)
12445 );
12446 rig.sync(vec![child]).await;
12447 assert!(!rig.live(&child_route));
12448 assert_eq!(
12449 child_route.closed_reason(),
12450 RouteCloseReason::ScopeParentEnded
12451 );
12452 }
12453
12454 #[tokio::test]
12455 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
12456 ) {
12457 let mut rig = rig().await;
12458 rig.sync(vec![session(1)]).await;
12459 let mut route = rig
12460 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12461 .await;
12462 let before = rig.forwarding.published_scope_tag(OWNER, "s");
12463 rig.sync(vec![session(1)]).await;
12464 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
12465 assert!(rig.live(&route) && route.untouched());
12466
12467 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
12471 .forwarding
12472 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
12473 .unwrap()
12474 else {
12475 panic!("the route is bound");
12476 };
12477 binding.flow.acquire_tagged(9, false).await.unwrap();
12478 let mut widened = session(1);
12479 widened.carriers.push(carrier(MAGIC, None));
12480 rig.sync(vec![widened]).await;
12481 assert!(rig.live(&route) && route.untouched());
12482 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12483 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
12484 assert_eq!(binding.flow.in_flight(), 1);
12487 binding
12488 .flow
12489 .acquire_tagged(10, false)
12490 .await
12491 .expect("the flow is still open");
12492 }
12493
12494 #[tokio::test]
12497 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
12498 let mut rig = rig().await;
12499 rig.sync(vec![session(1)]).await;
12500 let mut on_incumbent = rig
12501 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12502 .await;
12503
12504 let (candidate, _candidate_rx) = wide_ctx(9);
12507 let registration = rig
12508 .handler
12509 .registry
12510 .register_candidate_with_control_ops(
12511 manifest(PLEXUS, PROTOCOL_VERSION),
12512 PROTOCOL_VERSION,
12513 candidate.connection_id,
12514 module_baseline_control_ops(),
12515 )
12516 .unwrap();
12517 rig.forwarding
12518 .register_candidate_module_connection(
12519 candidate.connection_id,
12520 PLEXUS.to_string(),
12521 PROTOCOL_VERSION,
12522 manifest_concurrency(®istration.manifest),
12523 candidate.egress.clone(),
12524 )
12525 .unwrap();
12526 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
12527 rig.handler
12528 .registry
12529 .promote_candidate(PLEXUS)
12530 .unwrap()
12531 .unwrap();
12532 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
12533
12534 rig.sync(Vec::new()).await;
12535 assert!(!rig.live(&on_incumbent));
12536 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
12537 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12538 let goodbye = incumbent_rx
12539 .try_recv()
12540 .expect("the superseded endpoint is told")
12541 .frame;
12542 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12543 }
12544 }
12545}
12546
12547#[cfg(test)]
12548mod concurrency_default_exposure_tests {
12549 use super::*;
12550
12551 fn hello_body(role_json: &str) -> Vec<u8> {
12552 format!(
12553 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
12554 )
12555 .into_bytes()
12556 }
12557
12558 fn manifest_from(body: &[u8]) -> ModuleManifest {
12559 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
12560 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
12561 .expect("manifest parses")
12562 }
12563
12564 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
12565
12566 #[test]
12567 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
12568 let body = hello_body(&format!(
12569 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
12570 ));
12571 let manifest = manifest_from(&body);
12572 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
12575 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
12576 }
12577
12578 #[test]
12579 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
12580 let body = hello_body(&format!(
12581 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
12582 ));
12583 let manifest = manifest_from(&body);
12584 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
12585 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
12586 }
12587
12588 #[test]
12589 fn non_management_roles_are_never_reported() {
12590 let body = hello_body(
12591 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
12592 );
12593 let manifest = manifest_from(&body);
12594 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
12595 }
12596}