1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24 ProviderRole,
25 },
26 scope::{ScopeRecord, ScopeSelector, CAP_SCOPES_V1, SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP},
27 session::{
28 HealthReport, ModuleControlPush, ModuleControlRequest, ModuleControlRequestFromModule,
29 ModuleControlResponse, ModuleControlResponseToModule, MODULE_CONTROL_OP_HEALTH_CHECK,
30 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
31 },
32 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
33 Priority, RouteTarget, PROTOCOL_VERSION,
34};
35use tokio::time::{timeout_at, Instant};
36use tracing::{debug, info, warn};
37
38use crate::{
39 capability_requirements::{
40 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
41 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
42 RuntimeModule,
43 },
44 daemon_config::RestartRequiredSection,
45 forwarding::{
46 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
47 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
48 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
49 },
50 observability::{
51 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
52 },
53 provenance::{
54 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
55 },
56 registry::{ChannelState, ConnectionId, Registry, RegistryError},
57 router::{RouteCtx, RouterError},
58 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
59 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
60 stderr_tail::{CaptureState, TailEntry},
61 supervise::{
62 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
63 SupervisorHandle, SwapHelloAdmission,
64 },
65 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
66};
67
68pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
74
75const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
76const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
77const CAP_PING_PONG: &str = "ping_pong_v1";
78const CAP_SESSION_ATTACH: &str = "session_attach_v1";
79const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
80
81const SUBC_CONTROL_OPS: &[&str] = &[
82 ops::SERVER_DESCRIBE,
83 ops::CATALOG_LIST,
84 ops::ROUTE_OPEN,
85 ops::ROUTE_POLL,
86 ops::ROUTE_CLOSING,
87 ops::ROUTE_CLOSED,
88 ops::SUPERVISOR_LIST,
89 ops::SUPERVISOR_RESTART,
90 ops::SUPERVISOR_SWAP,
91 ops::SUPERVISOR_RELOAD,
92 ops::SUPERVISOR_RESCAN,
93 ops::SUPERVISOR_RELEASE_RESERVED,
94 ops::SUPERVISOR_SET_ENABLED,
95 ops::SUPERVISOR_HEALTH_PROBE,
96 ops::SUPERVISOR_HEALTH,
97 ops::SUPERVISOR_STDERR_TAIL,
98 ops::SUPERVISOR_TERMINALS,
99 ops::SUPERVISOR_ROUTES,
100 ops::SUPERVISOR_PROVENANCE,
101 ops::SUPERVISOR_SPAWN_SNAPSHOT,
102 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
103];
104
105const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
106 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
107 "supervisor.live_roots",
108 SCOPE_SYNC_OP,
109 SCOPE_DESCRIBE_OP,
110];
111
112const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
116
117const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
118
119pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
127
128pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
143
144pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
162
163const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
164const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
165
166fn reload_verdict(
167 configured: &Path,
168 spawned_from: Option<&Path>,
169 image: subc_control::RunningImageAgreement,
170) -> PendingReloadVerdict {
171 let path = match spawned_from {
172 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
173 Some(spawned_from) => ReloadPathAgreement::Mismatch {
174 configured: configured.to_path_buf(),
175 spawned_from: spawned_from.to_path_buf(),
176 },
177 None => ReloadPathAgreement::Unavailable {
178 reason: if matches!(
179 image,
180 subc_control::RunningImageAgreement::Unavailable {
181 reason: subc_control::RunningImageUnavailableReason::NotRunning
182 }
183 ) {
184 ReloadPathUnavailableReason::NotRunning
185 } else {
186 ReloadPathUnavailableReason::SpawnedPathUnavailable
187 },
188 },
189 };
190 PendingReloadVerdict { path, image }
191}
192
193#[derive(Clone)]
194struct DaemonProvenanceFacts {
195 build: DaemonBuildProvenance,
196 pid: Option<u32>,
197 started_at_ms: Option<u64>,
198 start_clock: Option<crate::clock::StartClock>,
199 executable_path: Option<PathBuf>,
200 executable_identity: Option<SpawnedFileIdentity>,
201 process_start_time: Option<u64>,
202 probe: ExecutableIdentityProbe,
203}
204
205impl Default for DaemonProvenanceFacts {
206 fn default() -> Self {
207 Self {
208 build: DaemonBuildProvenance {
209 build_git_sha: None,
210 build_lock_digest: None,
211 },
212 pid: None,
213 started_at_ms: None,
214 start_clock: None,
215 executable_path: None,
216 executable_identity: None,
217 process_start_time: None,
218 probe: ExecutableIdentityProbe::default(),
219 }
220 }
221}
222
223#[derive(Debug, Clone)]
224struct SupervisorRescanContext {
225 supervisor: Supervisor,
226 config_path: PathBuf,
227 configured_port: Option<u16>,
228 storage_config: Option<crate::daemon_config::StorageConfig>,
229 admission_facts_carrier_module_id: Option<String>,
230 admission_facts_targets: Option<Vec<String>>,
231 scope_authority_owners: Vec<String>,
232}
233
234const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
255 "reloading",
256 "supervisor_not_live",
257 "registration_not_active",
258 "no_forwarding_connection",
259 "relay_send_failed",
260];
261
262#[derive(Clone)]
264pub struct ControlHandler {
265 registry: Arc<Registry>,
266 forwarding: Arc<ForwardingTable>,
267 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
268 supervisor: SupervisorHandle,
269 subc_capabilities: Arc<[String]>,
270 route_bind_relay_timeout: Duration,
274 route_bind_relay_timeouts: BTreeMap<String, Duration>,
278 route_bind_breakers: RouteBindBreakers,
281 route_bind_concurrency: RouteBindConcurrency,
284 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
289 route_bind_breaker_threshold: u32,
291 route_bind_breaker_cooldown: Duration,
293 health_probe_timeout: Duration,
294 storage_config: Option<crate::daemon_config::StorageConfig>,
297 machine_id: Option<crate::machine_id::MachineId>,
301 admission_facts_carrier_module_id: Option<String>,
302 admission_facts_targets: Option<Vec<String>>,
303 scopes: Arc<RwLock<ScopeTable>>,
307 scope_authority_owners: Vec<String>,
310 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
313 rescan: Option<SupervisorRescanContext>,
314 connected_clients: ConnectedClients,
315 counters: DaemonCounters,
316 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
317 daemon_provenance: DaemonProvenanceFacts,
318 #[cfg(test)]
319 control_dispatch_delay: Option<Duration>,
320 #[cfg(test)]
321 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
322}
323
324impl fmt::Debug for ControlHandler {
325 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
326 f.debug_struct("ControlHandler")
327 .field("registry", &self.registry)
328 .field("forwarding", &self.forwarding)
329 .field("process_liveness", &self.process_liveness.is_some())
330 .field("supervisor", &self.supervisor)
331 .field("subc_capabilities", &self.subc_capabilities)
332 .finish()
333 }
334}
335
336struct RouteOpenRequest {
337 target: RouteTarget,
338 identity: BindIdentity,
339 consumer_identity: Option<ConsumerIdentity>,
340 consumer_capabilities: Option<Vec<String>>,
341 admission_facts: Option<serde_json::Value>,
342 scope: Option<ScopeSelector>,
343}
344
345struct RouteBindReservationGuard {
346 forwarding: Arc<ForwardingTable>,
347 endpoint: ModuleEndpointId,
348 relay_corr: u64,
349 armed: bool,
350}
351
352struct ModuleControlRpcGuard {
353 forwarding: Arc<ForwardingTable>,
354 endpoint: ModuleEndpointId,
355 corr: u64,
356 armed: bool,
357}
358
359impl ModuleControlRpcGuard {
360 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
361 Self {
362 forwarding,
363 endpoint,
364 corr,
365 armed: true,
366 }
367 }
368
369 fn disarm(&mut self) {
370 self.armed = false;
371 }
372}
373
374impl Drop for ModuleControlRpcGuard {
375 fn drop(&mut self) {
376 if self.armed {
377 let _ = self
378 .forwarding
379 .cancel_module_control_rpc(self.endpoint, self.corr);
380 }
381 }
382}
383
384impl RouteBindReservationGuard {
385 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
386 Self {
387 forwarding,
388 endpoint,
389 relay_corr,
390 armed: true,
391 }
392 }
393
394 fn release_and_disarm(&mut self) {
395 if !self.armed {
396 return;
397 }
398 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
399 self.endpoint,
400 self.relay_corr,
401 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
402 ) {
403 send_goodbye_target_best_effort(
404 &self.forwarding.counters(),
405 &target,
406 "canceled route.bind",
407 );
408 }
409 self.armed = false;
410 }
411
412 fn disarm(&mut self) {
413 self.armed = false;
414 }
415}
416
417impl Drop for RouteBindReservationGuard {
418 fn drop(&mut self) {
419 self.release_and_disarm();
420 }
421}
422
423#[derive(Debug, Clone, Default)]
453pub(crate) struct RouteBindBreakers {
454 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
455}
456
457#[derive(Debug, Clone, Default)]
458pub(crate) struct RouteBindConcurrency {
459 modules: Arc<Mutex<HashMap<String, usize>>>,
460}
461
462struct RouteBindConcurrencyGuard {
463 concurrency: RouteBindConcurrency,
464 module_id: String,
465}
466
467impl RouteBindConcurrency {
468 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
471 let mut modules = self
472 .modules
473 .lock()
474 .expect("route.bind concurrency mutex poisoned");
475 let in_flight = modules.entry(module_id.to_string()).or_default();
476 if *in_flight >= limit {
477 return Err(*in_flight);
478 }
479 *in_flight += 1;
480 Ok(RouteBindConcurrencyGuard {
481 concurrency: self.clone(),
482 module_id: module_id.to_string(),
483 })
484 }
485}
486
487impl Drop for RouteBindConcurrencyGuard {
488 fn drop(&mut self) {
489 let mut modules = self
490 .concurrency
491 .modules
492 .lock()
493 .expect("route.bind concurrency mutex poisoned");
494 let remove = {
495 let in_flight = modules
496 .get_mut(&self.module_id)
497 .expect("admitted route.bind has a concurrency entry");
498 *in_flight -= 1;
499 *in_flight == 0
500 };
501 if remove {
502 modules.remove(&self.module_id);
503 }
504 }
505}
506
507#[derive(Debug, Default)]
508struct ModuleBreakerState {
509 consecutive_timeouts: u32,
511 cooldown_until: Option<Instant>,
514 probe_in_flight: bool,
519}
520
521enum RouteBindAdmission<'a> {
523 Admitted {
524 guard: RouteBindBreakerGuard<'a>,
525 probe: bool,
528 },
529 Refused {
530 consecutive_timeouts: u32,
531 retry_in: Duration,
535 probe_in_flight: bool,
536 },
537}
538
539struct RouteBindBreakerGuard<'a> {
546 breakers: RouteBindBreakers,
547 module_id: &'a str,
548 settled: bool,
549}
550
551impl RouteBindBreakerGuard<'_> {
552 fn record_accepted(&mut self) -> bool {
556 self.settled = true;
557 self.breakers.record_accepted(self.module_id)
558 }
559
560 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
563 self.settled = true;
564 self.breakers
565 .record_timeout(self.module_id, threshold, cooldown)
566 }
567
568 fn record_inconclusive(&mut self) {
577 self.settled = true;
578 self.breakers.record_inconclusive(self.module_id);
579 }
580}
581
582impl Drop for RouteBindBreakerGuard<'_> {
583 fn drop(&mut self) {
584 if !self.settled {
585 self.breakers.record_inconclusive(self.module_id);
586 }
587 }
588}
589
590struct BreakerOpened {
594 consecutive_timeouts: u32,
595 reopened_after_probe: bool,
598}
599
600impl RouteBindBreakers {
601 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
602 self.modules
603 .lock()
604 .expect("route.bind breaker mutex poisoned")
605 }
606
607 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
610 let admitted = |probe| RouteBindAdmission::Admitted {
611 guard: RouteBindBreakerGuard {
612 breakers: self.clone(),
613 module_id,
614 settled: false,
615 },
616 probe,
617 };
618
619 let mut modules = self.lock();
620 let Some(state) = modules.get_mut(module_id) else {
621 return admitted(false);
622 };
623 let Some(cooldown_until) = state.cooldown_until else {
624 return admitted(false);
625 };
626 if state.probe_in_flight {
627 return RouteBindAdmission::Refused {
628 consecutive_timeouts: state.consecutive_timeouts,
629 retry_in: Duration::ZERO,
630 probe_in_flight: true,
631 };
632 }
633 let now = Instant::now();
634 if now < cooldown_until {
635 return RouteBindAdmission::Refused {
636 consecutive_timeouts: state.consecutive_timeouts,
637 retry_in: cooldown_until - now,
638 probe_in_flight: false,
639 };
640 }
641 state.probe_in_flight = true;
642 admitted(true)
643 }
644
645 fn record_accepted(&self, module_id: &str) -> bool {
646 self.lock()
647 .remove(module_id)
648 .is_some_and(|state| state.cooldown_until.is_some())
649 }
650
651 fn record_timeout(
652 &self,
653 module_id: &str,
654 threshold: u32,
655 cooldown: Duration,
656 ) -> Option<BreakerOpened> {
657 let mut modules = self.lock();
658 let state = modules.entry(module_id.to_string()).or_default();
659 let was_open = state.cooldown_until.is_some();
660 let was_probe = state.probe_in_flight;
661 state.probe_in_flight = false;
662 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
663 if state.consecutive_timeouts < threshold {
664 return None;
665 }
666 state.cooldown_until = Some(Instant::now() + cooldown);
667 Some(BreakerOpened {
668 consecutive_timeouts: state.consecutive_timeouts,
669 reopened_after_probe: was_open && was_probe,
670 })
671 }
672
673 fn record_inconclusive(&self, module_id: &str) {
674 if let Some(state) = self.lock().get_mut(module_id) {
675 state.probe_in_flight = false;
676 }
677 }
678
679 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
694 self.lock()
695 .remove(module_id)
696 .map(|state| state.consecutive_timeouts)
697 .filter(|discarded| *discarded > 0)
698 }
699
700 fn open_snapshot(&self) -> Option<serde_json::Value> {
707 let now = Instant::now();
708 let modules = self.lock();
709 let open = modules
710 .iter()
711 .filter_map(|(module_id, state)| {
712 let cooldown_until = state.cooldown_until?;
713 Some((
714 module_id.clone(),
715 serde_json::json!({
716 "consecutive_timeouts": state.consecutive_timeouts,
717 "cooldown_remaining_ms":
718 cooldown_until.saturating_duration_since(now).as_millis() as u64,
719 "probe_in_flight": state.probe_in_flight,
720 }),
721 ))
722 })
723 .collect::<serde_json::Map<String, serde_json::Value>>();
724 (!open.is_empty()).then_some(serde_json::Value::Object(open))
725 }
726}
727
728impl ControlHandler {
729 pub fn new(registry: Arc<Registry>) -> Self {
730 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
731 }
732
733 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
734 let counters = forwarding.counters();
735 let route_bind_breakers = forwarding.route_bind_breakers();
739 let route_bind_concurrency = forwarding.route_bind_concurrency();
740 let route_outages = forwarding.route_outages();
741 Self {
742 registry,
743 forwarding,
744 process_liveness: None,
745 supervisor: SupervisorHandle::new(),
746 subc_capabilities: Arc::from([
747 CAP_MANIFEST_REGISTRATION.to_string(),
748 CAP_CHANNEL_LIFECYCLE.to_string(),
749 CAP_PING_PONG.to_string(),
750 CAP_SESSION_ATTACH.to_string(),
751 CAP_ADMISSION_FACTS_RELAY.to_string(),
752 CAP_SCOPES_V1.to_string(),
753 ]),
754 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
755 route_bind_relay_timeouts: BTreeMap::new(),
756 route_bind_breakers,
757 route_bind_concurrency,
758 route_outages,
759 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
760 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
761 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
762 storage_config: None,
763 machine_id: None,
764 admission_facts_carrier_module_id: None,
765 admission_facts_targets: None,
766 scopes: Arc::new(RwLock::new(ScopeTable::new(
767 crate::daemon_config::default_scope_authority_owners(),
768 ))),
769 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
770 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
771 rescan: None,
772 connected_clients: ConnectedClients::new(),
773 counters,
774 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
775 daemon_provenance: DaemonProvenanceFacts::default(),
776 #[cfg(test)]
777 control_dispatch_delay: None,
778 #[cfg(test)]
779 provenance_probe_override: None,
780 }
781 }
782
783 pub fn with_storage_config(
786 mut self,
787 storage_config: Option<crate::daemon_config::StorageConfig>,
788 ) -> Self {
789 self.storage_config = storage_config;
790 self
791 }
792
793 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
796 self.machine_id = machine_id;
797 self
798 }
799
800 pub fn with_admission_facts_config(
804 mut self,
805 carrier_module_id: Option<String>,
806 targets: Option<Vec<String>>,
807 ) -> Self {
808 self.admission_facts_carrier_module_id = carrier_module_id;
809 self.admission_facts_targets = targets;
810 self
811 }
812
813 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
817 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
818 self.scope_authority_owners = owners;
819 self
820 }
821
822 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
825 self.route_bind_relay_timeout = timeout;
826 self
827 }
828
829 pub fn with_route_bind_relay_timeouts(
835 mut self,
836 timeouts: impl IntoIterator<Item = (String, Duration)>,
837 ) -> Self {
838 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
839 self
840 }
841
842 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
848 self.route_bind_relay_timeouts
849 .get(module_id)
850 .copied()
851 .unwrap_or(self.route_bind_relay_timeout)
852 }
853
854 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
862 self.route_bind_breaker_threshold = threshold.max(1);
863 self.route_bind_breaker_cooldown = cooldown;
864 self
865 }
866
867 #[cfg(test)]
868 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
869 self.health_probe_timeout = timeout;
870 self
871 }
872
873 #[cfg(test)]
874 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
875 self.control_dispatch_delay = Some(delay);
876 self
877 }
878
879 pub fn with_process_liveness(
880 mut self,
881 process_liveness: Arc<dyn ModuleProcessLiveness>,
882 ) -> Self {
883 self.process_liveness = Some(process_liveness);
884 self
885 }
886
887 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
888 self.supervisor = supervisor;
889 self
890 }
891
892 pub fn with_daemon_provenance(
893 mut self,
894 pid: u32,
895 started_at_ms: u64,
896 executable_path: Option<PathBuf>,
897 build_git_sha: Option<String>,
898 build_lock_digest: Option<String>,
899 ) -> Self {
900 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
901 let process_start_time = process_start_time(pid);
902 self.daemon_provenance = DaemonProvenanceFacts {
903 build: DaemonBuildProvenance {
904 build_git_sha,
905 build_lock_digest,
906 },
907 pid: Some(pid),
908 started_at_ms: Some(started_at_ms),
909 start_clock: None,
910 executable_path,
911 executable_identity,
912 process_start_time,
913 probe: ExecutableIdentityProbe::default(),
914 };
915 self
916 }
917
918 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
919 self.daemon_provenance.start_clock = Some(clock);
920 self
921 }
922
923 #[cfg(test)]
924 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
925 self.provenance_probe_override = Some(result);
926 self
927 }
928
929 pub fn with_capability_config(
933 self,
934 modules: impl IntoIterator<Item = (String, bool)>,
935 reserved_capabilities: BTreeMap<String, String>,
936 ) -> Self {
937 self.capability_evaluator
938 .configure(modules, reserved_capabilities);
939 self
940 }
941
942 pub fn with_supervisor_rescan(
943 mut self,
944 supervisor: Supervisor,
945 config_path: impl Into<PathBuf>,
946 configured_port: Option<u16>,
947 ) -> Self {
948 self.rescan = Some(SupervisorRescanContext {
949 supervisor,
950 config_path: config_path.into(),
951 configured_port,
952 storage_config: self.storage_config.clone(),
953 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
954 admission_facts_targets: self.admission_facts_targets.clone(),
955 scope_authority_owners: self.scope_authority_owners.clone(),
956 });
957 self
958 }
959
960 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
961 self.connected_clients = connected_clients;
962 self
963 }
964
965 pub fn forwarding(&self) -> Arc<ForwardingTable> {
966 Arc::clone(&self.forwarding)
967 }
968
969 pub(crate) fn counters(&self) -> DaemonCounters {
970 self.counters.clone()
971 }
972
973 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
976 tokio::spawn(async move {
977 loop {
978 self.capability_evaluator
979 .wait_for_change_or_deadline()
980 .await;
981 self.refresh_capability_requirements();
982 }
983 });
984 }
985
986 fn runtime_capability_snapshot(
987 &self,
988 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
989 let runtime = self
990 .supervisor
991 .list()
992 .into_iter()
993 .map(|module| {
994 let status = module.status().map_err(|err| {
995 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
996 })?;
997 Ok(RuntimeModule {
998 module_id: status.module_id,
999 state: status.state,
1000 enabled: status.enabled,
1001 })
1002 })
1003 .collect::<Result<Vec<_>, RouterError>>()?;
1004 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1005 RouterError::backend(
1006 0,
1007 0,
1008 format!("failed to list capability registrations: {err}"),
1009 )
1010 })?;
1011 let registrations = registrations
1012 .into_iter()
1013 .map(|registration| RegisteredModule {
1014 module_id: registration.manifest.module_id,
1015 module_version: registration.manifest.module_version,
1016 capabilities: registration.manifest.capabilities,
1017 })
1018 .collect();
1019 Ok((runtime, registrations))
1020 }
1021
1022 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1029 let cached_registration = RegisteredModule {
1030 module_id: registration.manifest.module_id.clone(),
1031 module_version: registration.manifest.module_version.clone(),
1032 capabilities: registration.manifest.capabilities.clone(),
1033 };
1034 if self.capability_evaluator.record_hello(&cached_registration) {
1035 warn!(
1036 module_id = %cached_registration.module_id,
1037 "capability claims drifted from the cached manifest"
1038 );
1039 }
1040 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1041 self.enforce_capability_denies();
1042 }
1043 self.refresh_capability_requirements();
1044 }
1045
1046 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1050 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1051 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1052 self.supervisor.set_swap_promotion_observer(observer);
1053 }
1054
1055 pub fn refresh_capability_requirements(&self) {
1056 match self.runtime_capability_snapshot() {
1057 Ok((runtime, registrations)) => {
1058 log_requirement_events(
1059 self.capability_evaluator
1060 .evaluate_now(&runtime, ®istrations),
1061 );
1062 }
1063 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1064 }
1065 }
1066
1067 fn enforce_capability_denies(&self) {
1071 let (_, registrations) = match self.registry.list_modules() {
1072 Ok(snapshot) => snapshot,
1073 Err(err) => {
1074 warn!(error = %err, "failed to read registrations for capability deny census");
1075 return;
1076 }
1077 };
1078 let manifests = registrations
1079 .into_iter()
1080 .map(|registration| {
1081 (
1082 registration.manifest.module_id.clone(),
1083 registration.manifest,
1084 )
1085 })
1086 .collect::<BTreeMap<_, _>>();
1087 let census = match self.forwarding.route_census(None) {
1088 Ok(census) => census,
1089 Err(err) => {
1090 warn!(error = %err, "failed to read route census for capability deny enforcement");
1091 return;
1092 }
1093 };
1094
1095 for (target_module_id, routes) in census {
1096 let Some(target_manifest) = manifests.get(&target_module_id) else {
1097 continue;
1098 };
1099 let mut closed_routes = Vec::new();
1100 let mut module_goodbyes = Vec::new();
1101 for route in routes {
1102 let Principal::Reserved {
1103 module_id: opening_module_id,
1104 } = &route.principal
1105 else {
1106 continue;
1107 };
1108 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1109 continue;
1110 };
1111 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1112 continue;
1113 };
1114
1115 match self.forwarding.release_client_route(
1116 route.goodbye_target.connection_id,
1117 route.goodbye_target.channel,
1118 route.goodbye_target.epoch,
1119 ) {
1120 Ok(RouteRelease::Removed(module_goodbye)) => {
1121 warn!(
1122 opening_module_id,
1123 target_module_id,
1124 capability,
1125 "force-closing route because an attested capability deny edge now matches"
1126 );
1127 closed_routes.push(route);
1128 module_goodbyes.push(module_goodbye);
1129 }
1130 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1131 Err(err) => warn!(
1132 opening_module_id,
1133 target_module_id,
1134 capability,
1135 error = %err,
1136 "failed to force-close capability-denied route"
1137 ),
1138 }
1139 }
1140
1141 if closed_routes.is_empty() {
1142 continue;
1143 }
1144 send_route_control_pushes(
1145 &self.forwarding,
1146 closed_routes,
1147 ClientControlPush::RouteClosed {
1148 module_id: target_module_id,
1149 reason: RouteCloseReason::CapabilityDenied,
1150 drained: false,
1151 abandoned: 0,
1152 excluded_subscriptions: 0,
1153 terminal: Some(false),
1154 },
1155 );
1156 self.emit_route_goodbyes(module_goodbyes);
1157 }
1158 }
1159
1160 fn not_ready_reason(
1166 &self,
1167 registration: &crate::registry::ModuleRegistration,
1168 ) -> Option<NotReadyReason> {
1169 if !registration.ready {
1170 return Some(NotReadyReason {
1171 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1172 capability: None,
1173 });
1174 }
1175 self.first_unprovided_required_capability(registration)
1176 .map(|capability| NotReadyReason {
1177 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1178 capability: Some(capability),
1179 })
1180 }
1181
1182 fn first_unprovided_required_capability(
1200 &self,
1201 registration: &crate::registry::ModuleRegistration,
1202 ) -> Option<String> {
1203 let required = registration
1204 .manifest
1205 .capabilities
1206 .iter()
1207 .flat_map(|declarations| declarations.requires.iter())
1208 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1209 .map(|requirement| requirement.capability.as_str())
1210 .collect::<BTreeSet<_>>();
1211 if required.is_empty() {
1212 return None;
1213 }
1214 let module_id = registration.manifest.module_id.as_str();
1215 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1216 if required
1217 .iter()
1218 .any(|capability| verdict(capability).is_none())
1219 {
1220 self.refresh_capability_requirements();
1221 }
1222 required
1223 .into_iter()
1224 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1225 .map(str::to_string)
1226 }
1227
1228 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1229 self.capability_evaluator
1230 .statuses()
1231 .into_iter()
1232 .map(capability_requirement_status)
1233 .collect()
1234 }
1235
1236 fn deregister_connection(
1243 &self,
1244 connection_id: ConnectionId,
1245 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1246 self.registry.deregister_connection(connection_id)
1247 }
1248
1249 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1250 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1251 return None;
1252 }
1253 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1254 parse_client_control_request(&frame.body)
1255 else {
1256 return None;
1257 };
1258 Some(target_module_id(&target).to_string())
1259 }
1260
1261 pub(crate) fn route_open_capacity_refusal(
1262 &self,
1263 ctx: &RouteCtx,
1264 frame: &Frame,
1265 target_module_id: &str,
1266 in_flight: usize,
1267 limit: usize,
1268 ) -> Result<Frame, RouterError> {
1269 self.route_open_admission_refusal_frame(
1270 ctx,
1271 frame,
1272 target_module_id,
1273 "open_admission_full",
1274 (in_flight, limit),
1275 format!(
1276 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1277 ),
1278 )
1279 }
1280
1281 fn route_open_target_capacity_refusal(
1282 &self,
1283 ctx: &RouteCtx,
1284 frame: &Frame,
1285 target_module_id: &str,
1286 in_flight: usize,
1287 ) -> Result<Frame, RouterError> {
1288 self.route_open_admission_refusal_frame(
1289 ctx,
1290 frame,
1291 target_module_id,
1292 "target_binds_full",
1293 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1294 format!(
1295 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1296 ),
1297 )
1298 }
1299
1300 fn route_open_admission_refusal_frame(
1308 &self,
1309 ctx: &RouteCtx,
1310 frame: &Frame,
1311 target_module_id: &str,
1312 reason: &'static str,
1313 (in_flight, limit): (usize, usize),
1314 message: impl Into<String>,
1315 ) -> Result<Frame, RouterError> {
1316 let code = error_codes::TARGET_UNAVAILABLE;
1317 self.counters.increment_route_open_refused(code);
1318 info!(
1319 target: "control",
1320 code,
1321 reason,
1322 module_id = ?target_module_id,
1323 connection_id = ctx.connection_id.get(),
1324 in_flight,
1325 limit,
1326 "route.open refused"
1327 );
1328 control_error_frame(frame, code, message.into())
1329 }
1330
1331 #[cfg(test)]
1337 pub fn handle_control(
1338 &self,
1339 connection_id: ConnectionId,
1340 frame: Frame,
1341 ) -> Result<Vec<Frame>, RouterError> {
1342 match frame.header.ty {
1343 FrameType::Ping => Ok(vec![pong(&frame)?]),
1344 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1345 FrameType::Goodbye => self.handle_goodbye(connection_id),
1346 ty => Ok(vec![control_error_frame(
1347 &frame,
1348 "unsupported_control_frame",
1349 format!("unsupported channel-0 frame {ty:?}"),
1350 )?]),
1351 }
1352 }
1353
1354 pub async fn handle_control_frame(
1355 &self,
1356 ctx: &RouteCtx,
1357 frame: Frame,
1358 ) -> Result<Vec<Frame>, RouterError> {
1359 self.handle_control_frame_timed(ctx, frame, None).await
1360 }
1361
1362 pub(crate) async fn handle_control_frame_timed(
1363 &self,
1364 ctx: &RouteCtx,
1365 frame: Frame,
1366 dispatch_started_at: Option<StdInstant>,
1367 ) -> Result<Vec<Frame>, RouterError> {
1368 match frame.header.ty {
1369 FrameType::Ping => Ok(vec![pong(&frame)?]),
1370 FrameType::Hello => {
1371 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1372 }
1373 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1374 FrameType::Cancel => {
1375 if self
1376 .supervisor
1377 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1378 {
1379 Ok(Vec::new())
1380 } else {
1381 Ok(vec![control_error_frame(
1382 &frame,
1383 "unknown_subscription",
1384 "no supervisor spawn subscription has this correlation id",
1385 )?])
1386 }
1387 }
1388 FrameType::Request => {
1389 if self
1390 .forwarding
1391 .module_endpoint_for_connection(ctx.connection_id)
1392 .map_err(RouterError::Forwarding)?
1393 .is_some()
1394 {
1395 if !is_known_module_request_op(&frame.body) {
1396 return Ok(vec![control_error_frame(
1397 &frame,
1398 "unsupported_control_frame",
1399 "module-originated channel-0 REQUEST is not supported",
1400 )?]);
1401 }
1402 let request = match parse_module_control_request_from_module(&frame.body) {
1403 Ok(request) => request,
1404 Err((err, ControlRequestBodyError::UnknownOp)) => {
1405 return Ok(vec![control_error_frame(
1406 &frame,
1407 "unsupported_control_frame",
1408 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1409 )?])
1410 }
1411 Err((err, ControlRequestBodyError::InvalidBody)) => {
1412 return Ok(vec![control_error_frame(
1413 &frame,
1414 "invalid_control_body",
1415 format!("malformed module control body: {err}"),
1416 )?])
1417 }
1418 };
1419 let op = module_control_request_op(&request);
1420 let corr = frame.header.corr;
1421 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1422 let result =
1423 self.handle_module_control_request(ctx.connection_id, frame, request);
1424 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1425 return result;
1426 }
1427
1428 if is_known_module_request_op(&frame.body) {
1429 return Ok(vec![control_error_frame(
1430 &frame,
1431 "not_registered",
1432 "catalog.update requires an active module registration owned by this connection",
1433 )?]);
1434 }
1435
1436 let request = match parse_client_control_request(&frame.body) {
1437 Ok(request) => request,
1438 Err((err, ControlRequestBodyError::UnknownOp)) => {
1439 return Ok(vec![control_error_frame(
1440 &frame,
1441 "unknown_control_op",
1442 format!("unknown client control op: {err}"),
1443 )?])
1444 }
1445 Err((err, ControlRequestBodyError::InvalidBody)) => {
1446 return Ok(vec![control_error_frame(
1447 &frame,
1448 "invalid_control_body",
1449 format!("malformed client control body: {err}"),
1450 )?])
1451 }
1452 };
1453 let op = client_control_request_op(&request);
1454 let corr = frame.header.corr;
1455 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1456 #[cfg(test)]
1457 if let Some(delay) = self.control_dispatch_delay {
1458 tokio::time::sleep(delay).await;
1459 }
1460 let result = self
1461 .handle_client_control_request(ctx, frame, request)
1462 .await;
1463 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1464 result
1465 }
1466 FrameType::Push => {
1467 let Some(endpoint) = self
1468 .forwarding
1469 .module_endpoint_for_connection(ctx.connection_id)
1470 .map_err(RouterError::Forwarding)?
1471 else {
1472 return Ok(vec![control_error_frame(
1473 &frame,
1474 "unsupported_control_frame",
1475 "client-originated channel-0 PUSH is not supported",
1476 )?]);
1477 };
1478 self.handle_status_update(endpoint, frame)
1479 }
1480 FrameType::Response | FrameType::Error
1481 if self
1482 .forwarding
1483 .module_endpoint_for_connection(ctx.connection_id)
1484 .map_err(RouterError::Forwarding)?
1485 .is_some() =>
1486 {
1487 self.handle_module_relay_response(ctx.connection_id, frame)
1488 }
1489 ty => Ok(vec![control_error_frame(
1490 &frame,
1491 "unsupported_control_frame",
1492 format!("unsupported channel-0 frame {ty:?}"),
1493 )?]),
1494 }
1495 }
1496
1497 pub fn cleanup_connection(
1498 &self,
1499 connection_id: ConnectionId,
1500 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1501 let crash_closed = self
1502 .registry
1503 .get_module_by_connection(connection_id)?
1504 .and_then(|registration| {
1505 self.forwarding
1506 .module_endpoint_for_connection(connection_id)
1507 .ok()
1508 .flatten()
1509 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1510 .map(|routes| (registration.manifest.module_id, routes))
1511 });
1512 let crash_closed = crash_closed.map(|(module_id, routes)| {
1513 let terminal = match self.supervisor.get(&module_id) {
1514 None => false,
1515 Some(module) => match module.will_recover_after_connection_loss() {
1516 Ok(will_recover) => !will_recover,
1517 Err(err) => {
1518 warn!(
1519 %module_id,
1520 error = %err,
1521 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1522 );
1523 false
1524 }
1525 },
1526 };
1527 let reason = match self.forwarding.is_daemon_draining() {
1531 Ok(true) => RouteCloseReason::Restart,
1532 Ok(false) => RouteCloseReason::Crash,
1533 Err(err) => {
1534 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1535 RouteCloseReason::Crash
1536 }
1537 };
1538 (module_id, routes, reason, terminal)
1539 });
1540 let registrations = self.deregister_connection(connection_id);
1541 let cleanup = self.forwarding.cleanup_connection_counted(connection_id);
1542 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1547 let abandoned = cleanup
1548 .as_ref()
1549 .map_or(0, |cleanup| cleanup.abandoned_relays);
1550 send_route_control_pushes(
1551 &self.forwarding,
1552 routes,
1553 ClientControlPush::RouteClosed {
1554 module_id,
1555 reason,
1556 drained: false,
1557 abandoned,
1558 excluded_subscriptions: 0,
1559 terminal: Some(terminal),
1560 },
1561 );
1562 }
1563 if let Ok(cleanup) = cleanup {
1564 self.emit_route_goodbyes(cleanup.released);
1565 }
1566 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1570 crate::supervise::notify_registration_release();
1571 self.capability_evaluator.wake_deadline_loop();
1572 self.refresh_capability_requirements();
1573 }
1574 self.supervisor.remove_spawn_subscribers(connection_id);
1575 self.hello_launch_nonces
1578 .lock()
1579 .unwrap_or_else(|poisoned| poisoned.into_inner())
1580 .forget(connection_id);
1581 self.scopes
1582 .write()
1583 .unwrap_or_else(|poisoned| poisoned.into_inner())
1584 .release_connection(connection_id);
1585 registrations
1586 }
1587
1588 pub(crate) fn handle_route_goodbye(
1589 &self,
1590 connection_id: ConnectionId,
1591 route_channel: u16,
1592 route_epoch: u32,
1593 ) -> Result<bool, RouterError> {
1594 debug!(
1595 connection_id = connection_id.get(),
1596 route_channel, route_epoch, "handling route GOODBYE"
1597 );
1598 let RouteRelease::Removed(released_route) = self
1599 .forwarding
1600 .release_client_route(connection_id, route_channel, route_epoch)
1601 .map_err(RouterError::Forwarding)?
1602 else {
1603 return Ok(false);
1604 };
1605 self.emit_route_goodbyes(vec![released_route]);
1606 Ok(true)
1607 }
1608
1609 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1610 for released in released_routes {
1611 let frame = match Frame::build_with_version(
1612 released.negotiated_ver,
1613 FrameType::Goodbye,
1614 control_flags(),
1615 released.channel,
1616 released.epoch,
1617 0,
1618 Vec::new(),
1619 ) {
1620 Ok(frame) => frame,
1621 Err(err) => {
1622 warn!(
1623 route_channel = released.channel,
1624 error = %err,
1625 "failed to build route GOODBYE frame"
1626 );
1627 continue;
1628 }
1629 };
1630 if !released.close_on_delivery_failure() {
1631 crate::forwarding::send_module_route_goodbye(
1632 &self.counters,
1633 &released.sink,
1634 frame,
1635 released.module_id.as_deref(),
1636 "client route released",
1637 );
1638 continue;
1639 }
1640 if let Err(err) = released.sink.try_send(frame) {
1641 warn!(
1642 target_connection_id = released.connection_id.get(),
1643 route_channel = released.channel,
1644 error = %err,
1645 "route GOODBYE was not delivered to client; closing target connection"
1646 );
1647 if self
1648 .forwarding
1649 .escalate_client_delivery_failure(
1650 released.connection_id,
1651 released.channel,
1652 released.epoch,
1653 CloseReason::new(
1654 "route_goodbye_delivery_failed",
1655 format!(
1656 "failed to enqueue route GOODBYE for channel {}: {err}",
1657 released.channel
1658 ),
1659 ),
1660 crate::forwarding::UndeliveredFrame {
1661 module_id: released.module_id.as_deref(),
1662 sink: &released.sink,
1663 },
1664 )
1665 .unwrap_or(false)
1666 {
1667 self.counters.increment_goodbye_relay_client_failed();
1668 }
1669 }
1670 }
1671 }
1672
1673 fn send_abandoned_route_bind_goodbye(
1685 &self,
1686 module_sink: &crate::FrameSink,
1687 negotiated_ver: u8,
1688 module_channel: u16,
1689 module_epoch: u32,
1690 ) {
1691 let frame = match Frame::build_with_version(
1692 negotiated_ver,
1693 FrameType::Goodbye,
1694 control_flags(),
1695 module_channel,
1696 module_epoch,
1697 0,
1698 Vec::new(),
1699 ) {
1700 Ok(frame) => frame,
1701 Err(err) => {
1702 warn!(
1703 route_channel = module_channel,
1704 error = %err,
1705 "failed to build GOODBYE for abandoned route.bind"
1706 );
1707 return;
1708 }
1709 };
1710 crate::forwarding::send_module_route_goodbye(
1711 &self.counters,
1712 module_sink,
1713 frame,
1714 None,
1715 "abandoned route.bind",
1716 );
1717 }
1718
1719 fn handle_hello(
1720 &self,
1721 connection_id: ConnectionId,
1722 sink: Option<crate::FrameSink>,
1723 frame: Frame,
1724 ) -> Result<Vec<Frame>, RouterError> {
1725 debug!(
1726 connection_id = connection_id.get(),
1727 corr = frame.header.corr,
1728 "handling HELLO"
1729 );
1730 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1731 Ok(value) => value,
1732 Err(err) => {
1733 return Ok(vec![control_error_frame(
1734 &frame,
1735 "invalid_hello",
1736 format!("malformed HELLO body: {err}"),
1737 )?])
1738 }
1739 };
1740 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1741 return Ok(vec![control_error_frame(
1742 &frame,
1743 "invalid_capability_grammar",
1744 err.to_string(),
1745 )?]);
1746 }
1747 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1748 return Ok(vec![control_error_frame(
1749 &frame,
1750 "invalid_manifest",
1751 err.to_string(),
1752 )?]);
1753 }
1754 if let Some(provenance) = hello_value
1755 .get("manifest")
1756 .and_then(|manifest| manifest.get("provenance"))
1757 {
1758 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1759 return Ok(vec![control_error_frame(
1760 &frame,
1761 "invalid_manifest",
1762 format!("malformed manifest provenance: {err}"),
1763 )?]);
1764 }
1765 }
1766 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1767 Ok(hello) => hello,
1768 Err(err) => {
1769 return Ok(vec![control_error_frame(
1770 &frame,
1771 "invalid_hello",
1772 format!("malformed HELLO body: {err}"),
1773 )?])
1774 }
1775 };
1776
1777 if hello.protocol_ver != hello.manifest.protocol_ver {
1778 return Ok(vec![control_error_frame(
1779 &frame,
1780 "invalid_manifest",
1781 format!(
1782 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1783 hello.protocol_ver, hello.manifest.protocol_ver
1784 ),
1785 )?]);
1786 }
1787
1788 if hello.manifest.module_id.trim().is_empty() {
1789 return Ok(vec![control_error_frame(
1790 &frame,
1791 "invalid_manifest",
1792 "manifest module_id must not be empty",
1793 )?]);
1794 }
1795
1796 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1797 Ok(negotiated_ver) => negotiated_ver,
1798 Err(message) => {
1799 return Ok(vec![control_error_frame(
1800 &frame,
1801 "version_unsupported",
1802 message,
1803 )?])
1804 }
1805 };
1806
1807 let swap_admission = self
1815 .supervisor
1816 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1817 if swap_admission == SwapHelloAdmission::Refused {
1818 warn!(
1819 module_id = %hello.manifest.module_id,
1820 connection_id = connection_id.get(),
1821 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1822 );
1823 return Ok(vec![control_error_frame(
1824 &frame,
1825 "swap_token_invalid",
1826 format!(
1827 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1828 hello.manifest.module_id
1829 ),
1830 )?]);
1831 }
1832 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1833
1834 if let Some(rejection) = (!swap_candidate)
1842 .then(|| {
1843 self.supervisor.reserved_hello_rejection(
1844 &hello.manifest.module_id,
1845 hello.launch_nonce.as_deref(),
1846 )
1847 })
1848 .flatten()
1849 {
1850 let message = match rejection {
1851 ReservedHelloRejection::Exact { module_id } => format!(
1852 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1853 ),
1854 ReservedHelloRejection::Prefix {
1855 prefix,
1856 owner_module_id,
1857 } => format!(
1858 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1859 hello.manifest.module_id
1860 ),
1861 };
1862 return Ok(vec![control_error_frame(
1863 &frame,
1864 "reserved_module",
1865 message,
1866 )?]);
1867 }
1868
1869 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1870 &hello.manifest.module_id,
1871 hello.manifest.capabilities.as_ref(),
1872 );
1873 if let Some(refusal) = reserved_capability_refusals.first() {
1874 let capability = refusal.capability.clone();
1875 let bound_module = refusal.claimants[0].clone();
1876 log_duplicate_claim_events(reserved_capability_refusals);
1877 return Ok(vec![control_error_frame(
1878 &frame,
1879 "reserved_capability",
1880 format!(
1881 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1882 capability, bound_module, hello.manifest.module_id
1883 ),
1884 )?]);
1885 }
1886
1887 if self
1890 .forwarding
1891 .connection_has_client_routes(connection_id)
1892 .map_err(RouterError::Forwarding)?
1893 {
1894 return Ok(vec![control_error_frame(
1895 &frame,
1896 "invalid_hello",
1897 "connection has open client routes and cannot also register as a module",
1898 )?]);
1899 }
1900
1901 self.hello_launch_nonces
1906 .lock()
1907 .unwrap_or_else(|poisoned| poisoned.into_inner())
1908 .record(connection_id, hello.launch_nonce.as_deref());
1909 let control_ops = effective_module_control_ops(hello.control_ops);
1910 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1913 if swap_candidate {
1914 return self.register_swap_candidate(
1915 connection_id,
1916 sink,
1917 &frame,
1918 hello.manifest,
1919 negotiated_ver,
1920 control_ops,
1921 hello_ack,
1922 );
1923 }
1924 let registration = match self.registry.register_with_control_ops(
1925 hello.manifest,
1926 negotiated_ver,
1927 connection_id,
1928 control_ops,
1929 ) {
1930 Ok(registration) => registration,
1931 Err(RegistryError::DuplicateModuleId { module_id }) => {
1932 return Ok(vec![control_error_frame(
1933 &frame,
1934 "duplicate_module_id",
1935 format!(
1936 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1937 ),
1938 )?])
1939 }
1940 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1941 return Ok(vec![control_error_frame(
1942 &frame,
1943 "invalid_module_id",
1944 err.to_string(),
1945 )?])
1946 }
1947 Err(err) => {
1948 return Ok(vec![control_error_frame(
1949 &frame,
1950 "registry_error",
1951 err.to_string(),
1952 )?])
1953 }
1954 };
1955
1956 let reply = if let Some(sink) = sink {
1957 let concurrency = manifest_concurrency(®istration.manifest);
1973 if let Err(err) = self.forwarding.register_module_connection_acked(
1974 connection_id,
1975 registration.manifest.module_id.clone(),
1976 negotiated_ver,
1977 concurrency,
1978 sink,
1979 hello_ack,
1980 ) {
1981 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1985 crate::supervise::notify_registration_release();
1986 }
1987 return Ok(vec![control_error_frame(
1988 &frame,
1989 forwarding_error_code(&err),
1990 err.to_string(),
1991 )?]);
1992 }
1993 Vec::new()
1994 } else {
1995 vec![hello_ack]
1998 };
1999
2000 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2009 info!(
2010 module_id = %registration.manifest.module_id,
2011 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2012 );
2013 }
2014
2015 self.apply_registration_capabilities(®istration);
2016
2017 info!(
2018 module_id = %registration.manifest.module_id,
2019 module_version = %registration.manifest.module_version,
2020 negotiated_ver,
2021 routable_provider = manifest_provides_routable_role(®istration.manifest),
2022 connection_id = connection_id.get(),
2023 "module registered"
2024 );
2025
2026 Ok(reply)
2027 }
2028
2029 #[allow(clippy::too_many_arguments)]
2039 fn register_swap_candidate(
2040 &self,
2041 connection_id: ConnectionId,
2042 sink: Option<crate::FrameSink>,
2043 frame: &Frame,
2044 manifest: ModuleManifest,
2045 negotiated_ver: u8,
2046 control_ops: Vec<String>,
2047 hello_ack: Frame,
2048 ) -> Result<Vec<Frame>, RouterError> {
2049 let module_id = manifest.module_id.clone();
2050 let registration = match self.registry.register_candidate_with_control_ops(
2051 manifest,
2052 negotiated_ver,
2053 connection_id,
2054 control_ops,
2055 ) {
2056 Ok(registration) => registration,
2057 Err(RegistryError::DuplicateModuleId { module_id }) => {
2058 return Ok(vec![control_error_frame(
2059 frame,
2060 "duplicate_module_id",
2061 format!(
2062 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2063 ),
2064 )?])
2065 }
2066 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2067 return Ok(vec![control_error_frame(
2068 frame,
2069 "invalid_module_id",
2070 err.to_string(),
2071 )?])
2072 }
2073 Err(err) => {
2074 return Ok(vec![control_error_frame(
2075 frame,
2076 "registry_error",
2077 err.to_string(),
2078 )?])
2079 }
2080 };
2081 let reply = if let Some(sink) = sink {
2082 let concurrency = manifest_concurrency(®istration.manifest);
2086 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2087 connection_id,
2088 module_id.clone(),
2089 negotiated_ver,
2090 concurrency,
2091 sink,
2092 hello_ack,
2093 ) {
2094 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2095 crate::supervise::notify_registration_release();
2096 }
2097 return Ok(vec![control_error_frame(
2098 frame,
2099 forwarding_error_code(&err),
2100 err.to_string(),
2101 )?]);
2102 }
2103 Vec::new()
2104 } else {
2105 vec![hello_ack]
2106 };
2107 self.supervisor.mark_swap_candidate_admitted(&module_id);
2108 info!(
2109 module_id = %module_id,
2110 module_version = %registration.manifest.module_version,
2111 negotiated_ver,
2112 ready = registration.ready,
2113 connection_id = connection_id.get(),
2114 "swap candidate registered; not routable until cutover"
2115 );
2116 Ok(reply)
2117 }
2118
2119 fn build_hello_ack(
2120 &self,
2121 frame: &Frame,
2122 negotiated_ver: u8,
2123 module_id: &str,
2124 ) -> Result<Frame, RouterError> {
2125 let ack = ModuleHelloAckBody {
2126 negotiated_ver,
2127 subc_ops: module_subc_ops(),
2128 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2129 storage: self
2130 .storage_config
2131 .as_ref()
2132 .map(|cfg| cfg.descriptor_for(module_id)),
2133 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2134 };
2135 let body = serde_json::to_vec(&ack).map_err(|err| {
2136 RouterError::backend(
2137 0,
2138 frame.header.corr,
2139 format!("failed to encode HELLO_ACK: {err}"),
2140 )
2141 })?;
2142
2143 Frame::build_with_version(
2144 negotiated_ver,
2145 FrameType::HelloAck,
2146 control_flags(),
2147 0,
2148 0,
2149 frame.header.corr,
2150 body,
2151 )
2152 .map_err(RouterError::FrameBuild)
2153 }
2154
2155 async fn handle_client_control_request(
2156 &self,
2157 ctx: &RouteCtx,
2158 frame: Frame,
2159 request: ClientControlRequest,
2160 ) -> Result<Vec<Frame>, RouterError> {
2161 match request {
2162 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2163 ClientControlRequest::CatalogList { module_id } => {
2164 self.handle_catalog_list(frame, module_id)
2165 }
2166 ClientControlRequest::RouteOpen {
2167 target,
2168 identity,
2169 consumer_identity,
2170 consumer_capabilities,
2171 admission_facts,
2172 scope,
2173 } => {
2174 self.handle_route_open(
2175 ctx,
2176 frame,
2177 RouteOpenRequest {
2178 target,
2179 identity,
2180 consumer_identity,
2181 consumer_capabilities,
2182 admission_facts,
2183 scope,
2184 },
2185 )
2186 .await
2187 }
2188 ClientControlRequest::RoutePoll {
2189 route_channel,
2190 route_epoch,
2191 kind,
2192 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2193 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2194 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2195 self.handle_supervisor_spawn_snapshot(frame)
2196 }
2197 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2198 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2199 }
2200 ClientControlRequest::SupervisorRestart {
2201 module_id,
2202 drain_timeout_ms,
2203 } => {
2204 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2205 .await
2206 }
2207 ClientControlRequest::SupervisorSwap {
2208 module_id,
2209 ready_timeout_ms,
2210 } => {
2211 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2212 .await
2213 }
2214 ClientControlRequest::SupervisorReload { module_id } => {
2215 self.handle_supervisor_reload(frame, module_id).await
2216 }
2217 ClientControlRequest::SupervisorRescan { preview } => {
2218 self.handle_supervisor_rescan(frame, preview).await
2219 }
2220 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2221 self.handle_supervisor_release_reserved(frame, module_id)
2222 .await
2223 }
2224 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2225 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2226 .await
2227 }
2228 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2229 self.handle_supervisor_health_probe(frame, module_id).await
2230 }
2231 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2232 ClientControlRequest::SupervisorRoutes { module_id } => {
2233 self.handle_supervisor_routes(frame, module_id)
2234 }
2235 ClientControlRequest::SupervisorProvenance { module_id } => {
2236 self.handle_supervisor_provenance(frame, module_id).await
2237 }
2238 ClientControlRequest::SupervisorStderrTail {
2239 module_id,
2240 max_lines,
2241 max_bytes,
2242 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2243 ClientControlRequest::SupervisorTerminals { module_id } => {
2244 self.handle_supervisor_terminals(frame, module_id).await
2245 }
2246 }
2247 }
2248
2249 fn handle_module_control_request(
2250 &self,
2251 connection_id: ConnectionId,
2252 frame: Frame,
2253 request: ModuleControlRequestFromModule,
2254 ) -> Result<Vec<Frame>, RouterError> {
2255 match request {
2256 ModuleControlRequestFromModule::CatalogUpdate {
2257 provides,
2258 capabilities,
2259 ready,
2260 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2261 ModuleControlRequestFromModule::LiveRoots {} => {
2262 let registered = self
2263 .registry
2264 .get_module_by_connection(connection_id)
2265 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2266 let Some(registration) = registered else {
2267 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2268 };
2269 let response = self
2270 .forwarding
2271 .live_roots(®istration.manifest.module_id)
2272 .map_err(RouterError::Forwarding)?;
2273 Ok(vec![control_response_body_frame(
2274 &frame,
2275 &response,
2276 "ModuleControlResponseToModule::LiveRoots",
2277 )?])
2278 }
2279 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2280 self.handle_scope_sync(connection_id, frame, generation, scopes)
2281 }
2282 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2283 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2284 }
2285 }
2286 }
2287
2288 fn handle_scope_sync(
2292 &self,
2293 connection_id: ConnectionId,
2294 frame: Frame,
2295 generation: u64,
2296 scopes: Vec<ScopeRecord>,
2297 ) -> Result<Vec<Frame>, RouterError> {
2298 let Some(registration) = self
2299 .registry
2300 .get_module_by_connection(connection_id)
2301 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2302 else {
2303 return Ok(vec![control_error_frame(
2304 &frame,
2305 "not_registered",
2306 "scope.sync requires an active module registration owned by this connection",
2307 )?]);
2308 };
2309 let owner = registration.manifest.module_id;
2310 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2311 let is_current_launch = |connection: ConnectionId| {
2312 self.hello_launch_nonces
2313 .lock()
2314 .unwrap_or_else(|poisoned| poisoned.into_inner())
2315 .presented(connection, current_nonce.as_deref())
2316 };
2317 let mut table = self
2322 .scopes
2323 .write()
2324 .unwrap_or_else(|poisoned| poisoned.into_inner());
2325 let outcome = table.sync(&owner, connection_id, is_current_launch, generation, scopes);
2326 let drained = match &outcome {
2327 Ok(applied) => self
2328 .forwarding
2329 .publish_scope_changes(&applied.tag_changes)
2330 .map_err(RouterError::Forwarding)?,
2331 Err(_) => Vec::new(),
2332 };
2333 drop(table);
2334 match outcome {
2335 Ok(applied) => {
2336 info!(
2337 owner = %owner,
2338 generation,
2339 records = applied.results.len(),
2340 ended = applied.ended.len(),
2341 tag_changes = applied.tag_changes.len(),
2342 routes_closed = drained.len(),
2343 "scope sync accepted"
2344 );
2345 self.close_scope_drained_routes(drained);
2346 let response = ModuleControlResponseToModule::ScopeSync {
2347 generation,
2348 results: applied.results,
2349 ended: applied.ended,
2350 };
2351 Ok(vec![control_response_body_frame(
2352 &frame,
2353 &response,
2354 "ModuleControlResponseToModule::ScopeSync",
2355 )?])
2356 }
2357 Err(refusal) => {
2358 info!(
2359 owner = %owner,
2360 generation,
2361 code = refusal.code,
2362 "scope sync refused"
2363 );
2364 Ok(vec![control_error_frame(
2365 &frame,
2366 refusal.code,
2367 refusal.message,
2368 )?])
2369 }
2370 }
2371 }
2372
2373 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2379 if drained.is_empty() {
2380 return;
2381 }
2382 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2383 BTreeMap::new();
2384 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2385 for route in drained {
2386 warn!(
2387 module_id = %route.module_id,
2388 reason = ?route.reason,
2389 client_connection_id = route.client.connection_id.get(),
2390 route_channel = route.client.channel,
2391 "closing route because its scope changed"
2392 );
2393 pushes
2394 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2395 .or_insert_with(|| (route.reason, Vec::new()))
2396 .1
2397 .push(EndpointRoute {
2398 goodbye_target: route.client.clone(),
2399 principal: Principal::Unverified,
2400 bound_at: Instant::now(),
2401 draining: false,
2402 drain_reason: None,
2403 });
2404 goodbyes.push(route.module);
2405 goodbyes.push(route.client);
2406 }
2407 for ((module_id, _), (reason, routes)) in pushes {
2408 send_route_control_pushes(
2409 &self.forwarding,
2410 routes,
2411 ClientControlPush::RouteClosed {
2412 module_id,
2413 reason,
2414 drained: false,
2415 abandoned: 0,
2416 excluded_subscriptions: 0,
2417 terminal: Some(false),
2418 },
2419 );
2420 }
2421 self.emit_route_goodbyes(goodbyes);
2422 }
2423
2424 fn handle_scope_describe(
2427 &self,
2428 connection_id: ConnectionId,
2429 frame: Frame,
2430 owner: Principal,
2431 scope_ref: String,
2432 ) -> Result<Vec<Frame>, RouterError> {
2433 let registered = self
2434 .registry
2435 .get_module_by_connection(connection_id)
2436 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2437 if registered.is_none() {
2438 return Ok(vec![control_error_frame(
2439 &frame,
2440 "not_registered",
2441 "scope.describe requires an active module registration owned by this connection",
2442 )?]);
2443 }
2444 let description = self
2445 .scopes
2446 .read()
2447 .unwrap_or_else(|poisoned| poisoned.into_inner())
2448 .describe(&owner, &scope_ref);
2449 let owner_configured = match &owner {
2450 Principal::Reserved { module_id } => self.supervisor.get(module_id).is_some(),
2451 _ => false,
2452 };
2453 let response = ModuleControlResponseToModule::ScopeDescribe {
2454 status: description.status,
2455 scope_epoch: description.scope_epoch,
2456 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2457 owner_synced: description.owner_synced,
2458 owner_configured,
2459 scope: description.stamp,
2460 };
2461 Ok(vec![control_response_body_frame(
2462 &frame,
2463 &response,
2464 "ModuleControlResponseToModule::ScopeDescribe",
2465 )?])
2466 }
2467
2468 fn handle_catalog_update(
2469 &self,
2470 connection_id: ConnectionId,
2471 frame: Frame,
2472 provides: Vec<ProviderRole>,
2473 capabilities: Option<CapabilityDeclarations>,
2474 ready: Option<bool>,
2475 ) -> Result<Vec<Frame>, RouterError> {
2476 self.refresh_capability_requirements();
2477 let Some(registration) = self
2478 .registry
2479 .get_module_by_connection(connection_id)
2480 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2481 else {
2482 return Ok(vec![control_error_frame(
2483 &frame,
2484 "not_registered",
2485 "catalog.update requires an active module registration owned by this connection",
2486 )?]);
2487 };
2488
2489 if let Some(message) =
2490 catalog_update_frozen_field_message(®istration.manifest, &provides)
2491 {
2492 return Ok(vec![control_error_frame(
2493 &frame,
2494 "catalog_update_frozen_field",
2495 message,
2496 )?]);
2497 }
2498
2499 let mut candidate = registration.manifest.clone();
2500 candidate.provides = provides.clone();
2501 candidate.capabilities = capabilities
2502 .clone()
2503 .or_else(|| registration.manifest.capabilities.clone());
2504 if let Err(err) = candidate.validate_capability_grammar() {
2505 return Ok(vec![control_error_frame(
2506 &frame,
2507 "invalid_capability_grammar",
2508 err.to_string(),
2509 )?]);
2510 }
2511
2512 let updated = self
2513 .registry
2514 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2515 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2516 if updated.is_none() {
2517 return Ok(vec![control_error_frame(
2518 &frame,
2519 "not_registered",
2520 "catalog.update requires an active module registration owned by this connection",
2521 )?]);
2522 }
2523 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2524 log_duplicate_claim_events(
2525 self.capability_evaluator
2526 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2527 );
2528 }
2529 if capability_census_trigger(
2530 registration.manifest.capabilities.as_ref(),
2531 updated
2532 .as_ref()
2533 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2534 ) {
2535 self.enforce_capability_denies();
2536 }
2537 self.refresh_capability_requirements();
2538
2539 let response = ModuleControlResponseToModule::CatalogUpdate {};
2540 control_response_body_frame(
2541 &frame,
2542 &response,
2543 "ModuleControlResponseToModule::CatalogUpdate",
2544 )
2545 .map(|frame| vec![frame])
2546 }
2547
2548 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2549 self.refresh_capability_requirements();
2550 let mut counters = self.counters.snapshot();
2556 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2557 self.forwarding.client_route_concentration(),
2558 counters.as_object_mut(),
2559 ) {
2560 obj.insert(
2561 "client_connections_with_routes".into(),
2562 connections_with_routes.into(),
2563 );
2564 obj.insert("max_routes_on_one_connection".into(), max.into());
2565 }
2566 if let (Some(open_breakers), Some(obj)) = (
2572 self.route_bind_breakers.open_snapshot(),
2573 counters.as_object_mut(),
2574 ) {
2575 obj.insert("route_bind_breakers_open".into(), open_breakers);
2576 }
2577 let response = ClientControlResponse::ServerDescribe {
2578 protocol_ver: PROTOCOL_VERSION,
2579 subc_ops: subc_ops(),
2580 capabilities: self.subc_capabilities.as_ref().to_vec(),
2581 connected_clients: self.connected_clients.count(),
2582 counters: Some(counters),
2583 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2584 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2585 capability_requirements: self.capability_requirement_statuses(),
2586 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2587 };
2588 Ok(vec![control_response_body_frame(
2589 &frame,
2590 &response,
2591 "ClientControlResponse::ServerDescribe",
2592 )?])
2593 }
2594
2595 fn handle_catalog_list(
2596 &self,
2597 frame: Frame,
2598 module_id: Option<String>,
2599 ) -> Result<Vec<Frame>, RouterError> {
2600 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2601 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2602 })?;
2603 let entries = modules
2604 .into_iter()
2605 .filter(|registration| {
2606 module_id
2607 .as_deref()
2608 .map(|wanted| registration.manifest.module_id == wanted)
2609 .unwrap_or(true)
2610 })
2611 .map(|registration| {
2612 let not_ready = self.not_ready_reason(®istration);
2613 let roles = registration.manifest.provides;
2614 CatalogEntry {
2615 module_id: registration.manifest.module_id,
2616 ready: not_ready.is_none(),
2617 not_ready,
2618 module_version: Some(registration.manifest.module_version),
2619 roles,
2620 control_ops: registration.control_ops,
2621 capabilities: registration.manifest.capabilities,
2622 self_signals: registration.manifest.self_signals,
2623 }
2624 })
2625 .collect();
2626 let response = ClientControlResponse::CatalogList {
2627 generation,
2628 modules: entries,
2629 subc_ops: subc_ops(),
2630 };
2631 Ok(vec![control_response_body_frame(
2632 &frame,
2633 &response,
2634 "ClientControlResponse::CatalogList",
2635 )?])
2636 }
2637
2638 fn route_open_principal(
2639 &self,
2640 frame: &Frame,
2641 consumer_identity: Option<ConsumerIdentity>,
2642 ) -> Result<Result<Principal, Frame>, RouterError> {
2643 let Some(consumer_identity) = consumer_identity else {
2644 return Ok(Ok(Principal::Direct));
2645 };
2646
2647 if self.supervisor.spawned_consumer_authorized(
2648 &consumer_identity.module_id,
2649 &consumer_identity.launch_nonce,
2650 ) {
2651 return Ok(Ok(Principal::Reserved {
2652 module_id: consumer_identity.module_id,
2653 }));
2654 }
2655
2656 Ok(Err(control_error_frame(
2657 frame,
2658 "bad_consumer_identity",
2659 format!(
2660 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2661 consumer_identity.module_id
2662 ),
2663 )?))
2664 }
2665
2666 fn route_open_refusal_frame(
2673 &self,
2674 ctx: &RouteCtx,
2675 frame: &Frame,
2676 module_id: &str,
2677 reason: &'static str,
2678 code: &'static str,
2679 message: impl Into<String>,
2680 ) -> Result<Frame, RouterError> {
2681 self.observe_route_open_refusal(ctx, module_id, reason, code);
2682 control_error_frame(frame, code, message.into())
2683 }
2684
2685 fn route_open_breaker_refusal_frame(
2703 &self,
2704 ctx: &RouteCtx,
2705 frame: &Frame,
2706 module_id: &str,
2707 consecutive_timeouts: u32,
2708 retry_in: Duration,
2709 probe_in_flight: bool,
2710 ) -> Result<Frame, RouterError> {
2711 self.counters
2712 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2713 debug!(
2714 target: "control",
2715 code = "module_timeout",
2716 module_id = ?module_id,
2717 connection_id = ctx.connection_id.get(),
2718 consecutive_timeouts,
2719 retry_in_ms = retry_in.as_millis() as u64,
2720 probe_in_flight,
2721 "route.open refused by open bind-relay breaker"
2722 );
2723 let detail = if probe_in_flight {
2724 "one probe bind is already in flight; retry once it settles".to_string()
2725 } else {
2726 format!("not relaying for another {retry_in:?}")
2727 };
2728 control_error_frame(
2729 frame,
2730 "module_timeout",
2731 format!(
2732 "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2733 relays; {detail}"
2734 ),
2735 )
2736 }
2737
2738 fn observe_route_open_refusal(
2751 &self,
2752 ctx: &RouteCtx,
2753 module_id: &str,
2754 reason: &'static str,
2755 code: &'static str,
2756 ) {
2757 self.counters.increment_route_open_refused(code);
2758 info!(
2759 target: "control",
2760 code,
2761 reason,
2762 module_id = ?module_id,
2763 connection_id = ctx.connection_id.get(),
2764 "route.open refused"
2765 );
2766 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
2767 self.route_outages.record_not_serving(module_id, reason);
2768 }
2769 }
2770
2771 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2816 self.route_outages.record_accepted(module_id);
2817 self.counters.increment_route_open_accepted(principal);
2818 info!(
2819 target: "control",
2820 principal,
2821 module_id,
2822 connection_id = ctx.connection_id.get(),
2823 "route.open accepted"
2824 );
2825 }
2826
2827 fn supervised_absent_route_open_refusal_frame(
2828 &self,
2829 ctx: &RouteCtx,
2830 frame: &Frame,
2831 module_id: &str,
2832 code: &'static str,
2833 status: &crate::supervise::ModuleStatus,
2834 ) -> Result<Frame, RouterError> {
2835 self.counters.increment_route_open_refused(code);
2836 info!(
2837 target: "control",
2838 code,
2839 reason = "supervised_not_registered",
2840 module_id = ?module_id,
2841 connection_id = ctx.connection_id.get(),
2842 state = %status.state,
2843 enabled = status.enabled,
2844 live = status.live,
2845 "route.open refused"
2846 );
2847 self.route_outages
2851 .record_not_serving(module_id, "supervised_not_registered");
2852 control_error_frame(
2853 frame,
2854 code,
2855 format!(
2856 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2857 status.state, status.enabled, status.live
2858 ),
2859 )
2860 }
2861
2862 async fn handle_route_open(
2863 &self,
2864 ctx: &RouteCtx,
2865 frame: Frame,
2866 request: RouteOpenRequest,
2867 ) -> Result<Vec<Frame>, RouterError> {
2868 let RouteOpenRequest {
2869 target,
2870 mut identity,
2871 consumer_identity,
2872 consumer_capabilities,
2873 admission_facts,
2874 scope,
2875 } = request;
2876 let target_module_id = target_module_id(&target).to_string();
2877 debug!(
2878 connection_id = ctx.connection_id.get(),
2879 corr = frame.header.corr,
2880 module_id = %target_module_id,
2881 "handling route.open"
2882 );
2883
2884 let Some(registration) = self
2903 .registry
2904 .get_module(&target_module_id)
2905 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2906 else {
2907 if let Some((status, warming)) =
2908 self.supervisor_status(&target_module_id, frame.header.corr)?
2909 {
2910 if status.protocol == ModuleProtocol::None {
2917 return Ok(vec![self.route_open_refusal_frame(
2918 ctx,
2919 &frame,
2920 &target_module_id,
2921 "protocol_none",
2922 error_codes::MODULE_NO_PROTOCOL,
2923 format!(
2924 "module_id '{target_module_id}' is declared protocol: none; \
2925 it speaks no subc wire and serves no routes"
2926 ),
2927 )?]);
2928 }
2929 let code = if warming {
2930 "module_warming"
2931 } else {
2932 "target_unavailable"
2933 };
2934 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2935 ctx,
2936 &frame,
2937 &target_module_id,
2938 code,
2939 &status,
2940 )?]);
2941 }
2942 if let Some(removed_ago_ms) =
2943 self.supervisor.removal_tombstone_age_ms(&target_module_id)
2944 {
2945 return Ok(vec![self.route_open_refusal_frame(
2946 ctx,
2947 &frame,
2948 &target_module_id,
2949 "removed",
2950 error_codes::MODULE_REMOVED,
2951 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
2952 )?]);
2953 }
2954 return Ok(vec![self.route_open_refusal_frame(
2955 ctx,
2956 &frame,
2957 &target_module_id,
2958 "not_registered",
2959 error_codes::UNKNOWN_MODULE,
2960 format!("module_id '{target_module_id}' is not registered"),
2961 )?]);
2962 };
2963
2964 if !registration.ready {
2968 self.counters
2969 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
2970 info!(
2971 target: "control",
2972 code = error_codes::MODULE_WARMING,
2973 module_id = ?target_module_id,
2974 connection_id = ctx.connection_id.get(),
2975 reason = "declared_not_ready",
2976 "route.open refused"
2977 );
2978 self.route_outages
2981 .record_not_serving(&target_module_id, "declared_not_ready");
2982 return Ok(vec![control_error_body_frame(
2983 &frame,
2984 ErrorBody {
2985 code: error_codes::MODULE_WARMING.to_string(),
2986 message: format!(
2987 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
2988 ),
2989 detail: Some(serde_json::json!({
2990 "reason": "declared_not_ready"
2991 })),
2992 },
2993 )?]);
2994 }
2995
2996 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3015 self.counters
3016 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3017 info!(
3018 target: "control",
3019 code = error_codes::MODULE_WARMING,
3020 module_id = ?target_module_id,
3021 connection_id = ctx.connection_id.get(),
3022 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3023 capability = %capability,
3024 "route.open refused"
3025 );
3026 return Ok(vec![control_error_body_frame(
3027 &frame,
3028 ErrorBody {
3029 code: error_codes::MODULE_WARMING.to_string(),
3030 message: format!(
3031 "module_id '{target_module_id}' requires capability '{capability}', \
3032 which no registered module provides; retry"
3033 ),
3034 detail: Some(serde_json::json!({
3035 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3036 "capability": capability,
3037 })),
3038 },
3039 )?]);
3040 }
3041
3042 if !target_has_required_role(&target, ®istration.manifest.provides) {
3043 return Ok(vec![self.route_open_refusal_frame(
3044 ctx,
3045 &frame,
3046 &target_module_id,
3047 "role_not_provided",
3048 "target_unavailable",
3049 format!("module_id '{target_module_id}' does not provide the requested target"),
3050 )?]);
3051 }
3052
3053 if registration.state != ChannelState::Active {
3054 return Ok(vec![self.route_open_refusal_frame(
3055 ctx,
3056 &frame,
3057 &target_module_id,
3058 "registration_not_active",
3059 "target_unavailable",
3060 format!("module_id '{target_module_id}' is not active"),
3061 )?]);
3062 }
3063
3064 if self
3065 .forwarding
3066 .module_is_draining(&target_module_id)
3067 .map_err(RouterError::Forwarding)?
3068 {
3069 return Ok(vec![self.route_open_refusal_frame(
3070 ctx,
3071 &frame,
3072 &target_module_id,
3073 "reloading",
3074 "module_reloading",
3075 format!("module_id '{target_module_id}' is reloading"),
3076 )?]);
3077 }
3078
3079 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3080 process_liveness.process_live(&target_module_id) == Some(false)
3081 }) {
3082 if process_liveness.process_replacing(&target_module_id) {
3089 return Ok(vec![self.route_open_refusal_frame(
3090 ctx,
3091 &frame,
3092 &target_module_id,
3093 "reloading",
3094 "module_reloading",
3095 format!("module_id '{target_module_id}' is reloading"),
3096 )?]);
3097 }
3098 return Ok(vec![self.route_open_refusal_frame(
3099 ctx,
3100 &frame,
3101 &target_module_id,
3102 "supervisor_not_live",
3103 "target_unavailable",
3104 format!("module_id '{target_module_id}' is not live"),
3105 )?]);
3106 }
3107
3108 if !self
3109 .forwarding
3110 .has_live_module_connection(&target_module_id)
3111 .map_err(RouterError::Forwarding)?
3112 {
3113 return Ok(vec![self.route_open_refusal_frame(
3114 ctx,
3115 &frame,
3116 &target_module_id,
3117 "no_forwarding_connection",
3118 "target_unavailable",
3119 format!("module_id '{target_module_id}' has no live forwarding connection"),
3120 )?]);
3121 }
3122
3123 if let Some(error) =
3124 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3125 {
3126 self.observe_route_open_refusal(
3127 ctx,
3128 &target_module_id,
3129 "op_not_allowed",
3130 "op_not_allowed",
3131 );
3132 return Ok(vec![error]);
3133 }
3134
3135 let principal = match self.route_open_principal(&frame, consumer_identity)? {
3136 Ok(principal) => principal,
3137 Err(error) => {
3138 self.observe_route_open_refusal(
3139 ctx,
3140 &target_module_id,
3141 "bad_consumer_identity",
3142 "bad_consumer_identity",
3143 );
3144 return Ok(vec![error]);
3145 }
3146 };
3147
3148 if let Principal::Reserved {
3152 module_id: opening_module_id,
3153 } = &principal
3154 {
3155 if let Some(opening_registration) = self
3156 .registry
3157 .get_module(opening_module_id)
3158 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3159 {
3160 if let Some(capability) =
3161 denied_capability(&opening_registration.manifest, ®istration.manifest)
3162 {
3163 warn!(
3164 opening_module_id,
3165 target_module_id,
3166 capability,
3167 "refusing route.open because an attested capability deny edge matches"
3168 );
3169 return Ok(vec![self.route_open_refusal_frame(
3170 ctx,
3171 &frame,
3172 &target_module_id,
3173 "capability_deny_edge",
3174 "capability_forbidden",
3175 format!(
3176 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3177 ),
3178 )?]);
3179 }
3180 }
3181 }
3182
3183 if admission_facts.is_some() {
3184 let carrier_matches = matches!(
3185 &principal,
3186 Principal::Reserved { module_id }
3187 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3188 );
3189 if !carrier_matches {
3190 return Ok(vec![self.route_open_refusal_frame(
3191 ctx,
3192 &frame,
3193 &target_module_id,
3194 "admission_facts_carrier_not_permitted",
3195 "admission_facts_not_permitted",
3196 "admission facts may only be carried by the configured reserved module",
3197 )?]);
3198 }
3199
3200 let target_allowed = self
3201 .admission_facts_targets
3202 .as_ref()
3203 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3204 if !target_allowed {
3205 return Ok(vec![self.route_open_refusal_frame(
3206 ctx,
3207 &frame,
3208 &target_module_id,
3209 "admission_facts_target_not_listed",
3210 "admission_facts_target_not_allowed",
3211 format!(
3212 "admission facts are not permitted for target module_id '{target_module_id}'"
3213 ),
3214 )?]);
3215 }
3216
3217 }
3221
3222 let (bound_scope, scope_stamp) = match scope {
3228 None => (None, None),
3229 Some(selector) => {
3230 let owner_configured = match &selector.owner {
3231 Principal::Reserved { module_id } => self.supervisor.get(module_id).is_some(),
3232 _ => false,
3233 };
3234 let admitted = self
3235 .scopes
3236 .read()
3237 .unwrap_or_else(|poisoned| poisoned.into_inner())
3238 .admit(&principal, &target_module_id, &selector, owner_configured);
3239 match admitted {
3240 Ok(admission) => (
3241 Some(BoundScope {
3242 owner: admission.owner,
3243 scope_ref: admission.stamp.scope_ref.clone(),
3244 tag: admission.tag,
3245 }),
3246 Some(admission.stamp),
3247 ),
3248 Err(refusal) => {
3249 return Ok(vec![self.route_open_refusal_frame(
3250 ctx,
3251 &frame,
3252 &target_module_id,
3253 refusal.code,
3254 refusal.code,
3255 refusal.message,
3256 )?]);
3257 }
3258 }
3259 }
3260 };
3261
3262 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3284 Ok(project_root) => project_root,
3285 Err(err) => {
3286 return Ok(vec![control_error_frame(
3287 &frame,
3288 "invalid_project_root",
3289 err.to_string(),
3290 )?])
3291 }
3292 };
3293 identity.project_root = project_root.as_path().to_path_buf();
3294
3295 let _concurrency_guard = match self
3306 .route_bind_concurrency
3307 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3308 {
3309 Ok(guard) => guard,
3310 Err(in_flight) => {
3311 return Ok(vec![self.route_open_target_capacity_refusal(
3312 ctx,
3313 &frame,
3314 &target_module_id,
3315 in_flight,
3316 )?]);
3317 }
3318 };
3319
3320 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3323 RouteBindAdmission::Admitted { guard, probe } => {
3324 if probe {
3325 info!(
3326 module_id = %target_module_id,
3327 connection_id = ctx.connection_id.get(),
3328 "route.bind breaker half-open: admitting one probe"
3329 );
3330 }
3331 guard
3332 }
3333 RouteBindAdmission::Refused {
3334 consecutive_timeouts,
3335 retry_in,
3336 probe_in_flight,
3337 } => {
3338 return Ok(vec![self.route_open_breaker_refusal_frame(
3339 ctx,
3340 &frame,
3341 &target_module_id,
3342 consecutive_timeouts,
3343 retry_in,
3344 probe_in_flight,
3345 )?]);
3346 }
3347 };
3348
3349 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3354 let relay_deadline = Instant::now() + route_bind_relay_timeout;
3355 let pending = match self
3356 .forwarding
3357 .begin_route_bind_relay_for(
3358 ctx.connection_id,
3359 ctx.egress.clone(),
3360 response_version(&frame),
3361 frame.header.corr,
3362 &target_module_id,
3363 principal.clone(),
3364 bound_scope,
3365 Some(project_root),
3366 relay_deadline,
3367 )
3368 .await
3369 {
3370 Ok(pending) => pending,
3371 Err(err) => {
3372 return Ok(vec![self.route_open_refusal_frame(
3373 ctx,
3374 &frame,
3375 &target_module_id,
3376 "relay_reservation_failed",
3377 forwarding_error_code(&err),
3378 err.to_string(),
3379 )?])
3380 }
3381 };
3382 let crate::forwarding::PendingRouteBindRelay {
3383 endpoint,
3384 module_sink,
3385 negotiated_ver,
3386 client_channel,
3387 client_epoch,
3388 module_channel,
3389 module_epoch,
3390 corr: relay_corr,
3391 receiver,
3392 } = pending;
3393 let mut reservation =
3394 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3395
3396 debug!(
3397 connection_id = ctx.connection_id.get(),
3398 client_channel,
3399 client_epoch,
3400 module_channel,
3401 module_epoch,
3402 "reserved route handle pair"
3403 );
3404 let principal_label = match &principal {
3407 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3408 Principal::Direct => "direct".to_string(),
3409 other => format!("{other:?}"),
3410 };
3411 let relay = ModuleControlRequest::RouteBind {
3412 route_channel: module_channel,
3413 epoch: module_epoch,
3414 target,
3415 identity,
3416 principal: Some(principal),
3417 consumer_capabilities,
3418 admission_facts,
3419 scope: scope_stamp,
3420 };
3421 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3422 RouterError::backend(
3423 0,
3424 frame.header.corr,
3425 format!("failed to encode route.bind request: {err}"),
3426 )
3427 })?;
3428 let relay_frame = Frame::build_with_version(
3429 negotiated_ver,
3430 FrameType::Request,
3431 control_flags(),
3432 0,
3433 0,
3434 relay_corr,
3435 relay_body,
3436 )
3437 .map_err(RouterError::FrameBuild)?;
3438
3439 if let Err(err) = module_sink.send(relay_frame).await {
3440 reservation.release_and_disarm();
3441 return Ok(vec![self.route_open_refusal_frame(
3442 ctx,
3443 &frame,
3444 &target_module_id,
3445 "relay_send_failed",
3446 "target_unavailable",
3447 err.to_string(),
3448 )?]);
3449 }
3450
3451 if !self
3452 .forwarding
3453 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3454 .map_err(RouterError::Forwarding)?
3455 {
3456 self.send_abandoned_route_bind_goodbye(
3457 &module_sink,
3458 negotiated_ver,
3459 module_channel,
3460 module_epoch,
3461 );
3462 }
3463
3464 match timeout_at(relay_deadline, receiver).await {
3465 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3466 reservation.disarm();
3467 if breaker.record_accepted() {
3468 info!(
3469 module_id = %target_module_id,
3470 "route.bind breaker closed: the probe was accepted"
3471 );
3472 }
3473 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3474 Ok(Vec::new())
3475 }
3476 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3477 reservation.release_and_disarm();
3478 breaker.record_inconclusive();
3482 let scope_code = match body.code.as_str() {
3486 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
3487 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
3488 _ => None,
3489 };
3490 if let Some(code) = scope_code {
3491 self.observe_route_open_refusal(
3492 ctx,
3493 &target_module_id,
3494 "scope_changed_before_commit",
3495 code,
3496 );
3497 return Ok(vec![control_error_body_frame(&frame, body)?]);
3498 }
3499 self.counters
3500 .increment_route_open_refused("module_rejected");
3501 info!(
3502 target: "control",
3503 code = "module_rejected",
3504 module_code = ?body.code,
3505 module_id = ?target_module_id,
3506 connection_id = ctx.connection_id.get(),
3507 "route.open refused"
3508 );
3509 Ok(vec![control_error_body_frame(&frame, body)?])
3510 }
3511 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3512 reservation.release_and_disarm();
3513 breaker.record_inconclusive();
3514 tracing::warn!(
3521 module_id = %target_module_id,
3522 "route.bind relay abandoned: {message}"
3523 );
3524 Ok(vec![self.route_open_refusal_frame(
3525 ctx,
3526 &frame,
3527 &target_module_id,
3528 "relay_abandoned",
3529 "target_unavailable",
3530 message,
3531 )?])
3532 }
3533 Ok(Err(_)) => {
3534 reservation.release_and_disarm();
3535 breaker.record_inconclusive();
3536 Ok(vec![self.route_open_refusal_frame(
3537 ctx,
3538 &frame,
3539 &target_module_id,
3540 "relay_waiter_canceled",
3541 "target_unavailable",
3542 "route.bind relay waiter was canceled before the module responded",
3543 )?])
3544 }
3545 Err(_) => {
3546 reservation.release_and_disarm();
3547 if let Some(opened) = breaker.record_timeout(
3551 self.route_bind_breaker_threshold,
3552 self.route_bind_breaker_cooldown,
3553 ) {
3554 warn!(
3555 module_id = %target_module_id,
3556 consecutive_timeouts = opened.consecutive_timeouts,
3557 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3558 reopened_after_probe = opened.reopened_after_probe,
3559 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3560 );
3561 }
3562 tracing::warn!(
3570 module_id = %target_module_id,
3571 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3572 "route.bind relay timed out: module did not ack within budget"
3573 );
3574 Ok(vec![self.route_open_refusal_frame(
3575 ctx,
3576 &frame,
3577 &target_module_id,
3578 "relay_timed_out",
3579 "module_timeout",
3580 format!(
3581 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3582 route_bind_relay_timeout
3583 ),
3584 )?])
3585 }
3586 }
3587 }
3588
3589 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3590 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3591 snapshot: self.supervisor.spawn_snapshot(),
3592 };
3593 Ok(vec![control_response_body_frame(
3594 &frame,
3595 &response,
3596 "ClientControlResponse::SupervisorSpawnSnapshot",
3597 )?])
3598 }
3599
3600 fn handle_supervisor_spawn_subscribe(
3601 &self,
3602 ctx: &RouteCtx,
3603 frame: Frame,
3604 since: Option<SpawnCursor>,
3605 ) -> Result<Vec<Frame>, RouterError> {
3606 match self.supervisor.subscribe_spawns(
3607 ctx.connection_id,
3608 frame.header.corr,
3609 response_version(&frame),
3610 since,
3611 ctx.egress.clone(),
3612 ) {
3613 Ok(()) => Ok(Vec::new()),
3614 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3615 Ok(vec![control_error_body_frame(
3616 &frame,
3617 ErrorBody {
3618 code: "spawn_cursor_incarnation_mismatch".to_string(),
3619 message: "spawn cursor belongs to a different daemon incarnation"
3620 .to_string(),
3621 detail: Some(serde_json::json!({
3622 "current_daemon_incarnation": current
3623 })),
3624 },
3625 )?])
3626 }
3627 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3628 &frame,
3629 ErrorBody {
3630 code: "spawn_cursor_too_old".to_string(),
3631 message: "spawn cursor predates the retained event ring".to_string(),
3632 detail: Some(serde_json::json!({
3633 "oldest_retained_cursor": oldest
3634 })),
3635 },
3636 )?]),
3637 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3638 0,
3639 frame.header.corr,
3640 format!("failed to open supervisor spawn subscription: {error}"),
3641 )),
3642 }
3643 }
3644
3645 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3646 let generation = self
3647 .registry
3648 .generation()
3649 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3650 let mut modules = Vec::new();
3651 for module in self.supervisor.list() {
3652 let status = module.status_for_control("list").map_err(|err| {
3653 RouterError::backend(
3654 0,
3655 frame.header.corr,
3656 format!("failed to read supervisor status: {err}"),
3657 )
3658 })?;
3659 let (configured, _) = module.configuration().map_err(|err| {
3660 RouterError::backend(
3661 0,
3662 frame.header.corr,
3663 format!("failed to read module configuration: {err}"),
3664 )
3665 })?;
3666 let image = module.running_image_agreement().await;
3668 let resources = Some(module.child_resource_usage());
3671 let pending_reload = Some(reload_verdict(
3672 &configured.program,
3673 status.spawned_from.as_deref(),
3674 image,
3675 ));
3676 modules.push(SupervisorEntry {
3677 launch_nonce_env: Some(
3678 configured.protocol != subc_control::ModuleProtocol::None
3679 && (!cfg!(unix) || configured.launch_nonce_env),
3680 ),
3681 module_id: status.module_id,
3682 state: status.state.to_string(),
3683 enabled: status.enabled,
3684 live: status.live,
3685 protocol: status.protocol,
3686 health: status.health.status,
3687 pending_reload,
3688 last_probe_ms: status.health.last_probe_ms,
3689 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3690 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3691 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3692 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3693 restart_count: Some(status.restart_count),
3694 max_restarts: Some(status.max_restarts),
3695 lifetime_restarts: Some(status.lifetime_restarts),
3696 spawn_generation: Some(status.spawn_generation),
3697 restart_window_secs: Some(status.restart_window.as_secs()),
3698 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3699 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3700 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3701 resources,
3702 });
3703 }
3704 let response = ClientControlResponse::SupervisorList {
3705 generation,
3706 modules,
3707 };
3708 Ok(vec![control_response_body_frame(
3709 &frame,
3710 &response,
3711 "ClientControlResponse::SupervisorList",
3712 )?])
3713 }
3714
3715 fn handle_supervisor_stderr_tail(
3716 &self,
3717 frame: Frame,
3718 module_id: String,
3719 max_lines: Option<u32>,
3720 max_bytes: Option<u32>,
3721 ) -> Result<Vec<Frame>, RouterError> {
3722 let Some(module) = self.supervisor.get(&module_id) else {
3723 return Ok(vec![control_error_frame(
3724 &frame,
3725 "unknown_module",
3726 format!("module_id '{module_id}' is not supervised"),
3727 )?]);
3728 };
3729
3730 let snapshot = module.stderr_tail(
3731 max_lines.map(|value| value as usize),
3732 max_bytes.map(|value| value as usize),
3733 );
3734
3735 let response = ClientControlResponse::SupervisorStderrTail {
3736 module_id,
3737 tail: StderrTail {
3738 capture: match snapshot.capture {
3739 CaptureState::Captured => StderrCaptureState::Captured,
3740 CaptureState::Incomplete { reason } => {
3741 StderrCaptureState::Incomplete { reason }
3742 }
3743 CaptureState::NotCaptured { reason } => {
3744 StderrCaptureState::NotCaptured { reason }
3745 }
3746 },
3747 entries: snapshot
3748 .entries
3749 .into_iter()
3750 .map(|entry| match entry {
3751 TailEntry::Line {
3752 text,
3753 truncated,
3754 at_ms,
3755 } => StderrTailEntry::Line {
3756 text,
3757 truncated,
3758 at_ms,
3759 },
3760 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3761 })
3762 .collect(),
3763 dropped_lines: snapshot.dropped_lines,
3764 },
3765 };
3766 Ok(vec![control_response_body_frame(
3767 &frame,
3768 &response,
3769 "ClientControlResponse::SupervisorStderrTail",
3770 )?])
3771 }
3772
3773 async fn handle_supervisor_terminals(
3774 &self,
3775 frame: Frame,
3776 module_id: String,
3777 ) -> Result<Vec<Frame>, RouterError> {
3778 let Some(module) = self.supervisor.get(&module_id) else {
3779 return Ok(vec![control_error_frame(
3780 &frame,
3781 "unknown_module",
3782 format!("module_id '{module_id}' is not supervised"),
3783 )?]);
3784 };
3785
3786 let terminals = module
3789 .read_durable_terminal_history()
3790 .await
3791 .map_err(|error| {
3792 RouterError::backend(
3793 0,
3794 frame.header.corr,
3795 format!("failed to read terminal history: {error}"),
3796 )
3797 })?;
3798 let response = ClientControlResponse::SupervisorTerminals {
3799 module_id,
3800 terminals,
3801 };
3802 Ok(vec![control_response_body_frame(
3803 &frame,
3804 &response,
3805 "ClientControlResponse::SupervisorTerminals",
3806 )?])
3807 }
3808
3809 fn handle_supervisor_routes(
3810 &self,
3811 frame: Frame,
3812 module_id: Option<String>,
3813 ) -> Result<Vec<Frame>, RouterError> {
3814 let modules = self
3815 .forwarding
3816 .route_census(module_id.as_deref())
3817 .map_err(RouterError::Forwarding)?
3818 .into_iter()
3819 .map(|(module_id, routes)| SupervisorRouteModule {
3820 module_id,
3821 routes: routes
3822 .into_iter()
3823 .map(|route| SupervisorRoute {
3824 consumer: match route.principal {
3825 Principal::Reserved { module_id } => {
3826 SupervisorRouteConsumer::Reserved { module_id }
3827 }
3828 Principal::Direct | Principal::Unverified => {
3829 SupervisorRouteConsumer::Direct {
3830 connection_id: route.goodbye_target.connection_id.get(),
3831 }
3832 }
3833 },
3834 age_ms: Instant::now()
3835 .saturating_duration_since(route.bound_at)
3836 .as_millis()
3837 .try_into()
3838 .unwrap_or(u64::MAX),
3839 draining: route.draining,
3840 drain_reason: route.drain_reason,
3841 })
3842 .collect(),
3843 })
3844 .collect();
3845 let response = ClientControlResponse::SupervisorRoutes { modules };
3846 Ok(vec![control_response_body_frame(
3847 &frame,
3848 &response,
3849 "ClientControlResponse::SupervisorRoutes",
3850 )?])
3851 }
3852
3853 async fn handle_supervisor_provenance(
3854 &self,
3855 frame: Frame,
3856 module_id: Option<String>,
3857 ) -> Result<Vec<Frame>, RouterError> {
3858 let mut selected = if let Some(module_id) = module_id {
3859 let Some(module) = self.supervisor.get(&module_id) else {
3860 return Ok(vec![control_error_frame(
3861 &frame,
3862 "unknown_module",
3863 format!("module_id '{module_id}' is not supervised"),
3864 )?]);
3865 };
3866 vec![module]
3867 } else {
3868 self.supervisor.list()
3869 };
3870
3871 let mut modules = Vec::with_capacity(selected.len());
3872 for module in selected.drain(..) {
3873 let status = module.status().map_err(|err| {
3874 RouterError::backend(
3875 0,
3876 frame.header.corr,
3877 format!("failed to read supervisor status: {err}"),
3878 )
3879 })?;
3880 let module_declared = self
3881 .registry
3882 .get_module(&status.module_id)
3883 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3884 .and_then(|registration| registration.manifest.provenance)
3885 .map(|build| ModuleDeclaredProvenance::Reported { build })
3886 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3887 #[cfg(test)]
3888 let running_image = match &self.provenance_probe_override {
3889 Some(result) => result.clone(),
3890 None => module.running_image_agreement().await,
3891 };
3892 #[cfg(not(test))]
3893 let running_image = module.running_image_agreement().await;
3894 modules.push(SupervisorModuleProvenance {
3895 module_id: status.module_id,
3896 module_declared,
3897 daemon_observed: SupervisorObservedProcess {
3898 pid: status.pid,
3899 spawned_at_ms: status.spawned_at_ms,
3900 spawned_from: status.spawned_from,
3901 running_image,
3902 },
3903 });
3904 }
3905 let daemon = SupervisorDaemonProvenance {
3906 daemon_build: self.daemon_provenance.build.clone(),
3907 daemon_observed: DaemonObservedProcess {
3908 pid: self.daemon_provenance.pid,
3909 started_at_ms: self
3910 .daemon_provenance
3911 .start_clock
3912 .map(|clock| clock.started_at_ms())
3913 .or(self.daemon_provenance.started_at_ms),
3914 running_image: self
3915 .daemon_provenance
3916 .probe
3917 .observe(
3918 self.daemon_provenance.pid,
3919 self.daemon_provenance.executable_path.as_deref(),
3920 self.daemon_provenance.executable_identity,
3921 self.daemon_provenance.process_start_time,
3922 )
3923 .await,
3924 },
3925 };
3926 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3927 Ok(vec![control_response_body_frame(
3928 &frame,
3929 &response,
3930 "ClientControlResponse::SupervisorProvenance",
3931 )?])
3932 }
3933
3934 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3935 self.refresh_capability_requirements();
3936 let generation = self
3937 .registry
3938 .generation()
3939 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3940 let modules = self
3941 .supervisor
3942 .list()
3943 .into_iter()
3944 .map(|module| {
3945 let status = module.status_for_control("health").map_err(|err| {
3946 RouterError::backend(
3947 0,
3948 frame.header.corr,
3949 format!("failed to read supervisor health: {err}"),
3950 )
3951 })?;
3952 let module_id = status.module_id;
3953 let capability_detail = self
3954 .capability_evaluator
3955 .required_problem_detail(&module_id);
3956 Ok(SupervisorHealthEntry {
3957 module_id,
3958 status: status.health.status,
3959 detail: append_capability_problem_detail(
3960 status.health.detail,
3961 capability_detail,
3962 ),
3963 metrics: status.health.metrics,
3964 consecutive_failures: status.health.consecutive_failures,
3965 late_answer_count: status.health.late_answer_count,
3966 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
3967 last_action: status.health.last_action,
3968 last_action_ms: status.health.last_action_ms,
3969 last_probe_ms: status.health.last_probe_ms,
3970 })
3971 })
3972 .collect::<Result<Vec<_>, RouterError>>()?;
3973 let response = ClientControlResponse::SupervisorHealth {
3974 generation,
3975 modules,
3976 };
3977 Ok(vec![control_response_body_frame(
3978 &frame,
3979 &response,
3980 "ClientControlResponse::SupervisorHealth",
3981 )?])
3982 }
3983
3984 async fn handle_supervisor_restart(
3985 &self,
3986 frame: Frame,
3987 module_id: String,
3988 drain_timeout_ms: Option<u64>,
3989 ) -> Result<Vec<Frame>, RouterError> {
3990 let operation_lock = self.supervisor.operation_lock();
3991 let _operation_guard = operation_lock.lock().await;
3992 let Some(module) = self.supervisor.get(&module_id) else {
3993 return Ok(vec![control_error_frame(
3994 &frame,
3995 "unknown_module",
3996 format!("module_id '{module_id}' is not supervised"),
3997 )?]);
3998 };
3999
4000 self.route_outages.mark_operator_action(&module_id);
4001 if let Err(err) = module.restart(drain_timeout_ms).await {
4002 self.route_outages
4003 .operator_action_ended_unrefused(&module_id);
4004 let (code, message) = match err {
4005 crate::supervise::SuperviseError::Disabled { .. } => {
4006 ("module_disabled", err.to_string())
4007 }
4008 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4009 ("swap_in_progress", err.to_string())
4010 }
4011 _ => (
4012 "target_unavailable",
4013 format!("failed to restart module_id '{module_id}': {err}"),
4014 ),
4015 };
4016 return Ok(vec![control_error_frame(&frame, code, message)?]);
4017 }
4018
4019 let response = ClientControlResponse::SupervisorAck {
4020 module_id,
4021 applied: true,
4022 };
4023 Ok(vec![control_response_body_frame(
4024 &frame,
4025 &response,
4026 "ClientControlResponse::SupervisorAck",
4027 )?])
4028 }
4029
4030 async fn handle_supervisor_swap(
4034 &self,
4035 frame: Frame,
4036 module_id: String,
4037 ready_timeout_ms: Option<u64>,
4038 ) -> Result<Vec<Frame>, RouterError> {
4039 let module = {
4046 let operation_lock = self.supervisor.operation_lock();
4047 let _operation_guard = operation_lock.lock().await;
4048 self.supervisor.get(&module_id)
4049 };
4050 let Some(module) = module else {
4051 return Ok(vec![control_error_frame(
4052 &frame,
4053 "unknown_module",
4054 format!("module_id '{module_id}' is not supervised"),
4055 )?]);
4056 };
4057
4058 self.route_outages.mark_operator_action(&module_id);
4059 if let Err(err) = module
4060 .swap(ready_timeout_ms.map(Duration::from_millis))
4061 .await
4062 {
4063 self.route_outages
4064 .operator_action_ended_unrefused(&module_id);
4065 use crate::supervise::SuperviseError;
4066 let message = err.to_string();
4067 let error = match err {
4068 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4069 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4070 code: "swap_refused".to_string(),
4071 message,
4072 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4073 },
4074 SuperviseError::SwapFailed {
4075 arm,
4076 candidate_exit,
4077 ..
4078 } => ErrorBody {
4079 code: "swap_failed".to_string(),
4080 message,
4081 detail: Some(serde_json::json!({
4082 "arm": arm.as_str(),
4083 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4084 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4085 })),
4086 },
4087 _ => ErrorBody::new(
4088 "target_unavailable",
4089 format!("failed to swap module_id '{module_id}': {message}"),
4090 ),
4091 };
4092 return Ok(vec![control_error_body_frame(&frame, error)?]);
4093 }
4094 self.route_outages
4098 .operator_action_ended_unrefused(&module_id);
4099
4100 let response = ClientControlResponse::SupervisorAck {
4101 module_id,
4102 applied: true,
4103 };
4104 Ok(vec![control_response_body_frame(
4105 &frame,
4106 &response,
4107 "ClientControlResponse::SupervisorAck",
4108 )?])
4109 }
4110
4111 async fn handle_supervisor_reload(
4112 &self,
4113 frame: Frame,
4114 module_id: String,
4115 ) -> Result<Vec<Frame>, RouterError> {
4116 let operation_lock = self.supervisor.operation_lock();
4117 let _operation_guard = operation_lock.lock().await;
4118 let Some(module) = self.supervisor.get(&module_id) else {
4119 return Ok(vec![control_error_frame(
4120 &frame,
4121 "unknown_module",
4122 format!("module_id '{module_id}' is not supervised"),
4123 )?]);
4124 };
4125
4126 self.route_outages.mark_operator_action(&module_id);
4127 if let Err(err) = module.reload().await {
4128 self.route_outages
4129 .operator_action_ended_unrefused(&module_id);
4130 let (code, message) = match err {
4131 crate::supervise::SuperviseError::Disabled { .. } => {
4132 ("module_disabled", err.to_string())
4133 }
4134 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4135 ("swap_in_progress", err.to_string())
4136 }
4137 _ => (
4138 "reload_failed",
4139 format!("failed to reload module_id '{module_id}': {err}"),
4140 ),
4141 };
4142 return Ok(vec![control_error_frame(&frame, code, message)?]);
4143 }
4144
4145 let response = ClientControlResponse::SupervisorAck {
4146 module_id,
4147 applied: true,
4148 };
4149 Ok(vec![control_response_body_frame(
4150 &frame,
4151 &response,
4152 "ClientControlResponse::SupervisorAck",
4153 )?])
4154 }
4155
4156 async fn handle_supervisor_rescan(
4157 &self,
4158 frame: Frame,
4159 preview: bool,
4160 ) -> Result<Vec<Frame>, RouterError> {
4161 let Some(context) = self.rescan.clone() else {
4162 return Ok(vec![control_error_frame(
4163 &frame,
4164 "rescan_unavailable",
4165 "the daemon was not started with a reloadable config path".to_string(),
4166 )?]);
4167 };
4168
4169 let operation_lock = self.supervisor.operation_lock();
4170 let _operation_guard = operation_lock.lock().await;
4171 let loaded = match crate::daemon_config::load(&context.config_path) {
4172 Ok(config) => config,
4173 Err(err) => {
4174 return Ok(vec![control_error_frame(
4175 &frame,
4176 "invalid_daemon_config",
4177 format!("supervisor rescan rejected daemon config: {err}"),
4178 )?])
4179 }
4180 };
4181 let Some(config) = loaded else {
4189 return Ok(vec![control_error_frame(
4190 &frame,
4191 "invalid_daemon_config",
4192 format!(
4193 "daemon config not found at {}; refusing to rescan (an absent config would \
4194 retire every supervised module)",
4195 context.config_path.display()
4196 ),
4197 )?]);
4198 };
4199 let (
4200 configured_port,
4201 storage_config,
4202 admission_facts_carrier_module_id,
4203 admission_facts_targets,
4204 scope_authority_owners,
4205 modules,
4206 reserved_capabilities,
4207 ) = (
4208 config.port,
4209 config.storage,
4210 config.admission_facts_carrier_module_id,
4211 config.admission_facts_targets,
4212 config.scope_authority_owners,
4213 config.modules,
4214 config.reserved_capabilities,
4215 );
4216
4217 let mut restart_required = Vec::new();
4226 for section in RestartRequiredSection::ALL {
4227 let changed = match section {
4228 RestartRequiredSection::Port => configured_port != context.configured_port,
4229 RestartRequiredSection::Storage => storage_config != context.storage_config,
4230 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4231 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4232 }
4233 RestartRequiredSection::AdmissionFactsTargets => {
4234 admission_facts_targets != context.admission_facts_targets
4235 }
4236 RestartRequiredSection::ScopeAuthorityOwners => {
4237 scope_authority_owners != context.scope_authority_owners
4238 }
4239 };
4240 if changed {
4241 restart_required.push(section.label().to_string());
4242 }
4243 }
4244 if !restart_required.is_empty() {
4245 warn!(
4246 config_path = %context.config_path.display(),
4247 sections = %restart_required.join(", "),
4248 "daemon config changed outside the modules section; restart the daemon to apply those changes"
4249 );
4250 }
4251
4252 for configured in &modules {
4253 if let Err(err) = validate_spec(&configured.module_spec()) {
4254 return Ok(vec![control_error_frame(
4255 &frame,
4256 "invalid_daemon_config",
4257 format!("supervisor rescan rejected daemon config: {err}"),
4258 )?]);
4259 }
4260 }
4261
4262 let configured_capabilities = modules
4263 .iter()
4264 .map(|module| (module.module_id.clone(), module.enabled))
4265 .collect::<Vec<_>>();
4266 let preview_capability_warnings = if preview {
4267 let (_, registrations) = self.runtime_capability_snapshot()?;
4268 let current_modules = self
4269 .supervisor
4270 .list()
4271 .into_iter()
4272 .map(|module| module.module_id().to_string())
4273 .collect::<BTreeSet<_>>();
4274 let resulting_modules = configured_capabilities.clone();
4275 let removed = current_modules
4276 .into_iter()
4277 .filter(|module_id| {
4278 !resulting_modules
4279 .iter()
4280 .any(|(configured_id, _)| configured_id == module_id)
4281 })
4282 .collect::<Vec<_>>();
4283 self.capability_evaluator.preview_removal_warnings(
4284 resulting_modules,
4285 &removed,
4286 ®istrations,
4287 )
4288 } else {
4289 Vec::new()
4290 };
4291 let result = match self
4292 .reconcile_supervised_modules(&context.supervisor, modules, preview)
4293 .await
4294 {
4295 Ok(result) => result,
4296 Err(message) => {
4297 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4298 }
4299 };
4300 if !preview {
4301 self.capability_evaluator
4302 .configure(configured_capabilities, reserved_capabilities);
4303 self.capability_evaluator.wake_deadline_loop();
4304 self.refresh_capability_requirements();
4305 }
4306 let mut result = result;
4307 result.restart_required = restart_required;
4308 result.capability_warnings = preview_capability_warnings;
4309 let response = ClientControlResponse::SupervisorRescan { result };
4310 Ok(vec![control_response_body_frame(
4311 &frame,
4312 &response,
4313 "ClientControlResponse::SupervisorRescan",
4314 )?])
4315 }
4316
4317 async fn handle_supervisor_release_reserved(
4318 &self,
4319 frame: Frame,
4320 module_id: String,
4321 ) -> Result<Vec<Frame>, RouterError> {
4322 let Some(context) = self.rescan.clone() else {
4323 return Ok(vec![control_error_frame(
4324 &frame,
4325 "release_unavailable",
4326 "reserved-id release requires a daemon started with a reloadable config path",
4327 )?]);
4328 };
4329 let operation_lock = self.supervisor.operation_lock();
4330 let _operation_guard = operation_lock.lock().await;
4331 let loaded = match crate::daemon_config::load(&context.config_path) {
4332 Ok(Some(config)) => config,
4333 Ok(None) => {
4334 return Ok(vec![control_error_frame(
4335 &frame,
4336 "invalid_daemon_config",
4337 format!(
4338 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4339 context.config_path.display()
4340 ),
4341 )?])
4342 }
4343 Err(err) => {
4344 return Ok(vec![control_error_frame(
4345 &frame,
4346 "invalid_daemon_config",
4347 format!("unable to verify reserved-id release against daemon config: {err}"),
4348 )?])
4349 }
4350 };
4351 if loaded
4352 .modules
4353 .iter()
4354 .any(|configured| configured.module_id == module_id)
4355 {
4356 return Ok(vec![control_error_frame(
4357 &frame,
4358 "reserved_module_configured",
4359 format!(
4360 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4361 ),
4362 )?]);
4363 }
4364 if !self.supervisor.release_retained_reserved_gate(&module_id) {
4365 return Ok(vec![control_error_frame(
4366 &frame,
4367 "reserved_gate_not_retained",
4368 format!(
4369 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4370 ),
4371 )?]);
4372 }
4373
4374 let response = ClientControlResponse::SupervisorAck {
4375 module_id,
4376 applied: true,
4377 };
4378 Ok(vec![control_response_body_frame(
4379 &frame,
4380 &response,
4381 "ClientControlResponse::SupervisorAck",
4382 )?])
4383 }
4384
4385 async fn reconcile_supervised_modules(
4394 &self,
4395 supervisor: &Supervisor,
4396 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4397 preview: bool,
4398 ) -> Result<SupervisorRescanResult, String> {
4399 let mut current = BTreeMap::new();
4400 for module in self.supervisor.list() {
4401 let (spec, health) = module.configuration().map_err(|err| {
4402 format!(
4403 "failed to read configuration for module_id '{}': {err}",
4404 module.module_id()
4405 )
4406 })?;
4407 let enabled = module
4408 .status()
4409 .map_err(|err| {
4410 format!(
4411 "failed to read status for module_id '{}': {err}",
4412 module.module_id()
4413 )
4414 })?
4415 .enabled;
4416 current.insert(
4417 module.module_id().to_string(),
4418 (module, spec, health, enabled),
4419 );
4420 }
4421 let configured = configured_modules
4422 .into_iter()
4423 .map(|module| (module.module_id.clone(), module))
4424 .collect::<BTreeMap<_, _>>();
4425
4426 let added = configured
4427 .keys()
4428 .filter(|module_id| !current.contains_key(*module_id))
4429 .cloned()
4430 .collect::<Vec<_>>();
4431 let removed = current
4432 .keys()
4433 .filter(|module_id| !configured.contains_key(*module_id))
4434 .cloned()
4435 .collect::<Vec<_>>();
4436 let mut changed_pending_reload = Vec::new();
4437 let mut configuration_changes = BTreeSet::new();
4438 let mut enabled_changes = BTreeSet::new();
4439 let mut unchanged = 0_u32;
4440
4441 for (module_id, configured_module) in &configured {
4442 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4443 else {
4444 continue;
4445 };
4446 let configuration_changed = *current_spec != configured_module.module_spec()
4447 || *current_health != configured_module.health;
4448 let enabled_changed = *current_enabled != configured_module.enabled;
4449 if configuration_changed {
4450 configuration_changes.insert(module_id.clone());
4451 changed_pending_reload.push(module_id.clone());
4452 }
4453 if enabled_changed {
4454 enabled_changes.insert(module_id.clone());
4455 }
4456 if !configuration_changed && !enabled_changed {
4457 unchanged = unchanged.saturating_add(1);
4458 }
4459 }
4460
4461 if preview {
4467 return Ok(SupervisorRescanResult {
4468 added,
4469 removed,
4470 changed_pending_reload,
4471 enabled_changes: enabled_changes.iter().cloned().collect(),
4472 unchanged,
4473 preview: true,
4474 restart_required: Vec::new(),
4478 capability_warnings: Vec::new(),
4479 });
4480 }
4481
4482 for module_id in &removed {
4483 let module = ¤t
4484 .get(module_id)
4485 .expect("removed module came from current supervisor state")
4486 .0;
4487 module.retire().await.map_err(|err| {
4488 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4489 })?;
4490 self.supervisor.record_rescan_removal(module_id);
4524 self.supervisor.retire(module_id);
4525 self.route_outages.forget(module_id);
4526 }
4527
4528 for module_id in configured.keys() {
4529 let Some((module, _, _, _)) = current.get(module_id) else {
4530 continue;
4531 };
4532 let configured_module = configured
4533 .get(module_id)
4534 .expect("configured module id came from configured map");
4535 if configuration_changes.contains(module_id) {
4536 module
4537 .update_configuration(
4538 configured_module.module_spec(),
4539 configured_module.health,
4540 configured_module.drain_timeout_ms,
4541 )
4542 .await
4543 .map_err(|err| {
4544 format!(
4545 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4546 )
4547 })?;
4548 }
4549 if enabled_changes.contains(module_id) {
4550 self.route_outages.mark_operator_action(module_id);
4553 module
4554 .set_enabled(configured_module.enabled)
4555 .await
4556 .map_err(|err| {
4557 self.route_outages.operator_action_ended_unrefused(module_id);
4558 format!(
4559 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4560 configured_module.enabled
4561 )
4562 })?;
4563 }
4564 }
4565
4566 for module_id in &added {
4567 let configured_module = configured
4568 .get(module_id)
4569 .expect("added module id came from configured map");
4570 supervisor
4571 .supervise_configured_with_health(
4572 configured_module.module_spec(),
4573 configured_module.enabled,
4574 configured_module.health,
4575 configured_module.drain_timeout_ms,
4576 configured_module.restart,
4577 )
4578 .map_err(|err| {
4579 format!("failed to add module_id '{module_id}' during rescan: {err}")
4580 })?;
4581 }
4582
4583 Ok(SupervisorRescanResult {
4584 added,
4585 removed,
4586 changed_pending_reload,
4587 enabled_changes: enabled_changes.iter().cloned().collect(),
4588 unchanged,
4589 preview: false,
4590 restart_required: Vec::new(),
4593 capability_warnings: Vec::new(),
4594 })
4595 }
4596
4597 async fn handle_supervisor_set_enabled(
4598 &self,
4599 frame: Frame,
4600 module_id: String,
4601 enabled: bool,
4602 ) -> Result<Vec<Frame>, RouterError> {
4603 let operation_lock = self.supervisor.operation_lock();
4604 let _operation_guard = operation_lock.lock().await;
4605 let Some(module) = self.supervisor.get(&module_id) else {
4606 return Ok(vec![control_error_frame(
4607 &frame,
4608 "unknown_module",
4609 format!("module_id '{module_id}' is not supervised"),
4610 )?]);
4611 };
4612
4613 self.route_outages.mark_operator_action(&module_id);
4616 let applied = match module.set_enabled(enabled).await {
4617 Ok(applied) => applied,
4618 Err(err) => {
4619 self.route_outages
4620 .operator_action_ended_unrefused(&module_id);
4621 return Ok(vec![control_error_frame(
4622 &frame,
4623 "target_unavailable",
4624 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4625 )?]);
4626 }
4627 };
4628 if !applied {
4629 self.route_outages
4632 .operator_action_ended_unrefused(&module_id);
4633 }
4634
4635 self.capability_evaluator.wake_deadline_loop();
4636 self.refresh_capability_requirements();
4637 let response = ClientControlResponse::SupervisorAck { module_id, applied };
4638 Ok(vec![control_response_body_frame(
4639 &frame,
4640 &response,
4641 "ClientControlResponse::SupervisorAck",
4642 )?])
4643 }
4644
4645 async fn handle_supervisor_health_probe(
4646 &self,
4647 frame: Frame,
4648 module_id: String,
4649 ) -> Result<Vec<Frame>, RouterError> {
4650 self.refresh_capability_requirements();
4651 let Some(registration) = self
4652 .registry
4653 .get_module(&module_id)
4654 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4655 else {
4656 return Ok(vec![control_error_frame(
4657 &frame,
4658 "unknown_module",
4659 format!("module_id '{module_id}' is not registered"),
4660 )?]);
4661 };
4662
4663 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4676 {
4677 return Ok(vec![control_error_frame(
4678 &frame,
4679 "health_not_advertised",
4680 format!("module_id '{module_id}' did not advertise health.check"),
4681 )?]);
4682 }
4683
4684 let deadline = Instant::now() + self.health_probe_timeout;
4685 let pending = match self.forwarding.begin_module_control_rpc_for(
4686 &module_id,
4687 MODULE_CONTROL_OP_HEALTH_CHECK,
4688 deadline,
4689 ) {
4690 Ok(pending) => pending,
4691 Err(err) => {
4692 return Ok(vec![control_error_frame(
4693 &frame,
4694 forwarding_error_code(&err),
4695 err.to_string(),
4696 )?])
4697 }
4698 };
4699
4700 let PendingModuleControlRpc {
4701 endpoint,
4702 module_sink,
4703 negotiated_ver,
4704 corr: probe_corr,
4705 receiver,
4706 } = pending;
4707 let mut guard =
4708 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4709 let probe_body =
4710 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4711 RouterError::backend(
4712 0,
4713 frame.header.corr,
4714 format!("failed to encode health.check request: {err}"),
4715 )
4716 })?;
4717 let probe_frame = Frame::build_with_version(
4718 negotiated_ver,
4719 FrameType::Request,
4720 control_flags(),
4721 0,
4722 0,
4723 probe_corr,
4724 probe_body,
4725 )
4726 .map_err(RouterError::FrameBuild)?;
4727
4728 if let Err(err) = module_sink.send(probe_frame).await {
4729 return Ok(vec![control_error_frame(
4730 &frame,
4731 "target_unavailable",
4732 err.to_string(),
4733 )?]);
4734 }
4735
4736 match timeout_at(deadline, receiver).await {
4737 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4738 guard.disarm();
4739 let Some(report) = response.health_report() else {
4740 return Ok(vec![control_error_frame(
4741 &frame,
4742 "invalid_control_body",
4743 "health.check RPC returned a non-health response",
4744 )?]);
4745 };
4746 let HealthReport {
4751 status,
4752 detail,
4753 metrics,
4754 } = report;
4755 let capability_detail = self
4756 .capability_evaluator
4757 .required_problem_detail(&module_id);
4758 let response = ClientControlResponse::SupervisorHealthProbe {
4759 module_id,
4760 status,
4761 detail: append_capability_problem_detail(detail, capability_detail),
4762 metrics,
4763 };
4764 Ok(vec![control_response_body_frame(
4765 &frame,
4766 &response,
4767 "ClientControlResponse::SupervisorHealthProbe",
4768 )?])
4769 }
4770 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4771 guard.disarm();
4772 Ok(vec![control_error_body_frame(&frame, body)?])
4773 }
4774 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4775 guard.disarm();
4776 Ok(vec![control_error_frame(
4777 &frame,
4778 "target_unavailable",
4779 message,
4780 )?])
4781 }
4782 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4783 guard.disarm();
4784 Ok(vec![control_error_frame(
4785 &frame,
4786 "invalid_control_body",
4787 message,
4788 )?])
4789 }
4790 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4791 guard.disarm();
4792 Ok(vec![control_error_frame(
4793 &frame,
4794 "invalid_control_body",
4795 format!("expected module-control op '{expected}', got '{actual}'"),
4796 )?])
4797 }
4798 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4799 guard.disarm();
4800 Ok(vec![control_error_frame(
4801 &frame,
4802 "module_timeout",
4803 format!(
4804 "module_id '{module_id}' answered health.check after {:?}",
4805 self.health_probe_timeout
4806 ),
4807 )?])
4808 }
4809 Ok(Err(_)) => Ok(vec![control_error_frame(
4810 &frame,
4811 "target_unavailable",
4812 "health.check waiter was canceled before the module responded",
4813 )?]),
4814 Err(_) => Ok(vec![control_error_frame(
4815 &frame,
4816 "module_timeout",
4817 format!(
4818 "module_id '{module_id}' did not answer health.check within {:?}",
4819 self.health_probe_timeout
4820 ),
4821 )?]),
4822 }
4823 }
4824
4825 fn supervisor_status(
4826 &self,
4827 module_id: &str,
4828 corr: u64,
4829 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4830 self.supervisor
4831 .get(module_id)
4832 .map(|module| {
4833 let warming = module.is_warming_for_control("status").map_err(|err| {
4834 RouterError::backend(
4835 0,
4836 corr,
4837 format!(
4838 "failed to read supervisor warming state for module_id '{module_id}': {err}"
4839 ),
4840 )
4841 })?;
4842 module.status_for_control("status").map_err(|err| {
4843 RouterError::backend(
4844 0,
4845 corr,
4846 format!(
4847 "failed to read supervisor status for module_id '{module_id}': {err}"
4848 ),
4849 )
4850 }).map(|status| (status, warming))
4851 })
4852 .transpose()
4853 }
4854
4855 fn guard_module_control_op(
4856 &self,
4857 frame: &Frame,
4858 module_id: &str,
4859 op: &str,
4860 ) -> Result<Option<Frame>, RouterError> {
4861 if self.module_grants_op(module_id, op, frame.header.corr)? {
4862 return Ok(None);
4863 }
4864
4865 Ok(Some(control_error_frame(
4866 frame,
4867 "op_not_allowed",
4868 format!("module_id '{module_id}' did not grant control op '{op}'"),
4869 )?))
4870 }
4871
4872 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4873 let Some(registration) = self
4874 .registry
4875 .get_module(module_id)
4876 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4877 else {
4878 return Ok(false);
4879 };
4880 Ok(module_registration_grants_op(®istration.control_ops, op))
4881 }
4882
4883 fn handle_status_update(
4884 &self,
4885 endpoint: ModuleEndpointId,
4886 frame: Frame,
4887 ) -> Result<Vec<Frame>, RouterError> {
4888 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4889 Ok(update) => update,
4890 Err(err) => {
4891 if is_known_module_push_op(&frame.body) {
4896 return Ok(vec![control_error_frame(
4897 &frame,
4898 "invalid_control_body",
4899 format!("malformed module control push body: {err}"),
4900 )?]);
4901 }
4902 return Ok(Vec::new());
4903 }
4904 };
4905
4906 match update {
4907 ModuleControlPush::RouteStatus {
4908 route_channel,
4909 route_epoch,
4910 status,
4911 } => {
4912 self.forwarding
4913 .cache_status(endpoint, route_channel, route_epoch, status)
4914 .map_err(RouterError::Forwarding)?;
4915 }
4916 }
4917 Ok(Vec::new())
4918 }
4919
4920 fn handle_route_poll(
4921 &self,
4922 ctx: &RouteCtx,
4923 frame: Frame,
4924 route_channel: u16,
4925 route_epoch: u32,
4926 kind: PollKind,
4927 ) -> Result<Vec<Frame>, RouterError> {
4928 let snapshot = self
4929 .forwarding
4930 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4931 .map_err(RouterError::Forwarding)?;
4932 let response = match (kind, snapshot) {
4933 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4934 ClientControlResponse::RoutePoll {
4935 route_channel,
4936 route_epoch,
4937 status,
4938 live: None,
4939 }
4940 }
4941 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4942 route_channel,
4943 route_epoch,
4944 status: None,
4945 live: None,
4946 },
4947 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
4948 let live = self
4974 .process_liveness
4975 .as_ref()
4976 .and_then(|source| source.process_live(&module_id))
4977 .unwrap_or(true);
4978 ClientControlResponse::RoutePoll {
4979 route_channel,
4980 route_epoch,
4981 status: None,
4982 live: Some(live),
4983 }
4984 }
4985 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4986 route_channel,
4987 route_epoch,
4988 status: None,
4989 live: Some(false),
4990 },
4991 };
4992
4993 Ok(vec![control_response_body_frame(
4994 &frame,
4995 &response,
4996 "ClientControlResponse::RoutePoll",
4997 )?])
4998 }
4999
5000 pub(crate) fn observe_module_control_completion(
5001 &self,
5002 completion: ModuleControlRpcCompletion,
5003 ) -> bool {
5004 match completion {
5005 ModuleControlRpcCompletion::Unknown => false,
5006 ModuleControlRpcCompletion::Settled => true,
5007 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5008 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5009 info!(
5010 module_id = %module_id,
5011 latency_ms,
5012 "late health.check answer proves the module is alive"
5013 );
5014 match self
5015 .supervisor
5016 .record_late_health_answer(&module_id, latency_ms)
5017 {
5018 Ok(true) => {}
5019 Ok(false) => debug!(
5020 module_id = %module_id,
5021 latency_ms,
5022 "late health.check answer has no active supervisor snapshot"
5023 ),
5024 Err(err) => warn!(
5025 module_id = %module_id,
5026 latency_ms,
5027 error = %err,
5028 "failed to record late health.check answer"
5029 ),
5030 }
5031 true
5032 }
5033 }
5034 }
5035
5036 fn refuse_to_end_module_connection_for_a_client(
5057 &self,
5058 module_connection_id: ConnectionId,
5059 corr: u64,
5060 err: ForwardingError,
5061 ) -> Result<(), RouterError> {
5062 if let ForwardingError::ConnectionClosing { connection_id } = err {
5063 if connection_id != module_connection_id {
5064 warn!(
5065 module_connection_id = module_connection_id.get(),
5066 client_connection_id = connection_id.get(),
5067 corr,
5068 "dropping a route.bind response for a closing client; the module connection keeps serving"
5069 );
5070 return Ok(());
5071 }
5072 }
5073 Err(RouterError::Forwarding(err))
5074 }
5075
5076 fn handle_module_relay_response(
5077 &self,
5078 connection_id: ConnectionId,
5079 frame: Frame,
5080 ) -> Result<Vec<Frame>, RouterError> {
5081 let mut secondary_error = None;
5082 let outcome = match frame.header.ty {
5083 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5084 Ok(probe) if probe.op == "route.bind" => {
5085 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5086 Ok(ModuleControlResponse::RouteBindAck {}) => {
5087 RouteBindRelayOutcome::Accepted
5088 }
5089 Ok(other) => {
5090 let message =
5091 format!("route.bind response carried unexpected body: {other:?}");
5092 secondary_error = Some(control_error_frame(
5093 &frame,
5094 "invalid_control_body",
5095 message.clone(),
5096 )?);
5097 RouteBindRelayOutcome::ModuleGone(message)
5098 }
5099 Err(err) => {
5100 let message = format!("malformed route.bind response body: {err}");
5101 secondary_error = Some(control_error_frame(
5102 &frame,
5103 "invalid_control_body",
5104 message.clone(),
5105 )?);
5106 RouteBindRelayOutcome::ModuleGone(message)
5107 }
5108 }
5109 }
5110 Ok(probe) => {
5111 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5112 {
5113 Ok(response) => ModuleControlRpcOutcome::Response(response),
5114 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5115 "malformed {} response body: {err}",
5116 probe.op
5117 )),
5118 };
5119 let completion = self
5120 .forwarding
5121 .complete_module_control_rpc(
5122 connection_id,
5123 frame.header.corr,
5124 Some(&probe.op),
5125 outcome,
5126 )
5127 .map_err(RouterError::Forwarding)?;
5128 if !self.observe_module_control_completion(completion) {
5129 debug!(
5130 connection_id = connection_id.get(),
5131 corr = frame.header.corr,
5132 op = %probe.op,
5133 "dropping late or unknown module-control RPC response"
5134 );
5135 }
5136 return Ok(Vec::new());
5137 }
5138 Err(err) => {
5139 if let Some(expected_op) = self
5140 .forwarding
5141 .pending_module_control_op(connection_id, frame.header.corr)
5142 .map_err(RouterError::Forwarding)?
5143 {
5144 let completion = self
5145 .forwarding
5146 .complete_module_control_rpc(
5147 connection_id,
5148 frame.header.corr,
5149 None,
5150 ModuleControlRpcOutcome::MalformedResponse(format!(
5151 "malformed {expected_op} response body: {err}"
5152 )),
5153 )
5154 .map_err(RouterError::Forwarding)?;
5155 if !self.observe_module_control_completion(completion) {
5156 debug!(
5157 connection_id = connection_id.get(),
5158 corr = frame.header.corr,
5159 "dropping late malformed module-control RPC response"
5160 );
5161 }
5162 return Ok(Vec::new());
5163 }
5164 let message = format!("malformed route.bind response body: {err}");
5165 secondary_error = Some(control_error_frame(
5166 &frame,
5167 "invalid_control_body",
5168 message.clone(),
5169 )?);
5170 RouteBindRelayOutcome::ModuleGone(message)
5171 }
5172 },
5173 FrameType::Error => {
5174 if self
5175 .forwarding
5176 .pending_module_control_op(connection_id, frame.header.corr)
5177 .map_err(RouterError::Forwarding)?
5178 .is_some()
5179 {
5180 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5181 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5182 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5183 "malformed module-control ERROR body: {err}"
5184 )),
5185 };
5186 let completion = self
5187 .forwarding
5188 .complete_module_control_rpc(
5189 connection_id,
5190 frame.header.corr,
5191 None,
5192 outcome,
5193 )
5194 .map_err(RouterError::Forwarding)?;
5195 if !self.observe_module_control_completion(completion) {
5196 debug!(
5197 connection_id = connection_id.get(),
5198 corr = frame.header.corr,
5199 "dropping late or unknown module-control RPC error"
5200 );
5201 }
5202 return Ok(Vec::new());
5203 }
5204 match serde_json::from_slice::<ErrorBody>(&frame.body) {
5205 Ok(body) => RouteBindRelayOutcome::Rejected(body),
5206 Err(err) => {
5207 let message = format!("malformed route.bind ERROR body: {err}");
5208 secondary_error = Some(control_error_frame(
5209 &frame,
5210 "invalid_control_body",
5211 message.clone(),
5212 )?);
5213 RouteBindRelayOutcome::ModuleGone(message)
5214 }
5215 }
5216 }
5217 ty => {
5218 return Ok(vec![control_error_frame(
5219 &frame,
5220 "unsupported_control_frame",
5221 format!("unsupported module channel-0 frame {ty:?}"),
5222 )?])
5223 }
5224 };
5225
5226 let settled =
5227 self.forwarding
5228 .complete_pending_relay(connection_id, frame.header.corr, outcome);
5229 let completion = match settled {
5230 Ok(completion) => completion,
5231 Err(err) => {
5232 self.refuse_to_end_module_connection_for_a_client(
5233 connection_id,
5234 frame.header.corr,
5235 err,
5236 )?;
5237 return Ok(secondary_error.into_iter().collect());
5238 }
5239 };
5240 if let Some(target) = completion.abandoned.as_ref() {
5241 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5242 }
5243 if !completion.settled {
5244 debug!(
5245 connection_id = connection_id.get(),
5246 corr = frame.header.corr,
5247 frame_type = ?frame.header.ty,
5248 "dropping late or unknown route.bind relay response"
5249 );
5250 }
5251 Ok(secondary_error.into_iter().collect())
5252 }
5253
5254 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5255 debug!(connection_id = connection_id.get(), "handling GOODBYE");
5256 let registrations = self
5257 .deregister_connection(connection_id)
5258 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5259 let released_routes = self
5260 .forwarding
5261 .cleanup_connection(connection_id)
5262 .map_err(RouterError::Forwarding)?;
5263 self.emit_route_goodbyes(released_routes);
5264 if !registrations.is_empty() {
5266 crate::supervise::notify_registration_release();
5267 }
5268 Ok(Vec::new())
5269 }
5270}
5271
5272impl Default for ControlHandler {
5273 fn default() -> Self {
5274 Self::new(Arc::new(Registry::default()))
5275 }
5276}
5277
5278impl crate::supervise::SwapPromotionObserver for ControlHandler {
5279 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5280 self.apply_registration_capabilities(registration);
5281 }
5282}
5283
5284fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5285 CapabilityRequirementStatus {
5286 consumer: status.consumer,
5287 capability: status.capability,
5288 need: match status.need {
5289 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5290 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5291 },
5292 verdict: status.verdict.as_str().to_string(),
5293 episode_seq: status.episode_seq,
5294 config_satisfiable: status.config_satisfiable,
5295 runtime_available: status.runtime_available,
5296 detail: status.detail,
5297 }
5298}
5299
5300fn append_capability_problem_detail(
5301 detail: Option<String>,
5302 capability_detail: Option<String>,
5303) -> Option<String> {
5304 match (detail, capability_detail) {
5305 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5306 (Some(detail), None) => Some(detail),
5307 (None, Some(capability_detail)) => Some(capability_detail),
5308 (None, None) => None,
5309 }
5310}
5311
5312fn subc_ops() -> Vec<String> {
5313 SUBC_CONTROL_OPS
5314 .iter()
5315 .map(|op| (*op).to_string())
5316 .collect()
5317}
5318
5319fn module_subc_ops() -> Vec<String> {
5320 SUBC_CONTROL_OPS
5321 .iter()
5322 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5323 .map(|op| (*op).to_string())
5324 .collect()
5325}
5326
5327#[cfg(test)]
5328fn module_baseline_control_ops() -> Vec<String> {
5329 MODULE_BASELINE_CONTROL_OPS
5330 .iter()
5331 .map(|op| (*op).to_string())
5332 .collect()
5333}
5334
5335fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5336 let mut seen = HashSet::new();
5337 let mut effective = Vec::new();
5338 for op in MODULE_BASELINE_CONTROL_OPS {
5339 if seen.insert((*op).to_string()) {
5340 effective.push((*op).to_string());
5341 }
5342 }
5343 for op in declared.unwrap_or_default() {
5344 if seen.insert(op.clone()) {
5345 effective.push(op);
5346 }
5347 }
5348 effective
5349}
5350
5351fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5352 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5353}
5354
5355fn target_module_id(target: &RouteTarget) -> &str {
5356 match target {
5357 RouteTarget::ToolProvider { module_id }
5358 | RouteTarget::ManagementSurface { module_id }
5359 | RouteTarget::InternalService { module_id, .. } => module_id,
5360 }
5361}
5362
5363fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5364 roles.iter().any(|role| match (target, role) {
5365 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5366 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5367 (
5368 RouteTarget::InternalService { service_id, .. },
5369 ProviderRole::InternalService {
5370 service_id: provided,
5371 ..
5372 },
5373 ) => service_id == provided,
5374 _ => false,
5375 })
5376}
5377
5378fn is_routable_role(role: &ProviderRole) -> bool {
5379 matches!(
5380 role,
5381 ProviderRole::ToolProvider { .. }
5382 | ProviderRole::ManagementSurface { .. }
5383 | ProviderRole::InternalService { .. }
5384 )
5385}
5386
5387#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5388enum ControlRequestBodyError {
5389 UnknownOp,
5390 InvalidBody,
5391}
5392
5393#[derive(Debug, Deserialize)]
5394struct ControlOpProbe {
5395 op: String,
5396}
5397
5398const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5401
5402fn is_known_module_push_op(body: &[u8]) -> bool {
5403 serde_json::from_slice::<ControlOpProbe>(body)
5404 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5405 .unwrap_or(false)
5406}
5407
5408fn is_known_module_request_op(body: &[u8]) -> bool {
5409 serde_json::from_slice::<ControlOpProbe>(body)
5410 .map(|probe| is_module_to_subc_op(&probe.op))
5411 .unwrap_or(false)
5412}
5413
5414fn is_module_to_subc_op(op: &str) -> bool {
5415 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5416}
5417
5418fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5419 debug!(
5420 op = %op,
5421 connection_id = connection_id.get(),
5422 corr,
5423 "control dispatch"
5424 );
5425}
5426
5427fn log_slow_control_dispatch(
5428 dispatch_started_at: Option<StdInstant>,
5429 op: &'static str,
5430 connection_id: ConnectionId,
5431 corr: u64,
5432) {
5433 let Some(dispatch_started_at) = dispatch_started_at else {
5434 return;
5435 };
5436 let elapsed = dispatch_started_at.elapsed();
5437 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5438 warn!(
5439 op = %op,
5440 connection_id = connection_id.get(),
5441 corr,
5442 elapsed_ms = elapsed.as_millis() as u64,
5443 "slow control dispatch"
5444 );
5445 }
5446}
5447
5448fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5449 match request {
5450 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5451 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5452 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5453 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5454 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5455 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5456 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5457 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5458 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5459 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5460 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5461 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5462 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5463 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5464 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5465 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5466 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5467 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5468 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5469 }
5470}
5471
5472fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5473 match request {
5474 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5475 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5476 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
5477 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
5478 }
5479}
5480
5481fn parse_client_control_request(
5482 body: &[u8],
5483) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5484 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5485 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5486 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5487 ControlRequestBodyError::InvalidBody
5488 }
5489 Ok(_) => ControlRequestBodyError::UnknownOp,
5490 Err(_) => ControlRequestBodyError::InvalidBody,
5491 };
5492 (err, classification)
5493 })
5494}
5495
5496fn parse_module_control_request_from_module(
5497 body: &[u8],
5498) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5499 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5500 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5501 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
5502 Ok(_) => ControlRequestBodyError::UnknownOp,
5503 Err(_) => ControlRequestBodyError::InvalidBody,
5504 };
5505 (err, classification)
5506 })
5507}
5508
5509#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5510enum ProviderRoleKind {
5511 ToolProvider,
5512 PipelineStage,
5513 ManagementSurface,
5514 InternalService,
5515}
5516
5517fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5518 match role {
5519 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5520 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5521 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5522 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5523 }
5524}
5525
5526fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5527 roles.iter().map(provider_role_kind).collect()
5528}
5529
5530fn capability_census_trigger(
5534 old: Option<&CapabilityDeclarations>,
5535 new: Option<&CapabilityDeclarations>,
5536) -> bool {
5537 let old_provides = old
5538 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5539 .unwrap_or_default();
5540 let old_denies = old
5541 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5542 .unwrap_or_default();
5543 let new = new.cloned().unwrap_or(CapabilityDeclarations {
5544 provides: Vec::new(),
5545 requires: Vec::new(),
5546 must_never_reach: Vec::new(),
5547 });
5548
5549 new.provides
5550 .iter()
5551 .any(|capability| !old_provides.contains(capability))
5552 || new
5553 .must_never_reach
5554 .iter()
5555 .any(|capability| !old_denies.contains(capability))
5556}
5557
5558fn denied_capability<'a>(
5561 opening_manifest: &'a ModuleManifest,
5562 target_manifest: &ModuleManifest,
5563) -> Option<&'a str> {
5564 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5565 let target_capabilities = target_manifest.capabilities.as_ref()?;
5566 opening_capabilities
5567 .must_never_reach
5568 .iter()
5569 .find(|denied| {
5570 target_capabilities
5571 .provides
5572 .iter()
5573 .any(|provided| provided == *denied)
5574 })
5575 .map(String::as_str)
5576}
5577
5578fn catalog_update_frozen_field_message(
5579 registered: &ModuleManifest,
5580 provides: &[ProviderRole],
5581) -> Option<String> {
5582 let old_has_provides = !registered.provides.is_empty();
5583 let new_has_provides = !provides.is_empty();
5584 if old_has_provides != new_has_provides {
5585 return Some(format!(
5586 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5587 registered.module_id
5588 ));
5589 }
5590
5591 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
5592 return Some(format!(
5593 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5594 registered.module_id
5595 ));
5596 }
5597
5598 let registered_concurrency = manifest_concurrency(registered);
5599 let mut candidate = registered.clone();
5600 candidate.provides = provides.to_vec();
5601 let candidate_concurrency = manifest_concurrency(&candidate);
5602 if candidate_concurrency != registered_concurrency {
5603 return Some(format!(
5604 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5605 registered.module_id, registered_concurrency, candidate_concurrency
5606 ));
5607 }
5608
5609 None
5612}
5613
5614fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5615 manifest.provides.iter().any(is_routable_role)
5616}
5617
5618fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5624 manifest
5625 .provides
5626 .iter()
5627 .find_map(|provider| match provider {
5628 ProviderRole::ToolProvider { concurrency, .. }
5629 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5630 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5631 })
5632 .unwrap_or(Concurrency::ModuleManaged)
5633}
5634
5635fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5642 let has_management_surface = manifest
5643 .provides
5644 .iter()
5645 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5646 if !has_management_surface {
5647 return false;
5648 }
5649 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5650 return false;
5651 };
5652 let Some(provides) = raw
5653 .get("manifest")
5654 .and_then(|manifest| manifest.get("provides"))
5655 .and_then(serde_json::Value::as_array)
5656 else {
5657 return false;
5658 };
5659 provides.iter().any(|role| {
5664 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5665 && role.get("concurrency").is_none()
5666 })
5667}
5668
5669fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5670 if peer_version != PROTOCOL_VERSION {
5671 return Err(format!(
5672 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5673 ));
5674 }
5675 Ok(PROTOCOL_VERSION)
5676}
5677
5678fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5679 Frame::build_with_version(
5680 response_version(frame),
5681 FrameType::Pong,
5682 frame.header.flags,
5683 0,
5684 0,
5685 frame.header.corr,
5686 Vec::new(),
5687 )
5688 .map_err(RouterError::FrameBuild)
5689}
5690
5691fn control_error_frame(
5692 frame: &Frame,
5693 code: &'static str,
5694 message: impl Into<String>,
5695) -> Result<Frame, RouterError> {
5696 control_error_body_frame(
5697 frame,
5698 ErrorBody {
5699 code: code.to_string(),
5700 message: message.into(),
5701 detail: None,
5702 },
5703 )
5704}
5705
5706fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5707 let body = serde_json::to_vec(&error).map_err(|err| {
5708 RouterError::backend(
5709 0,
5710 frame.header.corr,
5711 format!("failed to encode control ERROR: {err}"),
5712 )
5713 })?;
5714
5715 Frame::build_with_version(
5716 response_version(frame),
5717 FrameType::Error,
5718 control_flags(),
5719 0,
5720 0,
5721 frame.header.corr,
5722 body,
5723 )
5724 .map_err(RouterError::FrameBuild)
5725}
5726
5727fn control_response_body_frame<T: Serialize>(
5728 frame: &Frame,
5729 reply: &T,
5730 label: &'static str,
5731) -> Result<Frame, RouterError> {
5732 let body = serde_json::to_vec(reply).map_err(|err| {
5733 RouterError::backend(
5734 0,
5735 frame.header.corr,
5736 format!("failed to encode {label}: {err}"),
5737 )
5738 })?;
5739
5740 Frame::build_with_version(
5741 response_version(frame),
5742 FrameType::Response,
5743 control_flags(),
5744 0,
5745 0,
5746 frame.header.corr,
5747 body,
5748 )
5749 .map_err(RouterError::FrameBuild)
5750}
5751
5752fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5782 match err {
5783 ForwardingError::NoModuleConnection => "target_unavailable",
5784 ForwardingError::ModuleReloading { .. } => "module_reloading",
5785 ForwardingError::ClientRouteChannelExhausted { .. }
5786 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5787 ForwardingError::StaleModuleEndpoint
5788 | ForwardingError::UnknownReservation { .. }
5789 | ForwardingError::ConnectionClosing { .. }
5790 | ForwardingError::ClientEgressClosed { .. }
5791 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5792 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5795 ForwardingError::RelayCorrelationExhausted
5796 | ForwardingError::RouteOpenBuild(_)
5797 | ForwardingError::Poisoned => "forwarding_error",
5798 }
5799}
5800
5801fn response_version(frame: &Frame) -> u8 {
5802 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5803 frame.header.ver
5804 } else {
5805 PROTOCOL_VERSION
5806 }
5807}
5808
5809fn control_flags() -> Flags {
5810 Flags::new(false, Priority::Passive, false)
5811}
5812
5813fn send_goodbye_target_best_effort(
5818 counters: &DaemonCounters,
5819 target: &GoodbyeTarget,
5820 context: &'static str,
5821) {
5822 let Ok(frame) = Frame::build_with_version(
5823 target.negotiated_ver,
5824 FrameType::Goodbye,
5825 control_flags(),
5826 target.channel,
5827 target.epoch,
5828 0,
5829 Vec::new(),
5830 ) else {
5831 return;
5832 };
5833 crate::forwarding::send_module_route_goodbye(
5834 counters,
5835 &target.sink,
5836 frame,
5837 target.module_id.as_deref(),
5838 context,
5839 );
5840}
5841
5842pub(crate) fn send_route_control_pushes(
5843 forwarding: &ForwardingTable,
5844 routes: Vec<EndpointRoute>,
5845 push: ClientControlPush,
5846) {
5847 let body = match serde_json::to_vec(&push) {
5848 Ok(body) => body,
5849 Err(err) => {
5850 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5851 return;
5852 }
5853 };
5854 let mut targets = Vec::new();
5855 for route in routes {
5856 let target = route.goodbye_target;
5857 if let Some(existing) = targets
5858 .iter()
5859 .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5860 {
5861 debug_assert_eq!(
5862 existing.negotiated_ver, target.negotiated_ver,
5863 "one connection cannot negotiate multiple frame versions"
5864 );
5865 continue;
5866 }
5867 targets.push(target);
5868 }
5869 for target in targets {
5870 let frame = match Frame::build_with_version(
5871 target.negotiated_ver,
5872 FrameType::Push,
5873 control_flags(),
5874 0,
5875 0,
5876 0,
5877 body.clone(),
5878 ) {
5879 Ok(frame) => frame,
5880 Err(err) => {
5881 warn!(
5882 route_channel = target.channel,
5883 error = %err,
5884 "failed to build route lifecycle control PUSH frame"
5885 );
5886 continue;
5887 }
5888 };
5889 if let Err(err) = target.sink.try_send(frame) {
5890 if target.close_on_delivery_failure() {
5891 warn!(
5892 target_connection_id = target.connection_id.get(),
5893 route_channel = target.channel,
5894 error = %err,
5895 "route lifecycle control PUSH was not delivered to client; closing target connection"
5896 );
5897 let _ = forwarding.escalate_client_delivery_failure(
5898 target.connection_id,
5899 target.channel,
5900 target.epoch,
5901 CloseReason::new(
5902 "route_lifecycle_push_delivery_failed",
5903 format!(
5904 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
5905 target.channel
5906 ),
5907 ),
5908 crate::forwarding::UndeliveredFrame {
5909 module_id: target.module_id.as_deref(),
5910 sink: &target.sink,
5911 },
5912 );
5913 }
5914 }
5915 }
5916}
5917
5918#[cfg(test)]
5919mod tests {
5920 use std::{
5921 collections::BTreeMap,
5922 fmt,
5923 path::PathBuf,
5924 sync::{Arc, Mutex},
5925 time::Duration,
5926 };
5927 use subc_test_support::TestTempDir;
5928
5929 use serde_json::{json, Value};
5930 use subc_protocol::{
5931 manifest::{
5932 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
5933 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
5934 },
5935 session::HealthStatus,
5936 FrameType,
5937 };
5938
5939 use super::*;
5940 use crate::{
5941 forwarding::{DataRoute, DataRouteState},
5942 registry::ChannelState,
5943 router::FrameSink,
5944 stderr_tail::DEFAULT_MAX_LINE_BYTES,
5945 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
5946 RouteCtx, Router,
5947 };
5948 use tokio::{
5949 sync::mpsc,
5950 time::{sleep, Instant},
5951 };
5952 use tracing::{
5953 field::{Field, Visit},
5954 Event, Subscriber,
5955 };
5956 use tracing_subscriber::{layer::Context, prelude::*, Layer};
5957
5958 fn fake_aft_stub_path() -> PathBuf {
5977 let mut path = std::env::current_exe().expect("current_exe available in tests");
5978 path.pop(); path.pop(); path.push(if cfg!(windows) {
5981 "fake-aft-stub.exe"
5982 } else {
5983 "fake-aft-stub"
5984 });
5985 assert!(
5986 path.exists(),
5987 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
5988 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
5989 path.display()
5990 );
5991 path
5992 }
5993
5994 fn client_retries(code: &str) -> bool {
5999 subc_protocol::error_codes::is_retryable_route_open(code)
6000 }
6001
6002 #[test]
6013 fn retryability_of_forwarding_codes_matches_the_failure() {
6014 let transient = [
6017 ForwardingError::NoModuleConnection,
6018 ForwardingError::ModuleReloading {
6019 module_id: "m".into(),
6020 },
6021 ForwardingError::StaleModuleEndpoint,
6022 ForwardingError::UnknownReservation {
6023 client_channel: 1,
6024 module_channel: 1,
6025 },
6026 ForwardingError::ConnectionClosing {
6027 connection_id: ConnectionId::new(1),
6028 },
6029 ForwardingError::ClientEgressClosed {
6030 connection_id: ConnectionId::new(1),
6031 },
6032 ForwardingError::ModuleEgressUnavailable {
6033 connection_id: ConnectionId::new(1),
6034 },
6035 ];
6036 for err in transient {
6037 let code = forwarding_error_code(&err);
6038 assert!(
6039 client_retries(code),
6040 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6041 );
6042 }
6043
6044 let permanent = [
6049 ForwardingError::ClientRouteChannelExhausted {
6050 connection_id: ConnectionId::new(1),
6051 },
6052 ForwardingError::ModuleRouteChannelExhausted {
6053 endpoint: ModuleEndpointId {
6054 connection_id: ConnectionId::new(1),
6055 generation: 1,
6056 },
6057 },
6058 ForwardingError::RelayCorrelationExhausted,
6059 ForwardingError::RouteOpenBuild("x".into()),
6060 ForwardingError::Poisoned,
6061 ];
6062 for err in permanent {
6063 let code = forwarding_error_code(&err);
6064 assert!(
6065 !client_retries(code),
6066 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6067 );
6068 }
6069 }
6070
6071 #[tokio::test]
6099 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6100 let handler = ControlHandler::default();
6101 let frame =
6102 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6103
6104 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6108 assert_eq!(
6109 stamped,
6110 Principal::Direct,
6111 "a caller that proved nothing must not be stamped as a supervised module"
6112 );
6113
6114 let forged = handler
6119 .route_open_principal(
6120 &frame,
6121 Some(ConsumerIdentity {
6122 module_id: "aft".to_string(),
6123 launch_nonce: "not-a-real-nonce".to_string(),
6124 }),
6125 )
6126 .unwrap();
6127 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6128 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6129 }
6130
6131 #[test]
6148 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6149 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6150 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6151 let ClientControlRequest::RouteOpen {
6152 consumer_identity, ..
6153 } = parsed
6154 else {
6155 panic!("route.open body must parse as RouteOpen");
6156 };
6157 assert_eq!(
6158 consumer_identity,
6159 Some(ConsumerIdentity {
6160 module_id: "aft".to_string(),
6161 launch_nonce: "n".to_string(),
6162 }),
6163 "the wire field name must reach the value route_open_principal reads"
6164 );
6165 }
6166
6167 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6168 ModuleManifest::builder(module_id, "0.1.0")
6169 .protocol_ver(protocol_ver)
6170 .provides(vec![ProviderRole::ToolProvider {
6171 tools: vec![Tool {
6172 name: "read".to_string(),
6173 description: None,
6174 execution_mode: ExecutionMode::Pure,
6175 schema: json!({"type": "object"}),
6176 }],
6177 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6178 concurrency: Concurrency::ModuleManaged,
6179 emits_push: true,
6180 sub_supervises: true,
6181 }])
6182 .build()
6183 }
6184
6185 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6186 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6187 }
6188
6189 fn hello_frame_with_control_ops(
6190 module_id: &str,
6191 protocol_ver: u8,
6192 corr: u64,
6193 control_ops: Option<Vec<String>>,
6194 ) -> Frame {
6195 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6196 }
6197
6198 fn hello_frame_with_nonce(
6199 module_id: &str,
6200 protocol_ver: u8,
6201 corr: u64,
6202 launch_nonce: Option<&str>,
6203 ) -> Frame {
6204 hello_frame_full(
6205 module_id,
6206 protocol_ver,
6207 corr,
6208 None,
6209 launch_nonce.map(ToOwned::to_owned),
6210 )
6211 }
6212
6213 fn hello_frame_full(
6214 module_id: &str,
6215 protocol_ver: u8,
6216 corr: u64,
6217 control_ops: Option<Vec<String>>,
6218 launch_nonce: Option<String>,
6219 ) -> Frame {
6220 let body = serde_json::to_vec(&ModuleHelloBody {
6221 manifest: manifest(module_id, protocol_ver),
6222 protocol_ver,
6223 control_ops,
6224 launch_nonce,
6225 })
6226 .unwrap();
6227 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6228 }
6229
6230 fn non_routable_hello_frame_with_control_ops(
6231 module_id: &str,
6232 corr: u64,
6233 control_ops: Option<Vec<String>>,
6234 ) -> Frame {
6235 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6236 manifest.provides.clear();
6237 let body = serde_json::to_vec(&ModuleHelloBody {
6238 manifest,
6239 protocol_ver: PROTOCOL_VERSION,
6240 control_ops,
6241 launch_nonce: None,
6242 })
6243 .unwrap();
6244 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6245 }
6246
6247 fn capability_grammar_hello_frame(
6248 capabilities: Value,
6249 runtime_computed: Option<Value>,
6250 corr: u64,
6251 ) -> Frame {
6252 let mut body = serde_json::to_value(ModuleHelloBody {
6253 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6254 protocol_ver: PROTOCOL_VERSION,
6255 control_ops: None,
6256 launch_nonce: None,
6257 })
6258 .expect("HELLO body serializes");
6259 body["manifest"]["capabilities"] = capabilities;
6260 if let Some(runtime_computed) = runtime_computed {
6261 body["runtime_computed"] = runtime_computed;
6262 }
6263 Frame::build(
6264 FrameType::Hello,
6265 control_flags(),
6266 0,
6267 0,
6268 corr,
6269 serde_json::to_vec(&body).expect("HELLO body reserializes"),
6270 )
6271 .expect("HELLO frame builds")
6272 }
6273
6274 fn channel_request(channel: u16, corr: u64) -> Frame {
6275 Frame::build(
6276 FrameType::Request,
6277 Flags::new(true, Priority::Interactive, false),
6278 channel,
6279 0,
6280 corr,
6281 b"opaque".to_vec(),
6282 )
6283 .unwrap()
6284 }
6285
6286 fn route_ctx(
6287 connection_id: ConnectionId,
6288 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
6289 let (tx, rx) = mpsc::channel(8);
6290 (
6291 RouteCtx {
6292 connection_id,
6293 egress: FrameSink::new(tx),
6294 },
6295 rx,
6296 )
6297 }
6298
6299 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
6300 serde_json::from_slice(&frame.body).unwrap()
6301 }
6302
6303 async fn hello_via_sink(
6308 handler: &ControlHandler,
6309 ctx: &RouteCtx,
6310 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6311 hello: Frame,
6312 ) -> Frame {
6313 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
6314 assert!(
6315 replies.is_empty(),
6316 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
6317 );
6318 let ack = rx
6319 .try_recv()
6320 .expect("HELLO_ACK is queued on the module sink")
6321 .frame;
6322 assert_eq!(ack.header.ty, FrameType::HelloAck);
6323 ack
6324 }
6325
6326 fn parse_error(frame: &Frame) -> Value {
6327 serde_json::from_slice(&frame.body).unwrap()
6328 }
6329
6330 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
6331 serde_json::from_slice(&frame.body).unwrap()
6332 }
6333
6334 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
6335 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
6336 route_channel,
6337 route_epoch: 0,
6338 kind,
6339 })
6340 .unwrap();
6341 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6342 }
6343
6344 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
6345 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
6346 module_id: module_id.to_string(),
6347 })
6348 .unwrap();
6349 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6350 }
6351
6352 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
6353 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
6354 }
6355
6356 fn route_open_frame_with_consumer_capabilities(
6357 corr: u64,
6358 module_id: &str,
6359 project_root: TestTempDir,
6360 consumer_capabilities: Option<Vec<String>>,
6361 ) -> Frame {
6362 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6363 target: RouteTarget::ToolProvider {
6364 module_id: module_id.to_string(),
6365 },
6366 identity: BindIdentity::new(
6367 project_root.path().to_path_buf(),
6368 "unit".to_string(),
6369 "session".to_string(),
6370 ),
6371 consumer_identity: None,
6372 consumer_capabilities,
6373 admission_facts: None,
6374 scope: None,
6375 })
6376 .unwrap();
6377 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6378 }
6379
6380 fn route_open_frame_with_admission_facts(
6381 corr: u64,
6382 module_id: &str,
6383 project_root: TestTempDir,
6384 consumer_identity: Option<subc_control::ConsumerIdentity>,
6385 facts: Option<Value>,
6386 ) -> Frame {
6387 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6388 target: RouteTarget::ToolProvider {
6389 module_id: module_id.to_string(),
6390 },
6391 identity: BindIdentity::new(
6392 project_root.path().to_path_buf(),
6393 "unit".to_string(),
6394 format!("session-{corr}"),
6395 ),
6396 consumer_identity,
6397 consumer_capabilities: None,
6398 admission_facts: facts,
6399 scope: None,
6400 })
6401 .unwrap();
6402 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6403 }
6404
6405 #[derive(Clone, Default)]
6406 struct EventCapture {
6407 events: Arc<Mutex<Vec<CapturedEvent>>>,
6408 }
6409
6410 #[derive(Clone, Debug)]
6411 struct CapturedEvent {
6412 target: String,
6413 level: tracing::Level,
6414 fields: BTreeMap<String, String>,
6415 }
6416
6417 impl EventCapture {
6418 fn events(&self) -> Vec<CapturedEvent> {
6419 self.events.lock().unwrap().clone()
6420 }
6421 }
6422
6423 impl<S> Layer<S> for EventCapture
6424 where
6425 S: Subscriber,
6426 {
6427 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6428 let mut visitor = EventFieldVisitor::default();
6429 event.record(&mut visitor);
6430 self.events.lock().unwrap().push(CapturedEvent {
6431 target: event.metadata().target().to_string(),
6432 level: *event.metadata().level(),
6433 fields: visitor.fields,
6434 });
6435 }
6436 }
6437
6438 #[derive(Default)]
6439 struct EventFieldVisitor {
6440 fields: BTreeMap<String, String>,
6441 }
6442
6443 impl Visit for EventFieldVisitor {
6444 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6445 self.fields
6446 .insert(field.name().to_string(), format!("{value:?}"));
6447 }
6448 }
6449
6450 fn health_response(corr: u64, status: HealthStatus) -> Frame {
6451 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6452 status,
6453 detail: Some("warming".to_string()),
6454 metrics: Some(json!({"queue_depth": 3})),
6455 })
6456 .unwrap();
6457 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6458 }
6459
6460 fn route_bind_ack(corr: u64) -> Frame {
6461 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6462 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6463 }
6464
6465 fn unique_project_root(label: &str) -> TestTempDir {
6466 TestTempDir::new(label)
6467 }
6468
6469 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6470 match parse_route_poll(frame) {
6471 ClientControlResponse::RoutePoll {
6472 status: None,
6473 live: Some(live),
6474 ..
6475 } => assert_eq!(live, expected_live),
6476 other => panic!("unexpected route.poll response: {other:?}"),
6477 }
6478 }
6479
6480 fn bind_liveness_route(
6481 registry: &Registry,
6482 forwarding: &ForwardingTable,
6483 module_id: &str,
6484 ) -> (RouteCtx, u16, u32) {
6485 let module_connection = ConnectionId::new(101);
6486 let client_connection = ConnectionId::new(202);
6487 let registration = registry
6488 .register_with_control_ops(
6489 manifest(module_id, PROTOCOL_VERSION),
6490 PROTOCOL_VERSION,
6491 module_connection,
6492 module_baseline_control_ops(),
6493 )
6494 .unwrap();
6495 let (module_tx, _module_rx) = mpsc::channel(8);
6496 let endpoint = forwarding
6497 .register_module_connection(
6498 module_connection,
6499 module_id.to_string(),
6500 PROTOCOL_VERSION,
6501 manifest_concurrency(®istration.manifest),
6502 FrameSink::new(module_tx),
6503 )
6504 .unwrap();
6505 let (client_ctx, _client_rx) = route_ctx(client_connection);
6506 let pending = forwarding
6507 .begin_route_bind_relay_for_test(
6508 client_connection,
6509 client_ctx.egress.clone(),
6510 1,
6511 module_id,
6512 )
6513 .unwrap();
6514 assert_eq!(pending.endpoint, endpoint);
6515 let route_channel = pending.client_channel;
6516 let route_epoch = pending.client_epoch;
6517 forwarding
6518 .complete_pending_relay(
6519 module_connection,
6520 pending.corr,
6521 RouteBindRelayOutcome::Accepted,
6522 )
6523 .unwrap();
6524 (client_ctx, route_channel, route_epoch)
6525 }
6526
6527 struct FakeProcessLiveness {
6528 live: Option<bool>,
6529 }
6530
6531 impl ModuleProcessLiveness for FakeProcessLiveness {
6532 fn process_live(&self, _module_id: &str) -> Option<bool> {
6533 self.live
6534 }
6535 }
6536
6537 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6538 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6539 {
6540 let registry = Arc::new(Registry::default());
6541 let supervisor_handle = SupervisorHandle::new();
6542 let supervisor = Supervisor::new(
6543 Arc::clone(®istry),
6544 RestartPolicy::new(1, Duration::from_millis(10)),
6545 )
6546 .with_handle(supervisor_handle.clone());
6547 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6548 let module = supervisor
6549 .spawn(ModuleSpec {
6550 launch_nonce_env: true,
6551 module_id: "stderr-tail-wire".to_string(),
6552 program: fake_aft_stub_path(),
6553 args: Vec::new(),
6554 env: vec![
6555 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6556 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6557 ],
6558 reserved: false,
6559 reserved_prefixes: Vec::new(),
6560 protocol: ModuleProtocol::Subc,
6561 overlap: Default::default(),
6562 })
6563 .unwrap();
6564
6565 let deadline = Instant::now() + Duration::from_secs(5);
6566 loop {
6567 let tail = module.stderr_tail(None, None);
6568 if tail
6569 .entries
6570 .iter()
6571 .any(|entry| matches!(entry, TailEntry::ProcessStart))
6572 && tail.entries.iter().any(|entry| {
6573 matches!(
6574 entry,
6575 TailEntry::Line {
6576 truncated: true,
6577 ..
6578 }
6579 )
6580 })
6581 {
6582 break;
6583 }
6584 assert!(
6585 Instant::now() < deadline,
6586 "module did not produce a truncated line and restart boundary: {tail:?}"
6587 );
6588 sleep(Duration::from_millis(10)).await;
6589 }
6590
6591 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6592 let request = ClientControlRequest::SupervisorStderrTail {
6593 module_id: "stderr-tail-wire".to_string(),
6594 max_lines: None,
6595 max_bytes: None,
6596 };
6597 let frame = Frame::build(
6598 FrameType::Request,
6599 control_flags(),
6600 0,
6601 0,
6602 1,
6603 serde_json::to_vec(&request).unwrap(),
6604 )
6605 .unwrap();
6606 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6607 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6608 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6609 serde_json::from_slice(&responses[0].body).unwrap()
6610 else {
6611 panic!("expected supervisor.stderr_tail response");
6612 };
6613
6614 assert!(
6615 tail.entries
6616 .iter()
6617 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6618 "the control response lost the restart boundary"
6619 );
6620 let Some(StderrTailEntry::Line {
6621 text,
6622 truncated,
6623 at_ms,
6624 }) = tail.entries.iter().find(|entry| {
6625 matches!(
6626 entry,
6627 StderrTailEntry::Line {
6628 truncated: true,
6629 ..
6630 }
6631 )
6632 })
6633 else {
6634 panic!("the control response lost the truncated line");
6635 };
6636 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6637 assert!(*truncated);
6638 assert!(
6639 at_ms.is_some(),
6640 "the control response lost the line's capture time"
6641 );
6642 }
6643
6644 #[tokio::test(flavor = "current_thread")]
6649 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6650 let dir = TestTempDir::new("terminals-off-worker");
6651 let journal_path = dir.join("terminals.jsonl");
6652 let registry = Arc::new(Registry::default());
6653 let supervisor_handle = SupervisorHandle::new();
6654 let supervisor =
6655 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6656 .with_handle(supervisor_handle.clone())
6657 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6658 let module = supervisor
6659 .spawn(ModuleSpec {
6660 launch_nonce_env: true,
6661 module_id: "terminal-off-worker".to_string(),
6662 program: fake_aft_stub_path(),
6663 args: Vec::new(),
6664 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6665 reserved: false,
6666 reserved_prefixes: Vec::new(),
6667 protocol: ModuleProtocol::Subc,
6668 overlap: Default::default(),
6669 })
6670 .unwrap();
6671 let deadline = Instant::now() + Duration::from_secs(5);
6672 while module.terminal_history().entries.len() != 2 {
6673 assert!(Instant::now() < deadline, "module did not record two exits");
6674 sleep(Duration::from_millis(10)).await;
6675 }
6676
6677 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6678 let handler =
6679 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
6680 let frame = Frame::build(
6681 FrameType::Request,
6682 control_flags(),
6683 0,
6684 0,
6685 1,
6686 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6687 module_id: "terminal-off-worker".to_string(),
6688 })
6689 .unwrap(),
6690 )
6691 .unwrap();
6692 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6693 let spawned_at = std::time::Instant::now();
6694 let read = tokio::spawn({
6695 let handler = Arc::clone(&handler);
6696 async move { handler.handle_control_frame(&ctx, frame).await }
6697 });
6698 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6701 .await
6702 .unwrap()
6703 .expect("the history read reached its pause");
6704 let elapsed = spawned_at.elapsed();
6705 assert!(
6706 elapsed < Duration::from_secs(2) && !read.is_finished(),
6707 "this task could not run while the history read was paused \
6708 (resumed after {elapsed:?}, read finished: {})",
6709 read.is_finished()
6710 );
6711
6712 drop(release);
6713 let responses = read.await.unwrap().unwrap();
6714 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6715 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6716 panic!("expected supervisor.terminals response");
6717 };
6718 assert_eq!(terminals.entries.len(), 2);
6719 assert_eq!(terminals.journal_skipped_lines, 0);
6720 assert_eq!(terminals.journal_read_errors, 0);
6721 }
6722
6723 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6724 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6725 let registry = Arc::new(Registry::default());
6726 let supervisor_handle = SupervisorHandle::new();
6727 let supervisor =
6728 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6729 .with_handle(supervisor_handle.clone());
6730 let module = supervisor
6731 .spawn(ModuleSpec {
6732 launch_nonce_env: true,
6733 module_id: "terminal-golden".to_string(),
6734 program: fake_aft_stub_path(),
6735 args: Vec::new(),
6736 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6737 reserved: false,
6738 reserved_prefixes: Vec::new(),
6739 protocol: ModuleProtocol::Subc,
6740 overlap: Default::default(),
6741 })
6742 .unwrap();
6743
6744 let deadline = Instant::now() + Duration::from_secs(5);
6745 while module.terminal_history().entries.len() != 2 {
6746 assert!(
6747 Instant::now() < deadline,
6748 "module did not retain two terminal exits: {:?}",
6749 module.terminal_history()
6750 );
6751 sleep(Duration::from_millis(10)).await;
6752 }
6753
6754 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6755 let request = ClientControlRequest::SupervisorTerminals {
6756 module_id: "terminal-golden".to_string(),
6757 };
6758 let frame = Frame::build(
6759 FrameType::Request,
6760 control_flags(),
6761 0,
6762 0,
6763 1,
6764 serde_json::to_vec(&request).unwrap(),
6765 )
6766 .unwrap();
6767 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6768 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6769 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6770 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6771 panic!("expected supervisor.terminals response");
6772 };
6773 assert_eq!(terminals.entries.len(), 2);
6774 assert_eq!(terminals.dropped, 0);
6775
6776 let mut rendered = serde_json::to_value(response).unwrap();
6777 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6780 for (index, entry) in rendered["entries"]
6781 .as_array_mut()
6782 .expect("terminal response entries array")
6783 .iter_mut()
6784 .enumerate()
6785 {
6786 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6787 }
6788
6789 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6790 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6791 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6792 if std::env::var_os("UPDATE_GOLDEN").is_some() {
6793 std::fs::write(&golden_path, &serialized).unwrap();
6794 }
6795 let expected: Value =
6796 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6797 assert_eq!(rendered, expected);
6798 }
6799
6800 #[test]
6801 fn hello_registers_manifest_and_returns_ack() {
6802 let registry = Arc::new(Registry::default());
6803 let handler = ControlHandler::new(Arc::clone(®istry));
6804 let conn = ConnectionId::new(1);
6805
6806 let responses = handler
6807 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6808 .unwrap();
6809
6810 assert_eq!(responses.len(), 1);
6811 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6812 assert_eq!(responses[0].header.channel, 0);
6813 assert_eq!(responses[0].header.corr, 7);
6814 let ack = parse_ack(&responses[0]);
6815 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6816 assert!(ack
6817 .subc_capabilities
6818 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6819 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6820 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6821 assert!(ack
6822 .subc_ops
6823 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6824 assert!(ack
6825 .subc_ops
6826 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6827
6828 let registration = registry.get_module("aft").unwrap().unwrap();
6829 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6830 assert_eq!(registration.state, ChannelState::Active);
6831 assert_eq!(registration.connection_id, conn);
6832 assert_eq!(registration.control_ops, module_baseline_control_ops());
6833 }
6834
6835 #[test]
6836 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
6837 let invalid_identifiers = [
6838 ("case_change", "credentials-Provider/v1"),
6839 ("leading_zero", "credentials-provider/v01"),
6840 ("trailing_hyphen", "credentials-provider-/v1"),
6841 ("consecutive_hyphens", "credentials--provider/v1"),
6842 ("uppercase", "Credentials-provider/v1"),
6843 ("missing_v", "credentials-provider/1"),
6844 ("whitespace", "credentials provider/v1"),
6845 ("zero_version", "credentials-provider/v0"),
6846 ("out_of_range_version", "credentials-provider/v4294967296"),
6847 (
6848 "overlength_name",
6849 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
6850 ),
6851 ];
6852 let mut cases = invalid_identifiers
6853 .into_iter()
6854 .map(|(name, identifier)| {
6855 (
6856 format!("identifier_{name}"),
6857 "capabilities.provides[0]".to_string(),
6858 identifier.to_string(),
6859 json!({ "provides": [identifier] }),
6860 None,
6861 )
6862 })
6863 .collect::<Vec<_>>();
6864 cases.extend([
6865 (
6866 "unknown_need".to_string(),
6867 "capabilities.requires[0].need".to_string(),
6868 "deferred".to_string(),
6869 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
6870 None,
6871 ),
6872 (
6873 "duplicate_provides".to_string(),
6874 "capabilities.provides[1]".to_string(),
6875 "credentials-provider/v1".to_string(),
6876 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
6877 None,
6878 ),
6879 (
6880 "duplicate_must_never_reach".to_string(),
6881 "capabilities.must_never_reach[1]".to_string(),
6882 "credentials-provider/v1".to_string(),
6883 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
6884 None,
6885 ),
6886 (
6887 "duplicate_requires_same_need".to_string(),
6888 "capabilities.requires[1]".to_string(),
6889 "credentials-provider/v1".to_string(),
6890 json!({ "requires": [
6891 { "capability": "credentials-provider/v1", "need": "required" },
6892 { "capability": "credentials-provider/v1", "need": "required" }
6893 ] }),
6894 None,
6895 ),
6896 (
6897 "duplicate_requires_conflicting_need".to_string(),
6898 "capabilities.requires[1]".to_string(),
6899 "credentials-provider/v1".to_string(),
6900 json!({ "requires": [
6901 { "capability": "credentials-provider/v1", "need": "required" },
6902 { "capability": "credentials-provider/v1", "need": "optional" }
6903 ] }),
6904 None,
6905 ),
6906 (
6907 "capabilities_root_pointer".to_string(),
6908 "runtime_computed[0]".to_string(),
6909 "/capabilities".to_string(),
6910 json!({}),
6911 Some(json!(["/capabilities"])),
6912 ),
6913 (
6914 "capabilities_descendant_pointer".to_string(),
6915 "runtime_computed[0]".to_string(),
6916 "/capabilities/provides".to_string(),
6917 json!({}),
6918 Some(json!(["/capabilities/provides"])),
6919 ),
6920 (
6921 "malformed_pointer_without_leading_slash".to_string(),
6922 "runtime_computed[0]".to_string(),
6923 "capabilities".to_string(),
6924 json!({}),
6925 Some(json!(["capabilities"])),
6926 ),
6927 (
6928 "malformed_pointer_escape".to_string(),
6929 "runtime_computed[0]".to_string(),
6930 "/roles/~2/tools".to_string(),
6931 json!({}),
6932 Some(json!(["/roles/~2/tools"])),
6933 ),
6934 (
6935 "unknown_capabilities_field".to_string(),
6936 "capabilities.future".to_string(),
6937 "<array>".to_string(),
6938 json!({ "future": [] }),
6939 None,
6940 ),
6941 ]);
6942
6943 for (index, (name, field, value, capabilities, runtime_computed)) in
6944 cases.into_iter().enumerate()
6945 {
6946 let registry = Arc::new(Registry::default());
6947 let handler = ControlHandler::new(Arc::clone(®istry));
6948 let response = handler
6949 .handle_control(
6950 ConnectionId::new((index + 1) as u64),
6951 capability_grammar_hello_frame(
6952 capabilities,
6953 runtime_computed,
6954 index as u64 + 1,
6955 ),
6956 )
6957 .expect("invalid HELLO returns a refusal");
6958
6959 assert_eq!(response.len(), 1, "{name} must emit one refusal");
6960 let error = parse_error(&response[0]);
6961 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
6962 let message = error["message"]
6963 .as_str()
6964 .expect("error message is a string");
6965 assert!(
6966 message.contains(&field),
6967 "{name}: field missing from {message}"
6968 );
6969 assert!(
6970 message.contains(&value),
6971 "{name}: value missing from {message}"
6972 );
6973 assert_eq!(
6974 registry
6975 .active_registration_count()
6976 .expect("registry reads"),
6977 0,
6978 "{name}: refused HELLO must not create a catalog entry"
6979 );
6980 }
6981 }
6982
6983 #[test]
6984 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
6985 let registry = Arc::new(Registry::default());
6986 let handler = ControlHandler::new(Arc::clone(®istry));
6987 let capabilities = json!({
6988 "provides": ["credentials-provider/v1"],
6989 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
6990 "must_never_reach": ["federation-transport/v1"]
6991 });
6992 let response = handler
6993 .handle_control(
6994 ConnectionId::new(99),
6995 capability_grammar_hello_frame(
6996 capabilities.clone(),
6997 Some(json!(["/roles/0/tools"])),
6998 99,
6999 ),
7000 )
7001 .expect("valid HELLO registers");
7002 assert_eq!(response[0].header.ty, FrameType::HelloAck);
7003
7004 let request = Frame::build(
7005 FrameType::Request,
7006 control_flags(),
7007 0,
7008 0,
7009 100,
7010 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7011 .expect("catalog request serializes"),
7012 )
7013 .expect("catalog request frame builds");
7014 let response = handler
7015 .handle_catalog_list(request, None)
7016 .expect("catalog list succeeds");
7017 let ClientControlResponse::CatalogList { modules, .. } =
7018 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7019 else {
7020 panic!("catalog request must return catalog.list");
7021 };
7022 assert_eq!(modules.len(), 1);
7023 assert_eq!(
7024 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7025 capabilities
7026 );
7027 }
7028
7029 #[test]
7030 fn catalog_list_mirrors_management_operation_description() {
7031 let registry = Arc::new(Registry::default());
7032 let handler = ControlHandler::new(Arc::clone(®istry));
7033 let description = "List managed records and return their identifiers and metadata.";
7034 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7035 manifest.provides = vec![ProviderRole::ManagementSurface {
7036 operations: vec![ManagementOperation {
7037 name: "records.list".to_string(),
7038 kind: ManagementOperationKind::Query,
7039 description: Some(description.to_string()),
7040 }],
7041 config_schema: json!({"type": "object"}),
7042 observability: vec![ObservabilitySurface {
7043 name: "records.stats".to_string(),
7044 kind: ObservabilityKind::Snapshot,
7045 }],
7046 identity_scope: vec![IdentityScope::Project],
7047 concurrency: Concurrency::ModuleManaged,
7048 }];
7049 registry
7050 .register_with_control_ops(
7051 manifest,
7052 PROTOCOL_VERSION,
7053 ConnectionId::new(99),
7054 Vec::new(),
7055 )
7056 .expect("described management manifest registers");
7057
7058 let request = Frame::build(
7059 FrameType::Request,
7060 control_flags(),
7061 0,
7062 0,
7063 100,
7064 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7065 .expect("catalog request serializes"),
7066 )
7067 .expect("catalog request frame builds");
7068 let response = handler
7069 .handle_catalog_list(request, None)
7070 .expect("catalog list succeeds");
7071 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7072 assert_eq!(
7073 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7074 "catalog.list must preserve the declared operation description verbatim"
7075 );
7076 }
7077
7078 #[test]
7079 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7080 let registry = Arc::new(Registry::default());
7081 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7082 [("vault".to_string(), true), ("squatter".to_string(), true)],
7083 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7084 );
7085 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7086 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7087 provides: vec!["credentials-provider/v1".to_string()],
7088 requires: Vec::new(),
7089 must_never_reach: Vec::new(),
7090 });
7091 let frame = Frame::build(
7092 FrameType::Hello,
7093 control_flags(),
7094 0,
7095 0,
7096 77,
7097 serde_json::to_vec(&ModuleHelloBody {
7098 manifest: squatter,
7099 protocol_ver: PROTOCOL_VERSION,
7100 control_ops: None,
7101 launch_nonce: None,
7102 })
7103 .expect("HELLO serializes"),
7104 )
7105 .expect("HELLO frame builds");
7106 let response = handler
7107 .handle_control(ConnectionId::new(77), frame)
7108 .expect("reserved claim receives a typed refusal");
7109 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7110 assert_eq!(
7111 registry
7112 .active_registration_count()
7113 .expect("registry reads"),
7114 0,
7115 "a reserved capability refusal must not leave a catalog entry"
7116 );
7117 }
7118
7119 #[test]
7120 fn server_describe_surfaces_required_capability_verdict_fields() {
7121 let registry = Arc::new(Registry::default());
7122 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7123 [
7124 ("consumer".to_string(), true),
7125 ("provider".to_string(), false),
7126 ],
7127 BTreeMap::new(),
7128 );
7129 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
7130 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7131 provides: Vec::new(),
7132 requires: vec![subc_protocol::manifest::CapabilityRequirement {
7133 capability: "credentials-provider/v1".to_string(),
7134 need: subc_protocol::manifest::CapabilityNeed::Required,
7135 }],
7136 must_never_reach: Vec::new(),
7137 });
7138 let hello = Frame::build(
7139 FrameType::Hello,
7140 control_flags(),
7141 0,
7142 0,
7143 78,
7144 serde_json::to_vec(&ModuleHelloBody {
7145 manifest: consumer,
7146 protocol_ver: PROTOCOL_VERSION,
7147 control_ops: None,
7148 launch_nonce: None,
7149 })
7150 .expect("HELLO serializes"),
7151 )
7152 .expect("HELLO frame builds");
7153 handler
7154 .handle_control(ConnectionId::new(78), hello)
7155 .expect("consumer registers");
7156 let describe = Frame::build(
7157 FrameType::Request,
7158 control_flags(),
7159 0,
7160 0,
7161 79,
7162 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
7163 .expect("request serializes"),
7164 )
7165 .expect("describe frame builds");
7166 let response = handler
7167 .handle_server_describe(describe)
7168 .expect("server.describe succeeds");
7169 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
7170 let requirement = &rendered["capability_requirements"][0];
7171 assert_eq!(requirement["consumer"], "consumer");
7172 assert_eq!(requirement["verdict"], "never_provided");
7173 assert_eq!(requirement["episode_seq"], 1);
7174 assert_eq!(requirement["config_satisfiable"], false);
7175 assert_eq!(requirement["runtime_available"], false);
7176 assert!(requirement["detail"]
7177 .as_str()
7178 .expect("detail string")
7179 .contains("credentials-provider/v1"));
7180 }
7181
7182 #[test]
7183 fn catalog_list_omits_capabilities_for_legacy_manifest() {
7184 let registry = Arc::new(Registry::default());
7185 let handler = ControlHandler::new(Arc::clone(®istry));
7186 let hello = handler
7187 .handle_control(
7188 ConnectionId::new(101),
7189 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
7190 )
7191 .expect("legacy HELLO registers");
7192 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
7193
7194 let request = Frame::build(
7195 FrameType::Request,
7196 control_flags(),
7197 0,
7198 0,
7199 102,
7200 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7201 .expect("catalog request serializes"),
7202 )
7203 .expect("catalog request frame builds");
7204 let response = handler
7205 .handle_catalog_list(request, None)
7206 .expect("catalog list succeeds");
7207 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7208 assert!(
7209 body["modules"][0].get("capabilities").is_none(),
7210 "legacy manifest must retain an absent capabilities field on catalog.list"
7211 );
7212 }
7213
7214 #[test]
7215 fn hello_ack_omits_storage_when_no_storage_config() {
7216 let registry = Arc::new(Registry::default());
7217 let handler = ControlHandler::new(Arc::clone(®istry));
7218 let responses = handler
7219 .handle_control(
7220 ConnectionId::new(1),
7221 hello_frame("aft", PROTOCOL_VERSION, 7),
7222 )
7223 .unwrap();
7224 let ack = parse_ack(&responses[0]);
7225 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
7226 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
7227 }
7228
7229 #[tokio::test]
7230 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
7231 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
7232 let registry = Arc::new(Registry::default());
7233 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
7234 let responses = handler
7235 .handle_control(
7236 ConnectionId::new(1),
7237 hello_frame("aft", PROTOCOL_VERSION, 7),
7238 )
7239 .unwrap();
7240 let ack = parse_ack(&responses[0]);
7241 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
7242
7243 let described = handler
7244 .handle_control_frame(
7245 &route_ctx(ConnectionId::new(2)).0,
7246 Frame::build(
7247 FrameType::Request,
7248 control_flags(),
7249 0,
7250 0,
7251 9,
7252 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
7253 )
7254 .unwrap(),
7255 )
7256 .await
7257 .unwrap();
7258 let ClientControlResponse::ServerDescribe { machine_id, .. } =
7259 serde_json::from_slice(&described[0].body).unwrap()
7260 else {
7261 panic!("server.describe answered with another shape");
7262 };
7263 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
7264 }
7265
7266 #[test]
7267 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
7268 let registry = Arc::new(Registry::default());
7271 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
7272 crate::daemon_config::StorageConfig::Sqlite {
7273 data_home: std::path::PathBuf::from("/data"),
7274 },
7275 ));
7276
7277 let responses = handler
7278 .handle_control(
7279 ConnectionId::new(1),
7280 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
7281 )
7282 .unwrap();
7283 let ack = parse_ack(&responses[0]);
7284 assert_eq!(
7285 ack.storage,
7286 Some(serde_json::json!({
7287 "module_id": "alfonso-routing",
7288 "storage_namespace": "default",
7289 "isolation": { "kind": "module" },
7290 "backend": {
7291 "backend": "sqlite",
7292 "path": "/data/cortexkit/alfonso-routing/store.db"
7293 }
7294 })),
7295 "the delivered descriptor is the module's own sqlite store path"
7296 );
7297 }
7298
7299 #[test]
7300 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
7301 let registry = Arc::new(Registry::default());
7302 let handler = ControlHandler::new(Arc::clone(®istry));
7303 let conn = ConnectionId::new(1);
7304 let responses = handler
7305 .handle_control(
7306 conn,
7307 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7308 )
7309 .unwrap();
7310 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7311 let registration = registry.get_module("aft").unwrap().unwrap();
7312 assert_eq!(registration.control_ops, module_baseline_control_ops());
7313
7314 let frame =
7315 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
7316 assert!(handler
7317 .guard_module_control_op(&frame, "aft", "route.bind")
7318 .unwrap()
7319 .is_none());
7320 let error = handler
7321 .guard_module_control_op(&frame, "aft", "test.synthetic")
7322 .unwrap()
7323 .expect("synthetic ungranted op should be rejected");
7324 assert_eq!(error.header.ty, FrameType::Error);
7325 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
7326 }
7327
7328 #[test]
7329 fn hello_control_ops_some_adds_optional_grants() {
7330 let registry = Arc::new(Registry::default());
7331 let handler = ControlHandler::new(Arc::clone(®istry));
7332 handler
7333 .handle_control(
7334 ConnectionId::new(1),
7335 hello_frame_with_control_ops(
7336 "aft",
7337 PROTOCOL_VERSION,
7338 7,
7339 Some(vec![
7340 "future.synthetic".to_string(),
7341 "route.bind".to_string(),
7342 ]),
7343 ),
7344 )
7345 .unwrap();
7346 let registration = registry.get_module("aft").unwrap().unwrap();
7347 assert_eq!(
7348 registration.control_ops,
7349 vec![
7350 "route.bind".to_string(),
7351 "route.status".to_string(),
7352 "future.synthetic".to_string(),
7353 ]
7354 );
7355 let frame =
7356 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
7357 assert!(handler
7358 .guard_module_control_op(&frame, "aft", "future.synthetic")
7359 .unwrap()
7360 .is_none());
7361 }
7362
7363 #[tokio::test]
7364 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
7365 let registry = Arc::new(Registry::default());
7366 let forwarding = Arc::new(ForwardingTable::default());
7367 let handler =
7368 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7369 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
7370 hello_via_sink(
7371 &handler,
7372 &module_ctx,
7373 &mut module_rx,
7374 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7375 )
7376 .await;
7377
7378 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
7379 let responses = handler
7380 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
7381 .await
7382 .unwrap();
7383 assert_eq!(responses.len(), 1);
7384 assert_eq!(responses[0].header.ty, FrameType::Error);
7385 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
7386 assert!(module_rx.try_recv().is_err());
7387 }
7388
7389 #[tokio::test]
7390 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
7391 let registry = Arc::new(Registry::default());
7392 let forwarding = Arc::new(ForwardingTable::default());
7393 let handler =
7394 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7395 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
7396 hello_via_sink(
7397 &handler,
7398 &module_ctx,
7399 &mut module_rx,
7400 hello_frame_with_control_ops(
7401 "aft",
7402 PROTOCOL_VERSION,
7403 7,
7404 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
7405 ),
7406 )
7407 .await;
7408
7409 let project_root = unique_project_root("demux");
7410 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
7411 let route_handler = handler.clone();
7412 let route_task = tokio::spawn(async move {
7413 route_handler
7414 .handle_control_frame(
7415 &route_client_ctx,
7416 route_open_frame(100, "aft", project_root),
7417 )
7418 .await
7419 .unwrap()
7420 });
7421 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7422 .await
7423 .unwrap()
7424 .unwrap();
7425 assert!(matches!(
7426 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
7427 ModuleControlRequest::RouteBind { .. }
7428 ));
7429
7430 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
7431 let health_handler = handler.clone();
7432 let health_task = tokio::spawn(async move {
7433 health_handler
7434 .handle_control_frame(
7435 &health_client_ctx,
7436 supervisor_health_probe_frame(101, "aft"),
7437 )
7438 .await
7439 .unwrap()
7440 });
7441 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7442 .await
7443 .unwrap()
7444 .unwrap();
7445 assert_eq!(
7446 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
7447 ModuleControlRequest::HealthCheck {}
7448 );
7449
7450 handler
7451 .handle_control_frame(
7452 &module_ctx,
7453 health_response(health_frame.header.corr, HealthStatus::Degraded),
7454 )
7455 .await
7456 .unwrap();
7457 let health_response = health_task.await.unwrap();
7458 assert_eq!(health_response.len(), 1);
7459 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
7460 ClientControlResponse::SupervisorHealthProbe {
7461 module_id,
7462 status,
7463 detail,
7464 metrics,
7465 } => {
7466 assert_eq!(module_id, "aft");
7467 assert_eq!(status, HealthStatus::Degraded);
7468 assert_eq!(detail.as_deref(), Some("warming"));
7469 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
7470 }
7471 other => panic!("unexpected health response: {other:?}"),
7472 }
7473
7474 handler
7475 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7476 .await
7477 .unwrap();
7478 let route_response = route_task.await.unwrap();
7479 assert!(route_response.is_empty());
7480 let published = route_client_rx.recv().await.unwrap();
7481 assert!(matches!(
7482 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7483 ClientControlResponse::RouteOpen { .. }
7484 ));
7485 }
7486
7487 async fn relay_route_open(
7492 handler: &ControlHandler,
7493 client_connection: ConnectionId,
7494 client_egress: &FrameSink,
7495 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7496 corr: u64,
7497 module_id: &str,
7498 project_root_label: &str,
7499 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7500 let ctx = RouteCtx {
7501 connection_id: client_connection,
7502 egress: client_egress.clone(),
7503 };
7504 let handler = handler.clone();
7505 let project_root = unique_project_root(project_root_label);
7506 let module_id = module_id.to_string();
7507 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7508 let task = tokio::spawn(async move {
7509 let _guard = tracing::dispatcher::set_default(&dispatch);
7510 handler
7511 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
7512 .await
7513 .unwrap()
7514 });
7515 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
7516 .await
7517 .expect("module receives the relayed route.bind")
7518 .expect("module egress is open");
7519 (task, bind.frame)
7520 }
7521
7522 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
7523 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7524 ModuleControlRequest::RouteBind {
7525 route_channel,
7526 epoch,
7527 ..
7528 } => (route_channel, epoch),
7529 other => panic!("expected a route.bind request, got {other:?}"),
7530 }
7531 }
7532
7533 fn published_route(frame: &Frame) -> (u16, u32) {
7534 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7535 ClientControlResponse::RouteOpen {
7536 route_channel,
7537 route_epoch,
7538 } => (route_channel, route_epoch),
7539 other => panic!("expected a route.open response, got {other:?}"),
7540 }
7541 }
7542
7543 #[tokio::test]
7563 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7564 let registry = Arc::new(Registry::default());
7565 let forwarding = Arc::new(ForwardingTable::default());
7566 let handler =
7567 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7568
7569 let module_connection = ConnectionId::new(30);
7570 let (module_ctx, mut module_rx) = route_ctx(module_connection);
7571 hello_via_sink(
7572 &handler,
7573 &module_ctx,
7574 &mut module_rx,
7575 hello_frame("aft", PROTOCOL_VERSION, 7),
7576 )
7577 .await;
7578
7579 let dying_client = ConnectionId::new(31);
7580 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7581
7582 let (first_task, first_bind) = relay_route_open(
7585 &handler,
7586 dying_client,
7587 &dying_ctx.egress,
7588 &mut module_rx,
7589 100,
7590 "aft",
7591 "closing-first",
7592 )
7593 .await;
7594 handler
7595 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7596 .await
7597 .unwrap();
7598 assert!(first_task.await.unwrap().is_empty());
7599 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7600
7601 let (second_task, second_bind) = relay_route_open(
7603 &handler,
7604 dying_client,
7605 &dying_ctx.egress,
7606 &mut module_rx,
7607 101,
7608 "aft",
7609 "closing-second",
7610 )
7611 .await;
7612 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7613
7614 assert!(forwarding
7617 .escalate_client_delivery_failure(
7618 dying_client,
7619 first_channel,
7620 first_epoch,
7621 CloseReason::new(
7622 "module_to_client_delivery_failed",
7623 "client egress refused a module frame",
7624 ),
7625 crate::forwarding::UndeliveredFrame {
7626 module_id: None,
7627 sink: &dying_ctx.egress,
7628 },
7629 )
7630 .unwrap());
7631 assert!(!dying_ctx.egress.is_closed());
7632
7633 let ack = handler
7635 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7636 .await;
7637 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7638 if module_loop_error.is_some() {
7639 handler.cleanup_connection(module_connection).unwrap();
7643 }
7644 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7648 .await
7649 .ok()
7650 .flatten();
7651
7652 assert!(
7654 registry
7655 .get_module_by_connection(module_connection)
7656 .unwrap()
7657 .is_some(),
7658 "one client's closing connection ended the shared module connection: \
7659 {module_loop_error:?}"
7660 );
7661 let cotenant = ConnectionId::new(32);
7663 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7664 let (cotenant_task, cotenant_bind) = relay_route_open(
7665 &handler,
7666 cotenant,
7667 &cotenant_ctx.egress,
7668 &mut module_rx,
7669 102,
7670 "aft",
7671 "closing-cotenant",
7672 )
7673 .await;
7674 handler
7675 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7676 .await
7677 .unwrap();
7678 assert!(cotenant_task.await.unwrap().is_empty());
7679 let (cotenant_channel, cotenant_epoch) =
7680 published_route(&cotenant_rx.recv().await.unwrap());
7681 assert!(matches!(
7682 forwarding
7683 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7684 .unwrap(),
7685 DataRoute::Client(DataRouteState::Bound(_))
7686 ));
7687
7688 let goodbye = post_ack_module_frame
7691 .expect("module receives a GOODBYE for the abandoned route channel");
7692 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7693 assert_eq!(goodbye.header.channel, abandoned_channel);
7694 assert_eq!(goodbye.header.epoch, abandoned_epoch);
7695
7696 assert!(dying_rx.try_recv().is_err());
7700 let second_response = second_task.await.unwrap();
7701 assert_eq!(second_response.len(), 1);
7702 assert_eq!(
7703 parse_error(&second_response[0])["code"],
7704 "target_unavailable"
7705 );
7706 }
7707
7708 #[test]
7715 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7716 let handler = ControlHandler::default();
7717 let module_connection = ConnectionId::new(30);
7718 let client_connection = ConnectionId::new(31);
7719
7720 handler
7721 .refuse_to_end_module_connection_for_a_client(
7722 module_connection,
7723 77,
7724 ForwardingError::ConnectionClosing {
7725 connection_id: client_connection,
7726 },
7727 )
7728 .expect("a closing client must never end the module connection");
7729
7730 assert!(matches!(
7731 handler.refuse_to_end_module_connection_for_a_client(
7732 module_connection,
7733 78,
7734 ForwardingError::ConnectionClosing {
7735 connection_id: module_connection,
7736 },
7737 ),
7738 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7739 connection_id
7740 })) if connection_id == module_connection
7741 ));
7742 assert!(matches!(
7743 handler.refuse_to_end_module_connection_for_a_client(
7744 module_connection,
7745 79,
7746 ForwardingError::Poisoned,
7747 ),
7748 Err(RouterError::Forwarding(ForwardingError::Poisoned))
7749 ));
7750 assert!(matches!(
7751 handler.refuse_to_end_module_connection_for_a_client(
7752 module_connection,
7753 80,
7754 ForwardingError::StaleModuleEndpoint,
7755 ),
7756 Err(RouterError::Forwarding(
7757 ForwardingError::StaleModuleEndpoint
7758 ))
7759 ));
7760 }
7761
7762 #[tokio::test]
7775 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7776 let registry = Arc::new(Registry::default());
7777 let forwarding = Arc::new(ForwardingTable::default());
7778 let supervisor = SupervisorHandle::new();
7779 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7780 let handler =
7781 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7782 .with_supervisor(supervisor);
7783
7784 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7785 hello_via_sink(
7786 &handler,
7787 &target_ctx,
7788 &mut target_rx,
7789 hello_frame("target", PROTOCOL_VERSION, 1),
7790 )
7791 .await;
7792
7793 let wrong_nonce = handler
7797 .handle_control_frame(
7798 &route_ctx(ConnectionId::new(91)).0,
7799 route_open_frame_with_admission_facts(
7800 20,
7801 "target",
7802 unique_project_root("admission-facts"),
7803 Some(subc_control::ConsumerIdentity {
7804 module_id: "fed".to_string(),
7805 launch_nonce: "not-the-real-nonce".to_string(),
7806 }),
7807 None,
7808 ),
7809 )
7810 .await
7811 .unwrap();
7812 assert_eq!(
7813 parse_error(&wrong_nonce[0])["code"],
7814 "bad_consumer_identity",
7815 "a mismatched launch nonce must be refused, not stamped Reserved"
7816 );
7817
7818 let never_spawned = handler
7822 .handle_control_frame(
7823 &route_ctx(ConnectionId::new(92)).0,
7824 route_open_frame_with_admission_facts(
7825 21,
7826 "target",
7827 unique_project_root("admission-facts"),
7828 Some(subc_control::ConsumerIdentity {
7829 module_id: "never-spawned".to_string(),
7830 launch_nonce: "any-nonce".to_string(),
7831 }),
7832 None,
7833 ),
7834 )
7835 .await
7836 .unwrap();
7837 assert_eq!(
7838 parse_error(&never_spawned[0])["code"],
7839 "bad_consumer_identity",
7840 "an unspawned module_id must be refused rather than accepted for lack of a record"
7841 );
7842 }
7843
7844 #[tokio::test]
7867 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
7868 let registry = Arc::new(Registry::default());
7869 let forwarding = Arc::new(ForwardingTable::default());
7870 let supervisor = SupervisorHandle::new();
7871 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7872 let handler =
7873 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7874 .with_supervisor(supervisor);
7875
7876 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
7877 hello_via_sink(
7878 &handler,
7879 &target_ctx,
7880 &mut target_rx,
7881 hello_frame("target", PROTOCOL_VERSION, 1),
7882 )
7883 .await;
7884
7885 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
7886 let route_handler = handler.clone();
7887 let route_task = tokio::spawn(async move {
7888 route_handler
7889 .handle_control_frame(
7890 &client_ctx,
7891 route_open_frame_with_admission_facts(
7892 30,
7893 "target",
7894 unique_project_root("admission-facts"),
7895 Some(subc_control::ConsumerIdentity {
7896 module_id: "fed".to_string(),
7897 launch_nonce: "fed-nonce".to_string(),
7898 }),
7899 None,
7900 ),
7901 )
7902 .await
7903 .unwrap()
7904 });
7905
7906 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7913 .await
7914 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
7915 .expect("module control channel closed before route.bind");
7916 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7917 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
7918 panic!("expected route.bind")
7919 };
7920 assert_eq!(
7921 principal,
7922 Some(Principal::Reserved {
7923 module_id: "fed".to_string()
7924 }),
7925 "a correctly attested consumer must be stamped Reserved for its own id"
7926 );
7927
7928 handler
7929 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
7930 .await
7931 .unwrap();
7932 assert!(route_task.await.unwrap().is_empty());
7933 assert!(
7934 matches!(
7935 serde_json::from_slice::<ClientControlResponse>(
7936 &client_rx.recv().await.unwrap().body
7937 )
7938 .unwrap(),
7939 ClientControlResponse::RouteOpen { .. }
7940 ),
7941 "the route must actually open, not merely avoid an error"
7942 );
7943 }
7944
7945 #[tokio::test(start_paused = true)]
7946 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
7947 let registry = Arc::new(Registry::default());
7948 let forwarding = Arc::new(ForwardingTable::default());
7949 let supervisor = SupervisorHandle::new();
7950 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7951 let handler =
7952 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7953 .with_supervisor(supervisor);
7954
7955 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
7956 hello_via_sink(
7957 &handler,
7958 &target_ctx,
7959 &mut target_rx,
7960 hello_frame("target", PROTOCOL_VERSION, 1),
7961 )
7962 .await;
7963
7964 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
7965 let direct_handler = handler.clone();
7966 let direct_open = tokio::spawn(async move {
7967 direct_handler
7968 .handle_control_frame(
7969 &direct_ctx,
7970 route_open_frame(2, "target", unique_project_root("route-census-direct")),
7971 )
7972 .await
7973 .unwrap()
7974 });
7975 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7976 .await
7977 .expect("no direct route.bind within 5s")
7978 .expect("target control channel closed before direct route.bind");
7979 handler
7980 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
7981 .await
7982 .unwrap();
7983 assert!(direct_open.await.unwrap().is_empty());
7984 let _ = direct_rx.recv().await.unwrap();
7985
7986 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
7987 let reserved_handler = handler.clone();
7988 let reserved_open = tokio::spawn(async move {
7989 reserved_handler
7990 .handle_control_frame(
7991 &reserved_ctx,
7992 route_open_frame_with_admission_facts(
7993 3,
7994 "target",
7995 unique_project_root("admission-facts"),
7996 Some(ConsumerIdentity {
7997 module_id: "fed".to_string(),
7998 launch_nonce: "fed-nonce".to_string(),
7999 }),
8000 None,
8001 ),
8002 )
8003 .await
8004 .unwrap()
8005 });
8006 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8007 .await
8008 .expect("no reserved route.bind within 5s")
8009 .expect("target control channel closed before reserved route.bind");
8010 handler
8011 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
8012 .await
8013 .unwrap();
8014 assert!(reserved_open.await.unwrap().is_empty());
8015 let _ = reserved_rx.recv().await.unwrap();
8016
8017 forwarding
8018 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
8019 .unwrap();
8020 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
8021 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
8022 module_id: Some("target".to_string()),
8023 })
8024 .unwrap();
8025 let census_frame =
8026 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
8027 let response = handler
8028 .handle_control_frame(&census_ctx, census_frame)
8029 .await
8030 .unwrap()
8031 .pop()
8032 .unwrap();
8033 let actual: Value = serde_json::from_slice(&response.body).unwrap();
8034 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
8035 assert!(matches!(
8036 decoded,
8037 ClientControlResponse::SupervisorRoutes { .. }
8038 ));
8039 let routes = actual["modules"][0]["routes"].as_array().unwrap();
8040 assert_eq!(routes.len(), 2);
8041 assert!(routes.iter().all(|route| route["draining"] == true));
8042 assert!(
8045 routes.iter().all(|route| route["drain_reason"] == "reload"),
8046 "draining routes must name the drain's reason: {routes:?}"
8047 );
8048 assert!(routes.iter().any(|route| {
8049 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
8050 }));
8051 assert!(routes.iter().any(|route| {
8052 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
8053 }));
8054
8055 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8056 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
8057 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8058 std::fs::write(
8059 &golden_path,
8060 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
8061 )
8062 .unwrap();
8063 }
8064 let expected: Value =
8065 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
8066 assert_eq!(actual, expected);
8067 }
8068
8069 async fn query_live_roots(
8070 handler: &ControlHandler,
8071 module_ctx: &RouteCtx,
8072 ) -> ModuleControlResponseToModule {
8073 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
8074 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
8075 let response = handler
8076 .handle_control_frame(module_ctx, frame)
8077 .await
8078 .unwrap()
8079 .pop()
8080 .unwrap();
8081 serde_json::from_slice(&response.body).unwrap()
8082 }
8083
8084 #[tokio::test(start_paused = true)]
8085 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
8086 let registry = Arc::new(Registry::default());
8087 let forwarding = Arc::new(ForwardingTable::default());
8088 let handler = ControlHandler::with_forwarding(registry, forwarding);
8089 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
8090 hello_via_sink(
8091 &handler,
8092 &target_ctx,
8093 &mut target_rx,
8094 hello_frame("target", PROTOCOL_VERSION, 1),
8095 )
8096 .await;
8097 let root = unique_project_root("live-roots-known");
8098 let path = ProjectRootId::from_path_allowing_missing(root.path())
8099 .unwrap()
8100 .as_path()
8101 .to_path_buf();
8102 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
8103 let open_handler = handler.clone();
8104 let opened = tokio::spawn(async move {
8105 open_handler
8106 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
8107 .await
8108 .unwrap()
8109 });
8110 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8111 .await
8112 .unwrap()
8113 .unwrap();
8114 handler
8115 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
8116 .await
8117 .unwrap();
8118 assert!(opened.await.unwrap().is_empty());
8119 let _ = client_rx.recv().await.unwrap();
8120
8121 let root = unique_project_root("live-roots-pending");
8122 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
8123 .unwrap()
8124 .as_path()
8125 .to_path_buf();
8126 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
8127 let open_handler = handler.clone();
8128 let pending = tokio::spawn(async move {
8129 open_handler
8130 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
8131 .await
8132 .unwrap()
8133 });
8134 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8135 .await
8136 .unwrap()
8137 .unwrap();
8138 let actual = query_live_roots(&handler, &target_ctx).await;
8139 let ModuleControlResponseToModule::LiveRoots {
8140 roots,
8141 unknown_root_bindings,
8142 total_bindings,
8143 } = actual
8144 else {
8145 panic!("expected live roots")
8146 };
8147 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
8148 assert_eq!(unknown_root_bindings, 0);
8149 assert_eq!(
8150 roots.len(),
8151 2,
8152 "root-known arm must retain each canonical root"
8153 );
8154 assert_eq!(
8155 total_bindings,
8156 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8157 );
8158 let counts = roots
8159 .iter()
8160 .map(|root| (root.project_root.clone(), root.bound, root.pending))
8161 .collect::<Vec<_>>();
8162 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
8163 expected.sort_by(|a, b| a.0.cmp(&b.0));
8164 assert_eq!(
8165 counts, expected,
8166 "roots must sort by path and count pending separately"
8167 );
8168 handler
8169 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
8170 .await
8171 .unwrap();
8172 assert!(pending.await.unwrap().is_empty());
8173 }
8174
8175 #[tokio::test(start_paused = true)]
8176 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
8177 let forwarding = Arc::new(ForwardingTable::default());
8178 let handler =
8179 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8180 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
8181 hello_via_sink(
8182 &handler,
8183 &target_ctx,
8184 &mut target_rx,
8185 hello_frame("target", PROTOCOL_VERSION, 1),
8186 )
8187 .await;
8188 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
8189 let pending = forwarding
8190 .begin_route_bind_relay_for_test(
8191 client_ctx.connection_id,
8192 client_ctx.egress.clone(),
8193 2,
8194 "target",
8195 )
8196 .unwrap();
8197 forwarding
8198 .complete_pending_relay(
8199 target_ctx.connection_id,
8200 pending.corr,
8201 RouteBindRelayOutcome::Accepted,
8202 )
8203 .unwrap();
8204 let actual = query_live_roots(&handler, &target_ctx).await;
8205 let ModuleControlResponseToModule::LiveRoots {
8206 roots,
8207 unknown_root_bindings,
8208 total_bindings,
8209 } = actual
8210 else {
8211 panic!("expected live roots")
8212 };
8213 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
8214 assert_eq!(
8215 unknown_root_bindings, 1,
8216 "unknown-root arm must not read as no bindings"
8217 );
8218 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
8219 assert_eq!(
8220 total_bindings,
8221 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8222 );
8223 }
8224
8225 #[tokio::test(start_paused = true)]
8233 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
8234 let forwarding = Arc::new(ForwardingTable::default());
8235 let handler =
8236 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8237 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
8238 let replies = handler
8239 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
8240 .await
8241 .unwrap();
8242 let queued_by_hello = module_rx.len();
8243
8244 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
8245 let open_handler = handler.clone();
8246 let open = tokio::spawn(async move {
8247 open_handler
8248 .handle_control_frame(
8249 &client_ctx,
8250 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
8251 )
8252 .await
8253 .unwrap()
8254 });
8255 let mut spins = 0;
8257 while module_rx.len() == queued_by_hello {
8258 spins += 1;
8259 assert!(spins < 10_000, "route.open never queued a route.bind");
8260 tokio::task::yield_now().await;
8261 }
8262
8263 for reply in replies {
8265 module_ctx.egress.send(reply).await.unwrap();
8266 }
8267
8268 let first = module_rx.recv().await.unwrap().frame;
8269 assert_eq!(
8270 first.header.ty,
8271 FrameType::HelloAck,
8272 "the first frame a registering module reads must be its HELLO_ACK"
8273 );
8274 assert_eq!(first.header.corr, 7);
8275 let second = module_rx.recv().await.unwrap().frame;
8276 assert_eq!(second.header.ty, FrameType::Request);
8277 assert!(
8278 matches!(
8279 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
8280 ModuleControlRequest::RouteBind { .. }
8281 ),
8282 "the route.bind follows the ack"
8283 );
8284 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
8285
8286 handler
8287 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
8288 .await
8289 .unwrap();
8290 assert!(open.await.unwrap().is_empty());
8291 let _ = client_rx.recv().await.unwrap();
8292 }
8293
8294 #[tokio::test(start_paused = true)]
8295 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
8296 let handler = ControlHandler::with_forwarding(
8297 Arc::new(Registry::default()),
8298 Arc::new(ForwardingTable::default()),
8299 );
8300 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
8301 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
8302 hello_via_sink(
8303 &handler,
8304 &first_ctx,
8305 &mut first_rx,
8306 hello_frame("first", PROTOCOL_VERSION, 1),
8307 )
8308 .await;
8309 hello_via_sink(
8310 &handler,
8311 &second_ctx,
8312 &mut second_rx,
8313 hello_frame("second", PROTOCOL_VERSION, 2),
8314 )
8315 .await;
8316 let root = unique_project_root("second-only");
8317 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
8318 let cloned = handler.clone();
8319 let open = tokio::spawn(async move {
8320 cloned
8321 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
8322 .await
8323 .unwrap()
8324 });
8325 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
8326 .await
8327 .unwrap()
8328 .unwrap();
8329 let first = query_live_roots(&handler, &first_ctx).await;
8330 let second = query_live_roots(&handler, &second_ctx).await;
8331 assert!(
8332 matches!(
8333 first,
8334 ModuleControlResponseToModule::LiveRoots {
8335 total_bindings: 0,
8336 ..
8337 }
8338 ),
8339 "cross-module scope must not expose another module's roots"
8340 );
8341 assert!(
8342 matches!(
8343 second,
8344 ModuleControlResponseToModule::LiveRoots {
8345 total_bindings: 1,
8346 ..
8347 }
8348 ),
8349 "second module must see its pending route"
8350 );
8351 handler
8352 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
8353 .await
8354 .unwrap();
8355 assert!(open.await.unwrap().is_empty());
8356 }
8357
8358 #[tokio::test(start_paused = true)]
8359 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
8360 let handler = ControlHandler::with_forwarding(
8361 Arc::new(Registry::default()),
8362 Arc::new(ForwardingTable::default()),
8363 );
8364 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
8365 hello_via_sink(
8366 &handler,
8367 &target_ctx,
8368 &mut target_rx,
8369 hello_frame("target", PROTOCOL_VERSION, 1),
8370 )
8371 .await;
8372 let actual = query_live_roots(&handler, &target_ctx).await;
8373 let ModuleControlResponseToModule::LiveRoots {
8374 roots,
8375 unknown_root_bindings,
8376 total_bindings,
8377 } = actual
8378 else {
8379 panic!("expected live roots")
8380 };
8381 assert!(roots.is_empty());
8382 assert_eq!(unknown_root_bindings, 0);
8383 assert_eq!(total_bindings, 0);
8384 assert_eq!(
8385 total_bindings,
8386 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8387 );
8388 }
8389
8390 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
8397 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8398 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
8399 let text = std::fs::read_to_string(&path)
8400 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
8401 let vectors: Vec<(String, Value)> = text
8402 .lines()
8403 .filter(|line| !line.trim().is_empty())
8404 .map(|line| {
8405 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
8406 let id = entry["corpus_id"]
8407 .as_str()
8408 .expect("every vector carries a corpus_id")
8409 .to_string();
8410 (id, entry["package"].clone())
8411 })
8412 .collect();
8413 assert_eq!(
8417 vectors.len(),
8418 3,
8419 "vendored fed corpus changed size; re-sync from subc-federation"
8420 );
8421
8422 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
8437 let richest = vectors
8438 .iter()
8439 .filter_map(|(_, package)| package.as_object())
8440 .map(|object| {
8441 object
8442 .keys()
8443 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
8444 .count()
8445 })
8446 .max()
8447 .unwrap_or(0);
8448 assert!(
8449 richest >= 2,
8450 "vendored corpus no longer carries a package with unmodelled fields, \
8451 so the relay test can no longer distinguish a verbatim relay from a lossy one"
8452 );
8453
8454 vectors
8455 }
8456
8457 #[tokio::test]
8465 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
8466 for (corpus_id, package) in fed_admission_facts_vectors() {
8467 let registry = Arc::new(Registry::default());
8468 let forwarding = Arc::new(ForwardingTable::default());
8469 let supervisor = SupervisorHandle::new();
8470 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8471 let handler =
8472 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8473 .with_supervisor(supervisor)
8474 .with_admission_facts_config(
8475 Some("fed".to_string()),
8476 Some(vec!["target".to_string()]),
8477 );
8478
8479 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8480 hello_via_sink(
8481 &handler,
8482 &target_ctx,
8483 &mut target_rx,
8484 hello_frame("target", PROTOCOL_VERSION, 1),
8485 )
8486 .await;
8487
8488 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8489 let route_handler = handler.clone();
8490 let expected = package.clone();
8491 let route_task = tokio::spawn(async move {
8492 route_handler
8493 .handle_control_frame(
8494 &client_ctx,
8495 route_open_frame_with_admission_facts(
8496 20,
8497 "target",
8498 unique_project_root("admission-facts"),
8499 Some(subc_control::ConsumerIdentity {
8500 module_id: "fed".to_string(),
8501 launch_nonce: "fed-nonce".to_string(),
8502 }),
8503 Some(package),
8504 ),
8505 )
8506 .await
8507 .unwrap()
8508 });
8509
8510 let bind_frame = target_rx.recv().await.unwrap();
8511 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8512 let ModuleControlRequest::RouteBind {
8513 admission_facts, ..
8514 } = bind
8515 else {
8516 panic!("{corpus_id}: expected route.bind")
8517 };
8518 assert_eq!(
8519 admission_facts,
8520 Some(expected),
8521 "{corpus_id}: relay must not add, drop or reshape any field"
8522 );
8523
8524 handler
8525 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8526 .await
8527 .unwrap();
8528 route_task.await.unwrap();
8529 }
8530 }
8531
8532 #[tokio::test]
8533 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8534 let registry = Arc::new(Registry::default());
8535 let forwarding = Arc::new(ForwardingTable::default());
8536 let supervisor = SupervisorHandle::new();
8537 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8538 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8539 let handler =
8540 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8541 .with_supervisor(supervisor)
8542 .with_admission_facts_config(
8543 Some("fed".to_string()),
8544 Some(vec!["target".to_string()]),
8545 );
8546
8547 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8548 hello_via_sink(
8549 &handler,
8550 &target_ctx,
8551 &mut target_rx,
8552 hello_frame("target", PROTOCOL_VERSION, 1),
8553 )
8554 .await;
8555 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8556 hello_via_sink(
8557 &handler,
8558 &other_ctx,
8559 &mut other_rx,
8560 hello_frame("other", PROTOCOL_VERSION, 2),
8561 )
8562 .await;
8563
8564 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8565 let expected_facts = facts.clone();
8566 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8567 let route_handler = handler.clone();
8568 let route_task = tokio::spawn(async move {
8569 route_handler
8570 .handle_control_frame(
8571 &client_ctx,
8572 route_open_frame_with_admission_facts(
8573 10,
8574 "target",
8575 unique_project_root("admission-facts"),
8576 Some(subc_control::ConsumerIdentity {
8577 module_id: "fed".to_string(),
8578 launch_nonce: "fed-nonce".to_string(),
8579 }),
8580 Some(facts.clone()),
8581 ),
8582 )
8583 .await
8584 .unwrap()
8585 });
8586 let bind_frame = target_rx.recv().await.unwrap();
8587 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8588 let ModuleControlRequest::RouteBind {
8589 admission_facts, ..
8590 } = bind
8591 else {
8592 panic!("expected route.bind")
8593 };
8594 assert_eq!(admission_facts, Some(expected_facts));
8595 handler
8596 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8597 .await
8598 .unwrap();
8599 assert!(route_task.await.unwrap().is_empty());
8600 assert!(matches!(
8601 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8602 .unwrap(),
8603 ClientControlResponse::RouteOpen { .. }
8604 ));
8605
8606 let direct = handler
8607 .handle_control_frame(
8608 &route_ctx(ConnectionId::new(73)).0,
8609 route_open_frame_with_admission_facts(
8610 11,
8611 "target",
8612 unique_project_root("admission-facts"),
8613 None,
8614 Some(json!({"x": 1})),
8615 ),
8616 )
8617 .await
8618 .unwrap();
8619 assert_eq!(
8620 parse_error(&direct[0])["code"],
8621 "admission_facts_not_permitted"
8622 );
8623
8624 let different_reserved = handler
8625 .handle_control_frame(
8626 &route_ctx(ConnectionId::new(77)).0,
8627 route_open_frame_with_admission_facts(
8628 15,
8629 "target",
8630 unique_project_root("admission-facts"),
8631 Some(subc_control::ConsumerIdentity {
8632 module_id: "other".to_string(),
8633 launch_nonce: "other-nonce".to_string(),
8634 }),
8635 Some(json!({"x": 1})),
8636 ),
8637 )
8638 .await
8639 .unwrap();
8640 assert_eq!(
8641 parse_error(&different_reserved[0])["code"],
8642 "admission_facts_not_permitted"
8643 );
8644
8645 let other_target = handler
8646 .handle_control_frame(
8647 &route_ctx(ConnectionId::new(74)).0,
8648 route_open_frame_with_admission_facts(
8649 12,
8650 "other",
8651 unique_project_root("admission-facts"),
8652 Some(subc_control::ConsumerIdentity {
8653 module_id: "fed".to_string(),
8654 launch_nonce: "fed-nonce".to_string(),
8655 }),
8656 Some(json!({"x": 1})),
8657 ),
8658 )
8659 .await
8660 .unwrap();
8661 assert_eq!(
8662 parse_error(&other_target[0])["code"],
8663 "admission_facts_target_not_allowed"
8664 );
8665
8666 let nonexistent = handler
8667 .handle_control_frame(
8668 &route_ctx(ConnectionId::new(75)).0,
8669 route_open_frame_with_admission_facts(
8670 13,
8671 "missing",
8672 unique_project_root("admission-facts"),
8673 None,
8674 Some(json!({"x": 1})),
8675 ),
8676 )
8677 .await
8678 .unwrap();
8679 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8680
8681 let described = handler
8682 .handle_control_frame(
8683 &route_ctx(ConnectionId::new(76)).0,
8684 Frame::build(
8685 FrameType::Request,
8686 control_flags(),
8687 0,
8688 0,
8689 14,
8690 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8691 )
8692 .unwrap(),
8693 )
8694 .await
8695 .unwrap();
8696 let ClientControlResponse::ServerDescribe { capabilities, .. } =
8697 serde_json::from_slice(&described[0].body).unwrap()
8698 else {
8699 panic!("expected server.describe response")
8700 };
8701 assert!(capabilities
8702 .iter()
8703 .any(|cap| cap == "admission_facts_relay_v1"));
8704 }
8705
8706 #[tokio::test]
8707 async fn admission_facts_without_configured_carrier_are_rejected() {
8708 let registry = Arc::new(Registry::default());
8709 let forwarding = Arc::new(ForwardingTable::default());
8710 let handler = ControlHandler::with_forwarding(registry, forwarding);
8711 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8712 hello_via_sink(
8713 &handler,
8714 &target_ctx,
8715 &mut target_rx,
8716 hello_frame("target", PROTOCOL_VERSION, 1),
8717 )
8718 .await;
8719
8720 let responses = handler
8721 .handle_control_frame(
8722 &route_ctx(ConnectionId::new(79)).0,
8723 route_open_frame_with_admission_facts(
8724 16,
8725 "target",
8726 unique_project_root("admission-facts"),
8727 None,
8728 Some(json!({"x": 1})),
8729 ),
8730 )
8731 .await
8732 .unwrap();
8733 assert_eq!(
8734 parse_error(&responses[0])["code"],
8735 "admission_facts_not_permitted"
8736 );
8737 }
8738
8739 #[tokio::test]
8740 async fn route_open_relays_consumer_capabilities_verbatim() {
8741 let registry = Arc::new(Registry::default());
8742 let forwarding = Arc::new(ForwardingTable::default());
8743 let handler =
8744 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8745 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8746 hello_via_sink(
8747 &handler,
8748 &module_ctx,
8749 &mut module_rx,
8750 hello_frame("aft", PROTOCOL_VERSION, 7),
8751 )
8752 .await;
8753
8754 let expected = vec!["elicitation".to_string(), "roots".to_string()];
8755 let expected_for_request = expected.clone();
8756 let project_root = unique_project_root("consumer-capabilities-present");
8757 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8758 let route_handler = handler.clone();
8759 let route_task = tokio::spawn(async move {
8760 route_handler
8761 .handle_control_frame(
8762 &client_ctx,
8763 route_open_frame_with_consumer_capabilities(
8764 401,
8765 "aft",
8766 project_root,
8767 Some(expected_for_request),
8768 ),
8769 )
8770 .await
8771 .unwrap()
8772 });
8773 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8774 .await
8775 .unwrap()
8776 .unwrap();
8777 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8778 let ModuleControlRequest::RouteBind {
8779 consumer_capabilities,
8780 ..
8781 } = bind
8782 else {
8783 panic!("expected route.bind request, got {bind:?}");
8784 };
8785 assert_eq!(consumer_capabilities, Some(expected.clone()));
8786
8787 handler
8788 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8789 .await
8790 .unwrap();
8791 let route_response = route_task.await.unwrap();
8792 assert!(route_response.is_empty());
8793 let published = client_rx.recv().await.unwrap();
8794 assert!(matches!(
8795 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8796 ClientControlResponse::RouteOpen { .. }
8797 ));
8798 }
8799
8800 #[tokio::test]
8801 async fn route_open_without_consumer_capabilities_relays_none() {
8802 let registry = Arc::new(Registry::default());
8803 let forwarding = Arc::new(ForwardingTable::default());
8804 let handler =
8805 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8806 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8807 hello_via_sink(
8808 &handler,
8809 &module_ctx,
8810 &mut module_rx,
8811 hello_frame("aft", PROTOCOL_VERSION, 7),
8812 )
8813 .await;
8814
8815 let project_root = unique_project_root("consumer-capabilities-absent");
8816 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8817 let route_handler = handler.clone();
8818 let route_task = tokio::spawn(async move {
8819 route_handler
8820 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
8821 .await
8822 .unwrap()
8823 });
8824 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8825 .await
8826 .unwrap()
8827 .unwrap();
8828 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8829 let ModuleControlRequest::RouteBind {
8830 consumer_capabilities,
8831 ..
8832 } = bind
8833 else {
8834 panic!("expected route.bind request, got {bind:?}");
8835 };
8836 assert_eq!(consumer_capabilities, None);
8837
8838 handler
8839 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8840 .await
8841 .unwrap();
8842 let route_response = route_task.await.unwrap();
8843 assert!(route_response.is_empty());
8844 let published = client_rx.recv().await.unwrap();
8845 assert!(matches!(
8846 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8847 ClientControlResponse::RouteOpen { .. }
8848 ));
8849 }
8850
8851 #[tokio::test]
8852 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
8853 let registry = Arc::new(Registry::default());
8854 let forwarding = Arc::new(ForwardingTable::default());
8855 let handler =
8856 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8857 .with_health_probe_timeout(Duration::from_secs(5));
8858 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
8859 hello_via_sink(
8860 &handler,
8861 &module_ctx,
8862 &mut module_rx,
8863 non_routable_hello_frame_with_control_ops(
8864 "mcp",
8865 300,
8866 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8867 ),
8868 )
8869 .await;
8870 assert!(registry
8871 .get_module("mcp")
8872 .unwrap()
8873 .unwrap()
8874 .manifest
8875 .provides
8876 .is_empty());
8877
8878 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
8879 let route_response = handler
8880 .handle_control_frame(
8881 &route_client_ctx,
8882 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
8883 )
8884 .await
8885 .unwrap();
8886 assert_eq!(route_response[0].header.ty, FrameType::Error);
8887 assert_eq!(
8888 parse_error(&route_response[0])["code"],
8889 "target_unavailable"
8890 );
8891 assert!(parse_error(&route_response[0])["message"]
8892 .as_str()
8893 .unwrap()
8894 .contains("does not provide the requested target"));
8895 assert!(module_rx.try_recv().is_err());
8896
8897 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
8898 let health_handler = handler.clone();
8899 let health_task = tokio::spawn(async move {
8900 health_handler
8901 .handle_control_frame(
8902 &health_client_ctx,
8903 supervisor_health_probe_frame(302, "mcp"),
8904 )
8905 .await
8906 .unwrap()
8907 });
8908 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8909 .await
8910 .unwrap()
8911 .unwrap();
8912 assert_eq!(
8913 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8914 ModuleControlRequest::HealthCheck {}
8915 );
8916 handler
8917 .handle_control_frame(
8918 &module_ctx,
8919 health_response(health_frame.header.corr, HealthStatus::Ok),
8920 )
8921 .await
8922 .unwrap();
8923 let health_response = health_task.await.unwrap();
8924 assert_eq!(health_response[0].header.ty, FrameType::Response);
8925 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8926 ClientControlResponse::SupervisorHealthProbe {
8927 module_id, status, ..
8928 } => {
8929 assert_eq!(module_id, "mcp");
8930 assert_eq!(status, HealthStatus::Ok);
8931 }
8932 other => panic!("unexpected health response: {other:?}"),
8933 }
8934
8935 forwarding
8940 .cleanup_connection(module_ctx.connection_id)
8941 .unwrap();
8942 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
8943 let cleanup_response = tokio::time::timeout(
8944 Duration::from_millis(200),
8945 handler.handle_control_frame(
8946 &cleanup_probe_ctx,
8947 supervisor_health_probe_frame(303, "mcp"),
8948 ),
8949 )
8950 .await
8951 .expect("probe should fail immediately when the control lane is gone")
8952 .unwrap();
8953 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
8954 assert_eq!(
8955 parse_error(&cleanup_response[0])["code"],
8956 "target_unavailable"
8957 );
8958 assert!(parse_error(&cleanup_response[0])["message"]
8959 .as_str()
8960 .unwrap()
8961 .contains("no module connection"));
8962
8963 handler
8964 .cleanup_connection(module_ctx.connection_id)
8965 .unwrap();
8966 }
8967
8968 #[tokio::test]
8969 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
8970 let registry = Arc::new(Registry::default());
8971 let supervisor_handle = SupervisorHandle::new();
8972 let supervisor =
8973 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8974 .with_handle(supervisor_handle.clone())
8975 .with_connection_file_path(
8976 std::env::temp_dir()
8977 .join(format!("subc-route-open-warming-{}", std::process::id())),
8978 );
8979 let module = supervisor
8980 .supervise_configured(
8981 ModuleSpec {
8982 launch_nonce_env: true,
8983 module_id: "warming".to_string(),
8984 program: fake_aft_stub_path(),
8985 args: Vec::new(),
8986 env: Vec::new(),
8987 reserved: false,
8988 reserved_prefixes: Vec::new(),
8989 protocol: ModuleProtocol::Subc,
8990 overlap: Default::default(),
8991 },
8992 true,
8993 )
8994 .unwrap();
8995 assert_eq!(module.state().unwrap(), ModuleState::Running);
8996
8997 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8998 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
8999 let response = handler
9000 .handle_control_frame(
9001 &ctx,
9002 route_open_frame(304, "warming", unique_project_root("warming")),
9003 )
9004 .await
9005 .unwrap();
9006 module.stop().await.unwrap();
9007
9008 assert_eq!(response[0].header.ty, FrameType::Error);
9009 let error = parse_error(&response[0]);
9010 assert_eq!(error["code"], "module_warming");
9011 assert!(error["message"]
9012 .as_str()
9013 .unwrap()
9014 .contains("state=running, enabled=true, live=false"));
9015 }
9016
9017 #[test]
9018 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
9019 let handler = ControlHandler::new(Arc::new(Registry::default()));
9020 let capture = EventCapture::default();
9021 let _subscriber =
9022 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9023 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
9024 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
9025 let pending = (0..limit).collect::<Vec<_>>();
9026 let response = handler
9027 .route_open_capacity_refusal(
9028 &ctx,
9029 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
9030 "busy",
9031 pending.len(),
9032 limit,
9033 )
9034 .unwrap();
9035 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9036 let event = capture
9037 .events()
9038 .into_iter()
9039 .find(|event| {
9040 event.target == "control"
9041 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
9042 })
9043 .expect("connection admission refusal event");
9044 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9045 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9046 }
9047
9048 #[test]
9049 fn route_open_target_cap_logs_admission_reason_and_capacity() {
9050 let handler = ControlHandler::new(Arc::new(Registry::default()));
9051 let capture = EventCapture::default();
9052 let _subscriber =
9053 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9054 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
9055 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
9056 let guards = (0..limit)
9057 .map(|_| {
9058 handler
9059 .route_bind_concurrency
9060 .try_admit("busy", limit)
9061 .unwrap()
9062 })
9063 .collect::<Vec<_>>();
9064 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
9065 Err(in_flight) => in_flight,
9066 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
9067 };
9068 let response = handler
9069 .route_open_target_capacity_refusal(
9070 &ctx,
9071 &route_open_frame(397, "busy", unique_project_root("target-cap")),
9072 "busy",
9073 in_flight,
9074 )
9075 .unwrap();
9076 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9077 let event = capture
9078 .events()
9079 .into_iter()
9080 .find(|event| {
9081 event.target == "control"
9082 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
9083 })
9084 .expect("target admission refusal event");
9085 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9086 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9087 drop(guards);
9088 }
9089
9090 #[tokio::test]
9095 async fn route_open_refusal_names_the_check_that_refused() {
9096 let handler = ControlHandler::new(Arc::new(Registry::default()));
9097 let capture = EventCapture::default();
9098 let _subscriber =
9099 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9100 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9101 let response = handler
9102 .handle_control_frame(
9103 &ctx,
9104 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
9105 )
9106 .await
9107 .unwrap();
9108
9109 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9110 let event = capture
9111 .events()
9112 .into_iter()
9113 .find(|event| {
9114 event.target == "control"
9115 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9116 })
9117 .expect("route.open refusal event");
9118 assert_eq!(
9119 event.fields.get("reason"),
9120 Some(&"\"not_registered\"".to_string())
9121 );
9122 }
9123
9124 #[tokio::test]
9125 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
9126 let registry = Arc::new(Registry::default());
9127 let supervisor_handle = SupervisorHandle::new();
9128 let supervisor =
9129 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9130 .with_handle(supervisor_handle.clone())
9131 .with_connection_file_path(std::env::temp_dir().join(format!(
9132 "subc-route-open-refusal-info-{}",
9133 std::process::id()
9134 )));
9135 let module = supervisor
9136 .supervise_configured(
9137 ModuleSpec {
9138 launch_nonce_env: true,
9139 module_id: "warming".to_string(),
9140 program: fake_aft_stub_path(),
9141 args: Vec::new(),
9142 env: Vec::new(),
9143 reserved: false,
9144 reserved_prefixes: Vec::new(),
9145 protocol: ModuleProtocol::Subc,
9146 overlap: Default::default(),
9147 },
9148 true,
9149 )
9150 .unwrap();
9151 assert_eq!(module.state().unwrap(), ModuleState::Running);
9152
9153 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9154 assert!(handler
9155 .counters()
9156 .snapshot()
9157 .get("route_open_refused_by_code")
9158 .is_none());
9159 let capture = EventCapture::default();
9160 let _subscriber =
9161 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9162 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
9163 let response = handler
9164 .handle_control_frame(
9165 &ctx,
9166 route_open_frame(394, "warming", unique_project_root("refusal-info")),
9167 )
9168 .await
9169 .unwrap();
9170 module.stop().await.unwrap();
9171
9172 assert_eq!(parse_error(&response[0])["code"], "module_warming");
9173 let event = capture
9174 .events()
9175 .into_iter()
9176 .find(|event| {
9177 event.target == "control"
9178 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
9179 })
9180 .expect("route.open refusal event");
9181 assert_eq!(
9182 event.fields.get("module_id"),
9183 Some(&"\"warming\"".to_string())
9184 );
9185 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
9186 assert_eq!(
9187 event.fields.get("reason"),
9188 Some(&"\"supervised_not_registered\"".to_string())
9189 );
9190 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
9191 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
9192 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
9193 assert_eq!(
9194 handler.counters().snapshot()["route_open_refused_by_code"],
9195 json!({ "module_warming": 1 })
9196 );
9197 }
9198
9199 const OUTAGE_START: &str = "route.open refusing module: not serving";
9200 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
9201
9202 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
9203 capture
9204 .events()
9205 .into_iter()
9206 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
9207 .collect()
9208 }
9209
9210 fn supervise_stub(
9211 registry: &Arc<Registry>,
9212 module_id: &str,
9213 enabled: bool,
9214 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
9215 let supervisor_handle = SupervisorHandle::new();
9216 let supervisor =
9217 Supervisor::new(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
9218 .with_handle(supervisor_handle.clone())
9219 .with_connection_file_path(std::env::temp_dir().join(format!(
9220 "subc-route-outage-{module_id}-{}",
9221 std::process::id()
9222 )));
9223 let module = supervisor
9224 .supervise_configured(
9225 ModuleSpec {
9226 launch_nonce_env: true,
9227 module_id: module_id.to_string(),
9228 program: fake_aft_stub_path(),
9229 args: Vec::new(),
9230 env: Vec::new(),
9231 reserved: false,
9232 reserved_prefixes: Vec::new(),
9233 protocol: ModuleProtocol::Subc,
9234 overlap: Default::default(),
9235 },
9236 enabled,
9237 )
9238 .unwrap();
9239 (supervisor_handle, module)
9240 }
9241
9242 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
9243 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
9244 module_id: module_id.to_string(),
9245 drain_timeout_ms: Some(50),
9246 })
9247 .unwrap();
9248 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9249 }
9250
9251 #[test]
9255 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
9256 let registry = Arc::new(Registry::default());
9257 let forwarding = Arc::new(ForwardingTable::default());
9258 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9259 let second = ControlHandler::with_forwarding(registry, forwarding);
9260 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
9261 }
9262
9263 #[tokio::test(flavor = "current_thread")]
9267 async fn route_open_unknown_module_refusals_add_no_outage_state() {
9268 let handler = ControlHandler::new(Arc::new(Registry::default()));
9269 let capture = EventCapture::default();
9270 let _subscriber =
9271 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9272 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
9273 for corr in 0..8 {
9274 let response = handler
9275 .handle_control_frame(
9276 &ctx,
9277 route_open_frame(
9278 380 + corr,
9279 &format!("nobody-{corr}"),
9280 unique_project_root("outage-unknown"),
9281 ),
9282 )
9283 .await
9284 .unwrap();
9285 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9286 }
9287
9288 assert_eq!(handler.route_outages.tracked_module_count(), 0);
9289 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
9290 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
9291 }
9292
9293 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9298 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
9299 let registry = Arc::new(Registry::default());
9300 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
9301 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9302 let capture = EventCapture::default();
9303 let _subscriber =
9304 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9305 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
9306 handler.route_outages.record_accepted("outage-restart");
9309
9310 let response = handler
9311 .handle_control_frame(
9312 &ctx,
9313 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
9314 )
9315 .await
9316 .unwrap();
9317 assert_eq!(response[0].header.ty, FrameType::Error);
9318 let starts = outage_lines(&capture, OUTAGE_START);
9319 assert_eq!(starts.len(), 1, "{starts:?}");
9320 assert_eq!(starts[0].level, tracing::Level::WARN);
9321 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
9322 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
9323 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
9324 handler.route_outages.record_accepted("outage-restart");
9325 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
9326
9327 let restart = handler
9328 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
9329 .await
9330 .unwrap();
9331 assert_eq!(
9332 restart[0].header.ty,
9333 FrameType::Response,
9334 "{:?}",
9335 parse_error(&restart[0])
9336 );
9337 handler
9338 .handle_control_frame(
9339 &ctx,
9340 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
9341 )
9342 .await
9343 .unwrap();
9344 module.stop().await.unwrap();
9345
9346 let starts = outage_lines(&capture, OUTAGE_START);
9347 assert_eq!(starts.len(), 2, "{starts:?}");
9348 assert_eq!(starts[1].level, tracing::Level::INFO);
9349 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
9350 }
9351
9352 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9356 async fn failed_operator_restart_leaves_no_operator_mark() {
9357 let registry = Arc::new(Registry::default());
9358 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
9359 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9360 let capture = EventCapture::default();
9361 let _subscriber =
9362 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9363 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
9364 handler.route_outages.record_accepted("outage-disabled");
9365
9366 let restart = handler
9367 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
9368 .await
9369 .unwrap();
9370 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
9371 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
9372
9373 handler
9374 .handle_control_frame(
9375 &ctx,
9376 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
9377 )
9378 .await
9379 .unwrap();
9380 let starts = outage_lines(&capture, OUTAGE_START);
9381 assert_eq!(starts.len(), 1, "{starts:?}");
9382 assert_eq!(starts[0].level, tracing::Level::WARN);
9383 }
9384
9385 #[tokio::test(flavor = "current_thread")]
9386 async fn route_open_unknown_module_escapes_target_module_id() {
9387 let handler = ControlHandler::new(Arc::new(Registry::default()));
9388 let capture = EventCapture::default();
9389 let _subscriber =
9390 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9391 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
9392 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9393 let response = handler
9394 .handle_control_frame(
9395 &ctx,
9396 route_open_frame(
9397 395,
9398 hostile_module_id,
9399 unique_project_root("hostile-target-module-id"),
9400 ),
9401 )
9402 .await
9403 .unwrap();
9404
9405 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9406 let event = capture
9407 .events()
9408 .into_iter()
9409 .find(|event| {
9410 event.target == "control"
9411 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9412 })
9413 .expect("route.open unknown-module refusal event");
9414 let logged = event.fields.get("module_id").expect("module_id field");
9415 assert!(!logged.bytes().any(|byte| byte < 0x20));
9416 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9417 }
9418
9419 #[tokio::test(flavor = "current_thread")]
9420 async fn route_open_module_rejection_uses_daemon_counter_key() {
9421 let registry = Arc::new(Registry::default());
9422 let forwarding = Arc::new(ForwardingTable::default());
9423 let handler =
9424 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9425 let module_connection = ConnectionId::new(95);
9426 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9427 hello_via_sink(
9428 &handler,
9429 &module_ctx,
9430 &mut module_rx,
9431 hello_frame("aft", PROTOCOL_VERSION, 395),
9432 )
9433 .await;
9434
9435 let client_connection = ConnectionId::new(96);
9436 let (client_ctx, _client_rx) = route_ctx(client_connection);
9437 let capture = EventCapture::default();
9438 let _subscriber =
9439 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9440 let (route_task, bind) = relay_route_open(
9441 &handler,
9442 client_connection,
9443 &client_ctx.egress,
9444 &mut module_rx,
9445 396,
9446 "aft",
9447 "hostile-module-code",
9448 )
9449 .await;
9450 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
9451 let rejection = Frame::build(
9452 FrameType::Error,
9453 control_flags(),
9454 0,
9455 0,
9456 bind.header.corr,
9457 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
9458 )
9459 .unwrap();
9460 handler
9461 .handle_control_frame(&module_ctx, rejection)
9462 .await
9463 .unwrap();
9464
9465 let response = route_task.await.unwrap();
9466 assert_eq!(parse_error(&response[0])["code"], hostile_code);
9467 let counters = handler.counters().snapshot();
9468 assert_eq!(
9469 counters["route_open_refused_by_code"],
9470 json!({ "module_rejected": 1 })
9471 );
9472 assert!(counters["route_open_refused_by_code"]
9473 .get(hostile_code)
9474 .is_none());
9475
9476 let event = capture
9477 .events()
9478 .into_iter()
9479 .find(|event| {
9480 event.target == "control"
9481 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
9482 })
9483 .expect("route.open module-rejection refusal event");
9484 let logged = event.fields.get("module_code").expect("module_code field");
9485 assert!(!logged.bytes().any(|byte| byte < 0x20));
9486 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9487 }
9488
9489 #[tokio::test]
9490 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
9491 let registry = Arc::new(Registry::default());
9492 let supervisor_handle = SupervisorHandle::new();
9493 let missing_program = std::env::temp_dir().join(format!(
9494 "subc-route-open-missing-program-{}",
9495 std::process::id()
9496 ));
9497 let supervisor =
9498 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9499 .with_handle(supervisor_handle.clone());
9500 let module = supervisor
9501 .supervise_configured(
9502 ModuleSpec {
9503 launch_nonce_env: true,
9504 module_id: "failed".to_string(),
9505 program: missing_program,
9506 args: Vec::new(),
9507 env: Vec::new(),
9508 reserved: false,
9509 reserved_prefixes: Vec::new(),
9510 protocol: ModuleProtocol::Subc,
9511 overlap: Default::default(),
9512 },
9513 true,
9514 )
9515 .unwrap();
9516 assert_eq!(module.state().unwrap(), ModuleState::Failed);
9517
9518 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9519 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
9520 let response = handler
9521 .handle_control_frame(
9522 &ctx,
9523 route_open_frame(305, "failed", unique_project_root("failed")),
9524 )
9525 .await
9526 .unwrap();
9527
9528 assert_eq!(response[0].header.ty, FrameType::Error);
9529 let error = parse_error(&response[0]);
9530 assert_eq!(error["code"], "target_unavailable");
9531 assert!(error["message"]
9532 .as_str()
9533 .unwrap()
9534 .contains("state=failed, enabled=true, live=false"));
9535 }
9536
9537 #[tokio::test]
9538 async fn route_open_role_mismatch_remains_target_unavailable() {
9539 let registry = Arc::new(Registry::default());
9540 let handler = ControlHandler::new(Arc::clone(®istry));
9541 handler
9542 .handle_control(
9543 ConnectionId::new(41),
9544 non_routable_hello_frame_with_control_ops("health-only", 306, None),
9545 )
9546 .unwrap();
9547
9548 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
9549 let response = handler
9550 .handle_control_frame(
9551 &ctx,
9552 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
9553 )
9554 .await
9555 .unwrap();
9556
9557 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9558 assert!(parse_error(&response[0])["message"]
9559 .as_str()
9560 .unwrap()
9561 .contains("does not provide the requested target"));
9562 }
9563
9564 #[tokio::test]
9565 async fn route_open_inactive_registration_remains_target_unavailable() {
9566 let registry = Arc::new(Registry::default());
9567 let handler = ControlHandler::new(Arc::clone(®istry));
9568 handler
9569 .handle_control(
9570 ConnectionId::new(43),
9571 hello_frame("inactive", PROTOCOL_VERSION, 308),
9572 )
9573 .unwrap();
9574 assert!(registry
9575 .set_module_state_for_test("inactive", ChannelState::Closed)
9576 .unwrap());
9577
9578 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
9579 let response = handler
9580 .handle_control_frame(
9581 &ctx,
9582 route_open_frame(309, "inactive", unique_project_root("inactive")),
9583 )
9584 .await
9585 .unwrap();
9586
9587 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9588 assert!(parse_error(&response[0])["message"]
9589 .as_str()
9590 .unwrap()
9591 .contains("is not active"));
9592 }
9593
9594 #[tokio::test]
9595 async fn late_health_reply_is_recorded_through_the_module_response_path() {
9596 let registry = Arc::new(Registry::default());
9597 let forwarding = Arc::new(ForwardingTable::default());
9598 let supervisor_handle = SupervisorHandle::new();
9599 let supervisor = Supervisor::new(Arc::clone(®istry), crate::RestartPolicy::default())
9600 .with_forwarding(Arc::clone(&forwarding))
9601 .with_handle(supervisor_handle.clone());
9602 let module = supervisor
9603 .supervise_configured(
9604 crate::ModuleSpec {
9605 launch_nonce_env: true,
9606 module_id: "late-health-response".to_string(),
9607 program: PathBuf::from("disabled-module"),
9608 args: Vec::new(),
9609 env: Vec::new(),
9610 reserved: false,
9611 reserved_prefixes: Vec::new(),
9612 protocol: ModuleProtocol::Subc,
9613 overlap: Default::default(),
9614 },
9615 false,
9616 )
9617 .unwrap();
9618 let handler =
9619 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9620 .with_supervisor(supervisor_handle);
9621 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
9622 handler
9623 .handle_control_frame(
9624 &module_ctx,
9625 hello_frame_with_control_ops(
9626 "late-health-response",
9627 PROTOCOL_VERSION,
9628 7,
9629 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9630 ),
9631 )
9632 .await
9633 .unwrap();
9634 let probe_started_at = Instant::now() - Duration::from_millis(80);
9635 let pending = forwarding
9636 .begin_health_probe_rpc_for(
9637 "late-health-response",
9638 MODULE_CONTROL_OP_HEALTH_CHECK,
9639 probe_started_at,
9640 Instant::now() - Duration::from_millis(1),
9641 )
9642 .unwrap();
9643 assert!(forwarding
9644 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
9645 .unwrap());
9646
9647 let responses = handler
9648 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
9649 .await
9650 .unwrap();
9651
9652 assert!(responses.is_empty());
9653 let health = module.status().unwrap().health;
9654 assert_eq!(health.late_answer_count, 1);
9655 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
9656 }
9657
9658 #[tokio::test]
9659 async fn health_probe_timeout_and_module_death_are_typed() {
9660 let registry = Arc::new(Registry::default());
9661 let forwarding = Arc::new(ForwardingTable::default());
9662 let handler =
9663 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9664 .with_health_probe_timeout(Duration::from_millis(50));
9665 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9666 hello_via_sink(
9667 &handler,
9668 &module_ctx,
9669 &mut module_rx,
9670 hello_frame_with_control_ops(
9671 "aft",
9672 PROTOCOL_VERSION,
9673 7,
9674 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9675 ),
9676 )
9677 .await;
9678
9679 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9680 let responses = handler
9681 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
9682 .await
9683 .unwrap();
9684 assert_eq!(responses[0].header.ty, FrameType::Error);
9685 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
9686 let _ = module_rx.try_recv();
9687
9688 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
9689 let health_handler = handler.clone();
9690 let death_task = tokio::spawn(async move {
9691 health_handler
9692 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
9693 .await
9694 .unwrap()
9695 });
9696 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9697 .await
9698 .unwrap()
9699 .unwrap();
9700 handler
9701 .cleanup_connection(module_ctx.connection_id)
9702 .unwrap();
9703 let responses = death_task.await.unwrap();
9704 assert_eq!(responses[0].header.ty, FrameType::Error);
9705 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
9706 }
9707
9708 #[test]
9709 fn hello_requires_exact_protocol_version() {
9710 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
9711 let registry = Arc::new(Registry::default());
9712 let handler = ControlHandler::new(Arc::clone(®istry));
9713 let responses = handler
9714 .handle_control(
9715 ConnectionId::new(connection),
9716 hello_frame("aft", offered, 9),
9717 )
9718 .unwrap();
9719
9720 assert_eq!(responses.len(), 1);
9721 assert_eq!(responses[0].header.ty, FrameType::Error);
9722 let error = parse_error(&responses[0]);
9723 assert_eq!(error["code"], "version_unsupported");
9724 assert!(registry.get_module("aft").unwrap().is_none());
9725 assert_eq!(registry.active_registration_count().unwrap(), 0);
9726 }
9727 }
9728
9729 #[test]
9730 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
9731 let registry = Arc::new(Registry::default());
9732 let forwarding = Arc::new(ForwardingTable::default());
9733 let handler =
9734 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9735 let module_connection = ConnectionId::new(301);
9736 let registration = registry
9737 .register_with_control_ops(
9738 manifest("aft-push", PROTOCOL_VERSION),
9739 PROTOCOL_VERSION,
9740 module_connection,
9741 module_baseline_control_ops(),
9742 )
9743 .unwrap();
9744 let (module_tx, _module_rx) = mpsc::channel(8);
9745 let endpoint = forwarding
9746 .register_module_connection(
9747 module_connection,
9748 "aft-push".to_string(),
9749 PROTOCOL_VERSION,
9750 manifest_concurrency(®istration.manifest),
9751 FrameSink::new(module_tx),
9752 )
9753 .unwrap();
9754
9755 let unknown = Frame::build(
9757 FrameType::Push,
9758 control_flags(),
9759 0,
9760 0,
9761 5,
9762 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
9763 )
9764 .unwrap();
9765 let out = handler.handle_status_update(endpoint, unknown).unwrap();
9766 assert!(
9767 out.is_empty(),
9768 "unknown push op must be ignored, got {out:?}"
9769 );
9770
9771 let malformed = Frame::build(
9773 FrameType::Push,
9774 control_flags(),
9775 0,
9776 0,
9777 6,
9778 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
9779 )
9780 .unwrap();
9781 let out = handler.handle_status_update(endpoint, malformed).unwrap();
9782 assert_eq!(out.len(), 1);
9783 assert_eq!(out[0].header.ty, FrameType::Error);
9784 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
9785 }
9786
9787 #[test]
9788 fn hello_rejected_when_connection_already_owns_client_routes() {
9789 let registry = Arc::new(Registry::default());
9790 let forwarding = Arc::new(ForwardingTable::default());
9791 let handler =
9792 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9793 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
9795 let client_connection = ConnectionId::new(202);
9796
9797 let responses = handler
9800 .handle_control(
9801 client_connection,
9802 hello_frame("aft-second", PROTOCOL_VERSION, 9),
9803 )
9804 .unwrap();
9805 assert_eq!(responses[0].header.ty, FrameType::Error);
9806 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
9807 assert!(registry.get_module("aft-second").unwrap().is_none());
9808 }
9809
9810 #[test]
9811 fn reserved_module_hello_requires_matching_launch_nonce() {
9812 let registry = Arc::new(Registry::default());
9813 let supervisor = SupervisorHandle::new();
9814 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9817 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9818
9819 let no_nonce = handler
9821 .handle_control(
9822 ConnectionId::new(1),
9823 hello_frame("vault", PROTOCOL_VERSION, 1),
9824 )
9825 .unwrap();
9826 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
9827 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
9828 assert!(registry.get_module("vault").unwrap().is_none());
9829
9830 let wrong = handler
9832 .handle_control(
9833 ConnectionId::new(2),
9834 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
9835 )
9836 .unwrap();
9837 assert_eq!(wrong[0].header.ty, FrameType::Error);
9838 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
9839 assert!(registry.get_module("vault").unwrap().is_none());
9840
9841 let ok = handler
9843 .handle_control(
9844 ConnectionId::new(3),
9845 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
9846 )
9847 .unwrap();
9848 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
9849 assert!(registry.get_module("vault").unwrap().is_some());
9850 }
9851
9852 #[test]
9853 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
9854 let registry = Arc::new(Registry::default());
9855 let supervisor = SupervisorHandle::new();
9856 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9857 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9858 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9859
9860 let squat = handler
9861 .handle_control(
9862 ConnectionId::new(1),
9863 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
9864 )
9865 .unwrap();
9866 assert_eq!(squat[0].header.ty, FrameType::Error);
9867 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
9868 assert!(parse_error(&squat[0])["message"]
9869 .as_str()
9870 .unwrap()
9871 .contains("fed:"));
9872
9873 let accepted_peer = handler
9874 .handle_control(
9875 ConnectionId::new(2),
9876 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
9877 )
9878 .unwrap();
9879 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
9880
9881 let accepted_short = handler
9882 .handle_control(
9883 ConnectionId::new(3),
9884 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
9885 )
9886 .unwrap();
9887 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
9888
9889 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
9890 let response = handler
9891 .handle_control(
9892 ConnectionId::new(conn),
9893 hello_frame(module_id, PROTOCOL_VERSION, conn),
9894 )
9895 .unwrap();
9896 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
9897 }
9898 }
9899
9900 #[test]
9901 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
9902 let registry = Arc::new(Registry::default());
9903 let supervisor = SupervisorHandle::new();
9904 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9905 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9906 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
9907 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9908
9909 let owner_nonce = handler
9910 .handle_control(
9911 ConnectionId::new(1),
9912 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
9913 )
9914 .unwrap();
9915 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
9916 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
9917 assert!(registry.get_module("fed:special").unwrap().is_none());
9918
9919 let exact_nonce = handler
9920 .handle_control(
9921 ConnectionId::new(2),
9922 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
9923 )
9924 .unwrap();
9925 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
9926 assert!(registry.get_module("fed:special").unwrap().is_some());
9927 }
9928
9929 #[test]
9930 fn non_reserved_module_ignores_launch_nonce() {
9931 let registry = Arc::new(Registry::default());
9932 let handler = ControlHandler::new(Arc::clone(®istry));
9935 let no_nonce = handler
9936 .handle_control(
9937 ConnectionId::new(1),
9938 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
9939 )
9940 .unwrap();
9941 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
9942 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
9943
9944 let echoed_nonce = handler
9945 .handle_control(
9946 ConnectionId::new(2),
9947 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
9948 )
9949 .unwrap();
9950 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
9951 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
9952 }
9953
9954 #[test]
9955 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
9956 let handler = ControlHandler::default();
9957 let conn = ConnectionId::new(1);
9958 let malformed = Frame::build(
9959 FrameType::Hello,
9960 control_flags(),
9961 0,
9962 0,
9963 3,
9964 b"{not json".to_vec(),
9965 )
9966 .unwrap();
9967
9968 let error = handler.handle_control(conn, malformed).unwrap();
9969 assert_eq!(error[0].header.ty, FrameType::Error);
9970 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
9971
9972 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
9973 let pong = handler.handle_control(conn, ping).unwrap();
9974 assert_eq!(pong[0].header.ty, FrameType::Pong);
9975 assert_eq!(pong[0].header.corr, 4);
9976 }
9977
9978 #[test]
9979 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
9980 let registry = Arc::new(Registry::default());
9981 let handler = ControlHandler::new(Arc::clone(®istry));
9982
9983 handler
9984 .handle_control(
9985 ConnectionId::new(1),
9986 hello_frame("aft", PROTOCOL_VERSION, 1),
9987 )
9988 .unwrap();
9989 let duplicate = handler
9990 .handle_control(
9991 ConnectionId::new(2),
9992 hello_frame("aft", PROTOCOL_VERSION, 2),
9993 )
9994 .unwrap();
9995
9996 assert_eq!(duplicate[0].header.ty, FrameType::Error);
9997 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9998 let registration = registry.get_module("aft").unwrap().unwrap();
9999 assert_eq!(registration.connection_id, ConnectionId::new(1));
10000 }
10001
10002 #[test]
10003 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
10004 let registry = Arc::new(Registry::default());
10005 let forwarding = Arc::new(ForwardingTable::default());
10006 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
10007 let handler =
10008 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10009 .with_process_liveness(process_liveness);
10010 let (ctx, route_channel, route_epoch) =
10011 bind_liveness_route(®istry, &forwarding, "aft-dead");
10012 let responses = handler
10013 .handle_route_poll(
10014 &ctx,
10015 route_poll_frame(41, PollKind::Liveness, route_channel),
10016 route_channel,
10017 route_epoch,
10018 PollKind::Liveness,
10019 )
10020 .unwrap();
10021
10022 assert_eq!(responses.len(), 1);
10023 assert_eq!(responses[0].header.ty, FrameType::Response);
10024 assert_route_poll_liveness(&responses[0], false);
10025 }
10026
10027 #[test]
10028 fn liveness_poll_without_process_source_uses_bound_route() {
10029 let registry = Arc::new(Registry::default());
10030 let forwarding = Arc::new(ForwardingTable::default());
10031 let handler =
10032 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10033 let (ctx, route_channel, route_epoch) =
10034 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
10035 let responses = handler
10036 .handle_route_poll(
10037 &ctx,
10038 route_poll_frame(42, PollKind::Liveness, route_channel),
10039 route_channel,
10040 route_epoch,
10041 PollKind::Liveness,
10042 )
10043 .unwrap();
10044
10045 assert_route_poll_liveness(&responses[0], true);
10046 }
10047
10048 #[test]
10049 fn liveness_poll_untracked_process_source_uses_bound_route() {
10050 let registry = Arc::new(Registry::default());
10051 let forwarding = Arc::new(ForwardingTable::default());
10052 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
10053 let handler =
10054 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10055 .with_process_liveness(process_liveness);
10056 let (ctx, route_channel, route_epoch) =
10057 bind_liveness_route(®istry, &forwarding, "aft-untracked");
10058 let responses = handler
10059 .handle_route_poll(
10060 &ctx,
10061 route_poll_frame(43, PollKind::Liveness, route_channel),
10062 route_channel,
10063 route_epoch,
10064 PollKind::Liveness,
10065 )
10066 .unwrap();
10067
10068 assert_route_poll_liveness(&responses[0], true);
10069 }
10070
10071 #[tokio::test]
10072 async fn unknown_op_returns_unknown_control_op() {
10073 let handler = ControlHandler::default();
10074 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
10075 let request = Frame::build(
10076 FrameType::Request,
10077 control_flags(),
10078 0,
10079 0,
10080 55,
10081 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
10082 )
10083 .unwrap();
10084
10085 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10086
10087 assert_eq!(response.len(), 1);
10088 assert_eq!(response[0].header.ty, FrameType::Error);
10089 assert_eq!(response[0].header.corr, 55);
10090 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
10091 }
10092
10093 #[tokio::test]
10094 async fn supervisor_provenance_rejects_unknown_exact_module() {
10095 let handler = ControlHandler::default();
10096 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
10097 let request = Frame::build(
10098 FrameType::Request,
10099 control_flags(),
10100 0,
10101 0,
10102 57,
10103 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
10104 )
10105 .unwrap();
10106
10107 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10108
10109 assert_eq!(response.len(), 1);
10110 assert_eq!(response[0].header.ty, FrameType::Error);
10111 assert_eq!(response[0].header.corr, 57);
10112 let error = parse_error(&response[0]);
10113 assert_eq!(error["code"], "unknown_module");
10114 assert_eq!(error["message"], "module_id 'missing' is not supervised");
10115 }
10116
10117 #[test]
10118 fn provenance_probe_override_keeps_handler_tests_deterministic() {
10119 let expected = subc_control::RunningImageAgreement::Unavailable {
10120 reason: subc_control::RunningImageUnavailableReason::HashFailed,
10121 };
10122 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
10123 assert_eq!(handler.provenance_probe_override, Some(expected));
10124 }
10125
10126 #[test]
10127 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
10128 let verdict = reload_verdict(
10129 std::path::Path::new("/bin/new"),
10130 Some(std::path::Path::new("/bin/old")),
10131 subc_control::RunningImageAgreement::Unavailable {
10132 reason: subc_control::RunningImageUnavailableReason::HashFailed,
10133 },
10134 );
10135 assert!(matches!(
10136 verdict.path,
10137 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
10138 if configured == std::path::Path::new("/bin/new")
10139 && spawned_from == std::path::Path::new("/bin/old")
10140 ));
10141 }
10142
10143 #[test]
10144 fn reload_verdict_detects_replaced_image_at_same_path() {
10145 let image = subc_control::RunningImageAgreement::Mismatch {
10146 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
10147 digest: "old".into(),
10148 },
10149 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
10150 digest: "new".into(),
10151 },
10152 };
10153 let verdict = reload_verdict(
10154 std::path::Path::new("/bin/same"),
10155 Some(std::path::Path::new("/bin/same")),
10156 image.clone(),
10157 );
10158 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10159 assert_eq!(verdict.image, image);
10160 }
10161
10162 #[test]
10163 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
10164 let image = subc_control::RunningImageAgreement::Unavailable {
10165 reason: subc_control::RunningImageUnavailableReason::NotRunning,
10166 };
10167 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
10168 assert_eq!(
10169 verdict.path,
10170 subc_control::ReloadPathAgreement::Unavailable {
10171 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
10172 }
10173 );
10174 assert_eq!(verdict.image, image);
10175
10176 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
10177 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
10178 };
10179 let verdict = reload_verdict(
10180 std::path::Path::new("/bin/same"),
10181 Some(std::path::Path::new("/bin/same")),
10182 unconfirmed.clone(),
10183 );
10184 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10185 assert_eq!(verdict.image, unconfirmed);
10186 }
10187
10188 #[test]
10189 fn reload_verdict_preserves_each_image_unavailability_reason() {
10190 use subc_control::RunningImageUnavailableReason as Reason;
10191
10192 for reason in [
10193 Reason::NotRunning,
10194 Reason::UnsupportedPlatform,
10195 Reason::RunningExecutableUnreadable,
10196 Reason::SpawnedPathUnreadable,
10197 Reason::HashFailed,
10198 Reason::ProcessIdentityUnconfirmed,
10199 Reason::Unknown("future_probe_reason".to_string()),
10200 ] {
10201 let image = subc_control::RunningImageAgreement::Unavailable {
10202 reason: reason.clone(),
10203 };
10204 let verdict = reload_verdict(
10205 std::path::Path::new("/bin/same"),
10206 Some(std::path::Path::new("/bin/same")),
10207 image.clone(),
10208 );
10209 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10210 assert_eq!(verdict.image, image, "{reason:?}");
10211 }
10212 }
10213
10214 #[tokio::test]
10215 async fn malformed_control_bodies_return_invalid_control_body() {
10216 let handler = ControlHandler::default();
10217 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
10218
10219 for (corr, body) in [
10220 (56, br#"{"route_channel":1}"#.as_slice()),
10221 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
10222 (
10223 58,
10224 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
10225 ),
10226 ] {
10227 let request = Frame::build(
10228 FrameType::Request,
10229 control_flags(),
10230 0,
10231 0,
10232 corr,
10233 body.to_vec(),
10234 )
10235 .unwrap();
10236 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10237
10238 assert_eq!(response.len(), 1);
10239 assert_eq!(response[0].header.ty, FrameType::Error);
10240 assert_eq!(response[0].header.corr, corr);
10241 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
10242 }
10243 }
10244
10245 #[tokio::test]
10246 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
10247 let registry = Arc::new(Registry::default());
10248 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
10249 let router = Router::with_control_handler(Arc::clone(&control));
10250 let connection = router.begin_connection();
10251 let (ctx, mut rx) = route_ctx(connection.id());
10252
10253 router
10254 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
10255 .await
10256 .unwrap();
10257 let response = rx.recv().await.unwrap();
10258 let ack = parse_ack(&response);
10259 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10260 let channel = 1;
10261
10262 let goodbye =
10263 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
10264 router.route_for_connection(&ctx, goodbye).await.unwrap();
10265 assert!(rx.try_recv().is_err());
10266 assert!(registry.get_module("aft").unwrap().is_none());
10267
10268 router
10269 .route_for_connection(&ctx, channel_request(channel, 13))
10270 .await
10271 .unwrap();
10272 let error_frame = rx.recv().await.unwrap();
10273 assert_eq!(error_frame.header.ty, FrameType::Error);
10274 assert_eq!(error_frame.header.channel, channel);
10275 }
10276
10277 #[tokio::test]
10278 async fn dropping_router_connection_releases_registration() {
10279 let registry = Arc::new(Registry::default());
10280 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
10281 let router = Router::with_control_handler(control);
10282 let connection = router.begin_connection();
10283 let (ctx, mut rx) = route_ctx(connection.id());
10284
10285 router
10286 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
10287 .await
10288 .unwrap();
10289 let response = rx.recv().await.unwrap();
10290 let ack = parse_ack(&response);
10291 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10292 assert!(registry.get_module("aft").unwrap().is_some());
10293
10294 drop(connection);
10295
10296 assert!(registry.get_module("aft").unwrap().is_none());
10297 assert_eq!(registry.active_registration_count().unwrap(), 0);
10298 }
10299
10300 fn capability_manifest(
10301 module_id: &str,
10302 provides: &[&str],
10303 must_never_reach: &[&str],
10304 ) -> ModuleManifest {
10305 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
10306 manifest.capabilities = Some(CapabilityDeclarations {
10307 provides: provides
10308 .iter()
10309 .map(|capability| (*capability).to_string())
10310 .collect(),
10311 requires: Vec::new(),
10312 must_never_reach: must_never_reach
10313 .iter()
10314 .map(|capability| (*capability).to_string())
10315 .collect(),
10316 });
10317 manifest
10318 }
10319
10320 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
10321 Frame::build(
10322 FrameType::Hello,
10323 control_flags(),
10324 0,
10325 0,
10326 corr,
10327 serde_json::to_vec(&ModuleHelloBody {
10328 protocol_ver: manifest.protocol_ver,
10329 manifest,
10330 control_ops: None,
10331 launch_nonce: None,
10332 })
10333 .expect("capability test HELLO serializes"),
10334 )
10335 .expect("capability test HELLO frame builds")
10336 }
10337
10338 fn catalog_update_with_capabilities_frame(
10339 corr: u64,
10340 capabilities: CapabilityDeclarations,
10341 ) -> Frame {
10342 Frame::build(
10343 FrameType::Request,
10344 control_flags(),
10345 0,
10346 0,
10347 corr,
10348 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10349 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
10350 capabilities: Some(capabilities),
10351 ready: None,
10352 })
10353 .expect("capability catalog.update serializes"),
10354 )
10355 .expect("capability catalog.update frame builds")
10356 }
10357
10358 async fn register_capability_manifest(
10359 handler: &ControlHandler,
10360 ctx: &RouteCtx,
10361 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10362 manifest: ModuleManifest,
10363 corr: u64,
10364 ) {
10365 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
10366 }
10367
10368 async fn open_route_for_capability_test(
10369 handler: &ControlHandler,
10370 target_ctx: &RouteCtx,
10371 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10372 client_connection_id: u64,
10373 corr: u64,
10374 target_module_id: &str,
10375 consumer_identity: Option<ConsumerIdentity>,
10376 ) -> (
10377 mpsc::Receiver<crate::router::OutboundFrame>,
10378 ModuleControlRequest,
10379 ) {
10380 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
10381 let route_handler = handler.clone();
10382 let target_module_id = target_module_id.to_string();
10383 let route_task = tokio::spawn(async move {
10384 route_handler
10385 .handle_control_frame(
10386 &client_ctx,
10387 route_open_frame_with_admission_facts(
10388 corr,
10389 &target_module_id,
10390 unique_project_root("admission-facts"),
10391 consumer_identity,
10392 None,
10393 ),
10394 )
10395 .await
10396 .expect("capability test route.open succeeds")
10397 });
10398 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
10399 .await
10400 .expect("capability test route.open must reach route.bind")
10401 .expect("target control receiver stays open");
10402 let bind_request: ModuleControlRequest =
10403 serde_json::from_slice(&bind.body).expect("route.bind decodes");
10404 handler
10405 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
10406 .await
10407 .expect("capability test route.bind ACK succeeds");
10408 assert!(route_task.await.expect("route.open task joins").is_empty());
10409 let opened = client_rx
10410 .recv()
10411 .await
10412 .expect("successful route.open publishes a response");
10413 assert!(matches!(
10414 serde_json::from_slice::<ClientControlResponse>(&opened.body),
10415 Ok(ClientControlResponse::RouteOpen { .. })
10416 ));
10417 (client_rx, bind_request)
10418 }
10419
10420 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
10421 assert_eq!(frame.header.ty, FrameType::Push);
10422 assert_eq!(frame.header.channel, 0);
10423 assert_eq!(
10424 serde_json::from_slice::<ClientControlPush>(&frame.body)
10425 .expect("route.closed control push decodes"),
10426 ClientControlPush::RouteClosed {
10427 module_id: target_module_id.to_string(),
10428 reason: RouteCloseReason::CapabilityDenied,
10429 drained: false,
10430 abandoned: 0,
10431 excluded_subscriptions: 0,
10432 terminal: Some(false),
10433 }
10434 );
10435 }
10436
10437 #[tokio::test]
10438 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
10439 let registry = Arc::new(Registry::default());
10440 let forwarding = Arc::new(ForwardingTable::default());
10441 let supervisor = SupervisorHandle::new();
10442 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10443 let handler =
10444 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10445 .with_supervisor(supervisor);
10446 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
10447 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
10448 register_capability_manifest(
10449 &handler,
10450 &target_ctx,
10451 &mut target_rx,
10452 capability_manifest("target", &["credentials-provider/v1"], &[]),
10453 1,
10454 )
10455 .await;
10456 register_capability_manifest(
10457 &handler,
10458 &opener_ctx,
10459 &mut opener_rx,
10460 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10461 2,
10462 )
10463 .await;
10464
10465 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
10466 let replies = handler
10467 .handle_control_frame(
10468 &client_ctx,
10469 route_open_frame_with_admission_facts(
10470 3,
10471 "target",
10472 unique_project_root("admission-facts"),
10473 Some(ConsumerIdentity {
10474 module_id: "opener".to_string(),
10475 launch_nonce: "opener-nonce".to_string(),
10476 }),
10477 None,
10478 ),
10479 )
10480 .await
10481 .expect("denied route.open returns a typed frame");
10482 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10483 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10484 assert!(
10485 target_rx.try_recv().is_err(),
10486 "forbidden route.open must not relay route.bind"
10487 );
10488 }
10489
10490 #[tokio::test]
10491 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
10492 let registry = Arc::new(Registry::default());
10493 let forwarding = Arc::new(ForwardingTable::default());
10494 let supervisor = SupervisorHandle::new();
10495 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10496 let handler =
10497 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10498 .with_supervisor(supervisor);
10499 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
10500 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
10501 register_capability_manifest(
10502 &handler,
10503 &target_ctx,
10504 &mut target_rx,
10505 capability_manifest("target", &["credentials-provider/v1"], &[]),
10506 1,
10507 )
10508 .await;
10509 register_capability_manifest(
10510 &handler,
10511 &old_opener_ctx,
10512 &mut old_opener_rx,
10513 capability_manifest("opener", &[], &[]),
10514 2,
10515 )
10516 .await;
10517 let (mut client_rx, _) = open_route_for_capability_test(
10518 &handler,
10519 &target_ctx,
10520 &mut target_rx,
10521 712,
10522 3,
10523 "target",
10524 Some(ConsumerIdentity {
10525 module_id: "opener".to_string(),
10526 launch_nonce: "opener-nonce".to_string(),
10527 }),
10528 )
10529 .await;
10530 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10531
10532 handler
10533 .cleanup_connection(old_opener_ctx.connection_id)
10534 .expect("old opener registration cleans up");
10535 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
10536 register_capability_manifest(
10537 &handler,
10538 &new_opener_ctx,
10539 &mut new_opener_rx,
10540 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10541 4,
10542 )
10543 .await;
10544
10545 assert_capability_denied_push(
10546 client_rx
10547 .try_recv()
10548 .expect("HELLO deny addition must emit route.closed")
10549 .frame,
10550 "target",
10551 );
10552 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10553 assert!(matches!(
10554 target_rx.try_recv(),
10555 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10556 ));
10557 }
10558
10559 #[tokio::test]
10560 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
10561 let registry = Arc::new(Registry::default());
10562 let forwarding = Arc::new(ForwardingTable::default());
10563 let supervisor = SupervisorHandle::new();
10564 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10565 let handler =
10566 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10567 .with_supervisor(supervisor);
10568 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
10569 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
10570 register_capability_manifest(
10571 &handler,
10572 &target_ctx,
10573 &mut target_rx,
10574 capability_manifest("target", &[], &[]),
10575 1,
10576 )
10577 .await;
10578 register_capability_manifest(
10579 &handler,
10580 &opener_ctx,
10581 &mut opener_rx,
10582 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10583 2,
10584 )
10585 .await;
10586 let (mut client_rx, _) = open_route_for_capability_test(
10587 &handler,
10588 &target_ctx,
10589 &mut target_rx,
10590 722,
10591 3,
10592 "target",
10593 Some(ConsumerIdentity {
10594 module_id: "opener".to_string(),
10595 launch_nonce: "opener-nonce".to_string(),
10596 }),
10597 )
10598 .await;
10599 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10600
10601 let replies = handler
10602 .handle_control_frame(
10603 &target_ctx,
10604 catalog_update_with_capabilities_frame(
10605 4,
10606 CapabilityDeclarations {
10607 provides: vec!["credentials-provider/v1".to_string()],
10608 requires: Vec::new(),
10609 must_never_reach: Vec::new(),
10610 },
10611 ),
10612 )
10613 .await
10614 .expect("claim catalog.update succeeds");
10615 assert!(matches!(
10616 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
10617 Ok(ModuleControlResponseToModule::CatalogUpdate {})
10618 ));
10619 assert_capability_denied_push(
10620 client_rx
10621 .try_recv()
10622 .expect("claim addition must emit route.closed")
10623 .frame,
10624 "target",
10625 );
10626 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10627 assert!(matches!(
10628 target_rx.try_recv(),
10629 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10630 ));
10631 }
10632
10633 #[tokio::test]
10634 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
10635 let registry = Arc::new(Registry::default());
10636 let forwarding = Arc::new(ForwardingTable::default());
10637 let supervisor = SupervisorHandle::new();
10638 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10639 let handler =
10640 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10641 .with_supervisor(supervisor);
10642 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
10643 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
10644 register_capability_manifest(
10645 &handler,
10646 &target_ctx,
10647 &mut target_rx,
10648 capability_manifest("target", &["credentials-provider/v1"], &[]),
10649 1,
10650 )
10651 .await;
10652 register_capability_manifest(
10653 &handler,
10654 &opener_ctx,
10655 &mut opener_rx,
10656 capability_manifest("opener", &[], &[]),
10657 2,
10658 )
10659 .await;
10660 let (mut client_rx, _) = open_route_for_capability_test(
10661 &handler,
10662 &target_ctx,
10663 &mut target_rx,
10664 732,
10665 3,
10666 "target",
10667 Some(ConsumerIdentity {
10668 module_id: "opener".to_string(),
10669 launch_nonce: "opener-nonce".to_string(),
10670 }),
10671 )
10672 .await;
10673 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10674
10675 handler
10676 .handle_control_frame(
10677 &target_ctx,
10678 catalog_update_with_capabilities_frame(
10679 4,
10680 CapabilityDeclarations {
10681 provides: Vec::new(),
10682 requires: Vec::new(),
10683 must_never_reach: Vec::new(),
10684 },
10685 ),
10686 )
10687 .await
10688 .expect("claim removal catalog.update succeeds");
10689 assert_eq!(
10690 forwarding.active_binding_count().unwrap(),
10691 1,
10692 "removing an attested target claim must leave the route census unchanged"
10693 );
10694 assert!(
10695 client_rx.try_recv().is_err(),
10696 "claim removal must not emit route.closed capability_denied"
10697 );
10698 assert!(
10699 target_rx.try_recv().is_err(),
10700 "claim removal must not send the target a route GOODBYE"
10701 );
10702 }
10703
10704 #[tokio::test]
10707 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
10708 let registry = Arc::new(Registry::default());
10709 let forwarding = Arc::new(ForwardingTable::default());
10710 let supervisor = SupervisorHandle::new();
10711 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10712 let handler =
10713 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10714 .with_supervisor(supervisor);
10715 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
10716 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
10717 register_capability_manifest(
10718 &handler,
10719 &target_ctx,
10720 &mut target_rx,
10721 capability_manifest("target", &["credentials-provider/v1"], &[]),
10722 1,
10723 )
10724 .await;
10725 register_capability_manifest(
10726 &handler,
10727 &opener_ctx,
10728 &mut opener_rx,
10729 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10730 2,
10731 )
10732 .await;
10733
10734 let (_client_rx, bind) = open_route_for_capability_test(
10735 &handler,
10736 &target_ctx,
10737 &mut target_rx,
10738 742,
10739 3,
10740 "target",
10741 None,
10742 )
10743 .await;
10744 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10745 panic!("direct scope-honesty route must bind");
10746 };
10747 assert_eq!(principal, Some(Principal::Direct));
10748 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10749 }
10750
10751 #[tokio::test]
10754 async fn must_never_reach_self_route_is_capability_forbidden() {
10755 let registry = Arc::new(Registry::default());
10756 let forwarding = Arc::new(ForwardingTable::default());
10757 let supervisor = SupervisorHandle::new();
10758 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
10759 let handler =
10760 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10761 .with_supervisor(supervisor);
10762 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
10763 register_capability_manifest(
10764 &handler,
10765 &self_ctx,
10766 &mut self_rx,
10767 capability_manifest(
10768 "self-provider",
10769 &["credentials-provider/v1"],
10770 &["credentials-provider/v1"],
10771 ),
10772 1,
10773 )
10774 .await;
10775
10776 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
10777 let replies = handler
10778 .handle_control_frame(
10779 &client_ctx,
10780 route_open_frame_with_admission_facts(
10781 2,
10782 "self-provider",
10783 unique_project_root("admission-facts"),
10784 Some(ConsumerIdentity {
10785 module_id: "self-provider".to_string(),
10786 launch_nonce: "self-nonce".to_string(),
10787 }),
10788 None,
10789 ),
10790 )
10791 .await
10792 .expect("self-route refusal returns a typed frame");
10793 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10794 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10795 assert!(
10796 self_rx.try_recv().is_err(),
10797 "self denial must not relay route.bind"
10798 );
10799 }
10800
10801 #[test]
10802 fn unsupported_channel_zero_frame_returns_error() {
10803 let handler = ControlHandler::default();
10804 let request = Frame::build(
10805 FrameType::Request,
10806 control_flags(),
10807 0,
10808 0,
10809 21,
10810 b"opaque".to_vec(),
10811 )
10812 .unwrap();
10813
10814 let response = handler
10815 .handle_control(ConnectionId::new(1), request)
10816 .unwrap();
10817
10818 assert_eq!(response[0].header.ty, FrameType::Error);
10819 assert_eq!(
10820 parse_error(&response[0])["code"],
10821 "unsupported_control_frame"
10822 );
10823 }
10824
10825 mod swap {
10830 use super::*;
10831
10832 const INCUMBENT: ConnectionId = ConnectionId::new(30);
10833 const CANDIDATE: ConnectionId = ConnectionId::new(40);
10834
10835 struct Swap {
10836 registry: Arc<Registry>,
10837 forwarding: Arc<ForwardingTable>,
10838 handler: ControlHandler,
10839 incumbent_ctx: RouteCtx,
10840 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10841 candidate_ctx: RouteCtx,
10842 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10843 }
10844
10845 async fn swap_with_incumbent() -> Swap {
10846 let registry = Arc::new(Registry::default());
10847 let forwarding = Arc::new(ForwardingTable::default());
10848 let handler =
10849 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10850 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
10851 hello_via_sink(
10852 &handler,
10853 &incumbent_ctx,
10854 &mut incumbent_rx,
10855 hello_frame("aft", PROTOCOL_VERSION, 7),
10856 )
10857 .await;
10858 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
10859 Swap {
10860 registry,
10861 forwarding,
10862 handler,
10863 incumbent_ctx,
10864 incumbent_rx,
10865 candidate_ctx,
10866 candidate_rx,
10867 }
10868 }
10869
10870 fn register_candidate(swap: &Swap, ready: Option<bool>) {
10871 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
10872 candidate_manifest.ready = ready;
10873 let registration = swap
10874 .registry
10875 .register_candidate_with_control_ops(
10876 candidate_manifest,
10877 PROTOCOL_VERSION,
10878 CANDIDATE,
10879 module_baseline_control_ops(),
10880 )
10881 .unwrap();
10882 swap.forwarding
10883 .register_candidate_module_connection(
10884 CANDIDATE,
10885 "aft".to_string(),
10886 PROTOCOL_VERSION,
10887 manifest_concurrency(®istration.manifest),
10888 swap.candidate_ctx.egress.clone(),
10889 )
10890 .unwrap();
10891 }
10892
10893 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
10894 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
10895 swap.registry.promote_candidate("aft").unwrap().unwrap();
10896 cutover.incumbent.unwrap()
10897 }
10898
10899 fn keyed_total(counters: &Value, key: &str) -> u64 {
10900 counters[key]
10901 .as_object()
10902 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
10903 .unwrap_or(0)
10904 }
10905
10906 #[tokio::test]
10912 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
10913 let mut swap = swap_with_incumbent().await;
10914 let handler = swap.handler.clone();
10915
10916 let cotenant = ConnectionId::new(31);
10918 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10919 let (cotenant_task, cotenant_bind) = relay_route_open(
10920 &handler,
10921 cotenant,
10922 &cotenant_ctx.egress,
10923 &mut swap.incumbent_rx,
10924 100,
10925 "aft",
10926 "swap-cotenant",
10927 )
10928 .await;
10929 handler
10930 .handle_control_frame(
10931 &swap.incumbent_ctx,
10932 route_bind_ack(cotenant_bind.header.corr),
10933 )
10934 .await
10935 .unwrap();
10936 assert!(cotenant_task.await.unwrap().is_empty());
10937 let (cotenant_channel, cotenant_epoch) =
10938 published_route(&cotenant_rx.recv().await.unwrap());
10939
10940 let caller = ConnectionId::new(32);
10942 let (caller_ctx, mut caller_rx) = route_ctx(caller);
10943 let (caller_task, caller_bind) = relay_route_open(
10944 &handler,
10945 caller,
10946 &caller_ctx.egress,
10947 &mut swap.incumbent_rx,
10948 101,
10949 "aft",
10950 "swap-caller",
10951 )
10952 .await;
10953 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
10954
10955 register_candidate(&swap, None);
10956 cutover(&swap);
10957
10958 let ack = handler
10960 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
10961 .await;
10962 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10963 if module_loop_error.is_some() {
10964 handler.cleanup_connection(INCUMBENT).unwrap();
10967 }
10968
10969 assert!(
10971 cotenant_rx.try_recv().is_err(),
10972 "the co-tenant route on the incumbent was torn down by one late ack: \
10973 {module_loop_error:?}"
10974 );
10975 assert!(matches!(
10976 swap.forwarding
10977 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10978 .unwrap(),
10979 DataRoute::Client(DataRouteState::Bound(_))
10980 ));
10981 assert_eq!(module_loop_error, None);
10982 assert!(swap
10983 .registry
10984 .get_module_by_connection(INCUMBENT)
10985 .unwrap()
10986 .is_some());
10987
10988 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
10990 .await
10991 .expect("the incumbent is told to drop the abandoned binding")
10992 .unwrap()
10993 .frame;
10994 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10995 assert_eq!(goodbye.header.channel, abandoned_channel);
10996 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10997 assert!(swap.incumbent_rx.try_recv().is_err());
10998
10999 let response = caller_task.await.unwrap();
11001 assert_eq!(response.len(), 1);
11002 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
11003 assert!(caller_rx.try_recv().is_err());
11004
11005 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
11009 let counters = handler.counters().snapshot();
11010 assert_eq!(
11011 keyed_total(&counters, "route_open_accepted_by_principal"),
11012 1
11013 );
11014 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
11015 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
11016 }
11017
11018 #[tokio::test]
11022 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
11023 let mut swap = swap_with_incumbent().await;
11024 register_candidate(&swap, None);
11025 let incumbent = cutover(&swap);
11026 swap.forwarding
11027 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
11028 .unwrap()
11029 .expect("the incumbent is still registered");
11030
11031 let client = ConnectionId::new(33);
11032 let (client_ctx, mut client_rx) = route_ctx(client);
11033 let route_handler = swap.handler.clone();
11034 let open_ctx = RouteCtx {
11035 connection_id: client,
11036 egress: client_ctx.egress.clone(),
11037 };
11038 let mut route_task = tokio::spawn(async move {
11039 route_handler
11040 .handle_control_frame(
11041 &open_ctx,
11042 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
11043 )
11044 .await
11045 .unwrap()
11046 });
11047 let bind = tokio::select! {
11048 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
11049 response = &mut route_task => {
11050 let response = response.unwrap();
11051 panic!(
11052 "post-cutover route.open was refused instead of relayed to the candidate: {}",
11053 parse_error(&response[0])["code"]
11054 );
11055 }
11056 };
11057 swap.handler
11058 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
11059 .await
11060 .unwrap();
11061 assert!(route_task.await.unwrap().is_empty());
11062 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
11063 match swap
11064 .forwarding
11065 .lookup_data_route(client, channel, epoch)
11066 .unwrap()
11067 {
11068 DataRoute::Client(DataRouteState::Bound(route)) => {
11069 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
11070 }
11071 other => panic!("expected a bound route on the candidate, got {other:?}"),
11072 }
11073 assert!(swap.incumbent_rx.try_recv().is_err());
11074 }
11075
11076 #[tokio::test]
11081 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
11082 let swap = swap_with_incumbent().await;
11083 register_candidate(&swap, Some(false));
11084 let update = Frame::build(
11085 FrameType::Request,
11086 control_flags(),
11087 0,
11088 0,
11089 55,
11090 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11091 provides: manifest("aft", PROTOCOL_VERSION).provides,
11092 capabilities: None,
11093 ready: Some(true),
11094 })
11095 .unwrap(),
11096 )
11097 .unwrap();
11098
11099 let replies = swap
11100 .handler
11101 .handle_control_frame(&swap.candidate_ctx, update)
11102 .await
11103 .unwrap();
11104
11105 assert_eq!(replies.len(), 1);
11106 assert_eq!(
11107 replies[0].header.ty,
11108 FrameType::Response,
11109 "candidate catalog.update was refused: {:?}",
11110 serde_json::from_slice::<Value>(&replies[0].body).ok()
11111 );
11112 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
11113 assert_eq!(
11114 swap.registry
11115 .get_module("aft")
11116 .unwrap()
11117 .unwrap()
11118 .connection_id,
11119 INCUMBENT
11120 );
11121 }
11122 }
11123
11124 mod swap_admission {
11128 use super::*;
11129
11130 const INCUMBENT_NONCE: &str = "incumbent-nonce";
11131 const CANDIDATE_NONCE: &str = "candidate-nonce";
11132
11133 fn handler_with_incumbent(
11134 module_id: &str,
11135 reserved: bool,
11136 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
11137 let registry = Arc::new(Registry::default());
11138 let supervisor = SupervisorHandle::new();
11139 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
11140 if reserved {
11141 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
11142 }
11143 let handler =
11144 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
11145 let incumbent = handler
11146 .handle_control(
11147 ConnectionId::new(1),
11148 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
11149 )
11150 .unwrap();
11151 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
11152 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
11153 (registry, supervisor, handler)
11154 }
11155
11156 #[test]
11162 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
11163 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
11164
11165 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
11166 let replies = handler
11167 .handle_control(
11168 ConnectionId::new(connection),
11169 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
11170 )
11171 .unwrap();
11172 assert_eq!(replies[0].header.ty, FrameType::Error);
11173 assert_eq!(
11174 parse_error(&replies[0])["code"],
11175 "swap_token_invalid",
11176 "nonce {nonce:?}"
11177 );
11178 }
11179 assert!(registry.get_candidate("aft").unwrap().is_none());
11180 assert_eq!(
11181 registry.get_module("aft").unwrap().unwrap().connection_id,
11182 ConnectionId::new(1)
11183 );
11184
11185 let admitted = handler
11188 .handle_control(
11189 ConnectionId::new(4),
11190 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
11191 )
11192 .unwrap();
11193 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
11194 assert_eq!(
11195 registry
11196 .get_candidate("aft")
11197 .unwrap()
11198 .unwrap()
11199 .connection_id,
11200 ConnectionId::new(4)
11201 );
11202 assert_eq!(
11203 registry.get_module("aft").unwrap().unwrap().connection_id,
11204 ConnectionId::new(1),
11205 "the candidate must not take the active slot"
11206 );
11207 let replayed = handler
11208 .handle_control(
11209 ConnectionId::new(5),
11210 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
11211 )
11212 .unwrap();
11213 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
11214
11215 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
11219 let squatter = handler
11220 .handle_control(
11221 ConnectionId::new(6),
11222 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
11223 )
11224 .unwrap();
11225 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
11226 assert!(
11227 registry.get_module("aft").unwrap().is_none(),
11228 "a squatter took the active slot of an id being swapped"
11229 );
11230 }
11231
11232 #[test]
11237 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
11238 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
11239
11240 let replies = handler
11241 .handle_control(
11242 ConnectionId::new(2),
11243 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
11244 )
11245 .unwrap();
11246
11247 assert_eq!(
11248 replies[0].header.ty,
11249 FrameType::HelloAck,
11250 "reserved candidate refused: {:?}",
11251 serde_json::from_slice::<Value>(&replies[0].body).ok()
11252 );
11253 assert_eq!(
11254 registry
11255 .get_candidate("vault")
11256 .unwrap()
11257 .unwrap()
11258 .connection_id,
11259 ConnectionId::new(2)
11260 );
11261 }
11262
11263 #[test]
11266 fn without_an_open_swap_the_ordinary_gates_decide() {
11267 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
11268 supervisor.close_swap("vault");
11269
11270 let candidate = handler
11271 .handle_control(
11272 ConnectionId::new(2),
11273 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
11274 )
11275 .unwrap();
11276 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
11277 let duplicate = handler
11278 .handle_control(
11279 ConnectionId::new(3),
11280 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
11281 )
11282 .unwrap();
11283 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11284 assert!(registry.get_candidate("vault").unwrap().is_none());
11285 }
11286 }
11287
11288 mod scopes {
11292 use subc_protocol::scope::{
11293 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
11294 ScopeStatus,
11295 };
11296
11297 use super::*;
11298
11299 const OWNER: &str = "prefrontal-core";
11300
11301 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
11302 ScopeRecord {
11303 scope_ref: scope_ref.to_string(),
11304 scope_epoch,
11305 kind: ScopeKind::Head,
11306 parent: None,
11307 child_owners: Vec::new(),
11308 carriers: Vec::new(),
11309 attributes: Default::default(),
11310 }
11311 }
11312
11313 async fn call(
11314 handler: &ControlHandler,
11315 ctx: &RouteCtx,
11316 request: &ModuleControlRequestFromModule,
11317 ) -> Frame {
11318 let body = serde_json::to_vec(request).unwrap();
11319 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
11320 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
11321 assert_eq!(replies.len(), 1, "{replies:?}");
11322 replies.pop().unwrap()
11323 }
11324
11325 async fn sync(
11326 handler: &ControlHandler,
11327 ctx: &RouteCtx,
11328 generation: u64,
11329 scopes: Vec<ScopeRecord>,
11330 ) -> Result<ModuleControlResponseToModule, String> {
11331 let reply = call(
11332 handler,
11333 ctx,
11334 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
11335 )
11336 .await;
11337 match reply.header.ty {
11338 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
11339 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
11340 }
11341 }
11342
11343 async fn describe(
11344 handler: &ControlHandler,
11345 ctx: &RouteCtx,
11346 owner: &str,
11347 scope_ref: &str,
11348 ) -> ModuleControlResponseToModule {
11349 let reply = call(
11350 handler,
11351 ctx,
11352 &ModuleControlRequestFromModule::ScopeDescribe {
11353 owner: Principal::Reserved {
11354 module_id: owner.to_string(),
11355 },
11356 scope_ref: scope_ref.to_string(),
11357 },
11358 )
11359 .await;
11360 assert_eq!(
11361 reply.header.ty,
11362 FrameType::Response,
11363 "{:?}",
11364 parse_error(&reply)
11365 );
11366 serde_json::from_slice(&reply.body).unwrap()
11367 }
11368
11369 async fn module(
11371 handler: &ControlHandler,
11372 connection: u64,
11373 module_id: &str,
11374 nonce: Option<&str>,
11375 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11376 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
11377 hello_via_sink(
11378 handler,
11379 &ctx,
11380 &mut rx,
11381 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
11382 )
11383 .await;
11384 (ctx, rx)
11385 }
11386
11387 #[tokio::test]
11390 async fn a_client_connection_cannot_sync_or_describe() {
11391 let handler = ControlHandler::new(Arc::new(Registry::default()));
11392 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
11393 for request in [
11394 ModuleControlRequestFromModule::ScopeSync {
11395 generation: 1,
11396 scopes: vec![head("s", 1)],
11397 },
11398 ModuleControlRequestFromModule::ScopeDescribe {
11399 owner: Principal::Direct,
11400 scope_ref: "s".to_string(),
11401 },
11402 ] {
11403 let reply = call(&handler, &ctx, &request).await;
11404 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
11405 }
11406 assert!(
11407 !handler
11408 .scopes
11409 .read()
11410 .unwrap()
11411 .describe(
11412 &Principal::Reserved {
11413 module_id: OWNER.to_string()
11414 },
11415 "s"
11416 )
11417 .owner_synced
11418 );
11419 }
11420
11421 #[tokio::test]
11424 async fn a_module_without_a_supervised_launch_cannot_sync() {
11425 let handler = ControlHandler::new(Arc::new(Registry::default()));
11426 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
11427 assert_eq!(
11428 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
11429 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11430 );
11431 }
11432
11433 #[tokio::test]
11434 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
11435 let supervisor = SupervisorHandle::new();
11436 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
11437 let handler = ControlHandler::new(Arc::new(Registry::default()))
11438 .with_supervisor(supervisor.clone());
11439 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
11440 sync(&handler, &incumbent, 1, vec![head("s", 1)])
11441 .await
11442 .expect("the current launch syncs");
11443
11444 supervisor.open_swap(OWNER, "n2".to_string());
11447 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
11448 assert_eq!(
11449 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
11450 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11451 );
11452 sync(&handler, &incumbent, 2, vec![head("s", 1)])
11453 .await
11454 .expect("the serving owner syncs during the swap");
11455
11456 supervisor.close_swap(OWNER);
11459 assert_eq!(
11460 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
11461 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11462 );
11463 sync(&handler, &incumbent, 3, vec![head("s", 1)])
11464 .await
11465 .expect("the serving owner syncs after the rollback");
11466 handler.cleanup_connection(candidate.connection_id).unwrap();
11467
11468 supervisor.open_swap(OWNER, "n3".to_string());
11473 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
11474 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
11475 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
11476 .await
11477 .expect("the promoted launch takes authority");
11478 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
11479 panic!("unexpected reply {reply:?}");
11480 };
11481 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
11482 assert_eq!(
11483 sync(&handler, &incumbent, 4, Vec::new()).await,
11484 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11485 );
11486 }
11487
11488 #[tokio::test]
11491 async fn closing_the_authority_connection_frees_sync_authority() {
11492 let supervisor = SupervisorHandle::new();
11493 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
11494 let handler = ControlHandler::new(Arc::new(Registry::default()))
11495 .with_supervisor(supervisor.clone());
11496 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
11497 sync(&handler, &first, 10, vec![head("s", 1)])
11498 .await
11499 .unwrap();
11500 handler.cleanup_connection(first.connection_id).unwrap();
11501
11502 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
11503 sync(&handler, &second, 1, vec![head("s", 1)])
11504 .await
11505 .expect("the next connection takes the released authority");
11506 }
11507
11508 #[tokio::test]
11509 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
11510 let registry = Arc::new(Registry::default());
11511 let supervisor_handle = SupervisorHandle::new();
11512 let supervisor = Supervisor::new(Arc::clone(®istry), RestartPolicy::default())
11513 .with_handle(supervisor_handle.clone())
11514 .with_daemon_incarnation("incarnation-7".to_string());
11515 supervisor
11518 .supervise_configured(
11519 ModuleSpec {
11520 launch_nonce_env: true,
11521 module_id: OWNER.to_string(),
11522 program: PathBuf::from("/nonexistent/prefrontal-core"),
11523 args: Vec::new(),
11524 env: Vec::new(),
11525 reserved: false,
11526 reserved_prefixes: Vec::new(),
11527 protocol: ModuleProtocol::Subc,
11528 overlap: Default::default(),
11529 },
11530 false,
11531 )
11532 .unwrap();
11533 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
11534 let handler =
11535 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11536 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
11537
11538 let ModuleControlResponseToModule::ScopeDescribe {
11540 status,
11541 daemon_incarnation,
11542 owner_synced,
11543 owner_configured,
11544 scope,
11545 ..
11546 } = describe(&handler, &reader, OWNER, "s").await
11547 else {
11548 panic!("not a describe reply");
11549 };
11550 assert_eq!(status, ScopeStatus::NotLive);
11551 assert_eq!(daemon_incarnation, "incarnation-7");
11552 assert!(!owner_synced);
11553 assert!(owner_configured);
11554 assert!(scope.is_none());
11555
11556 let ModuleControlResponseToModule::ScopeDescribe {
11559 status,
11560 owner_configured,
11561 ..
11562 } = describe(&handler, &reader, "ghost", "s").await
11563 else {
11564 panic!("not a describe reply");
11565 };
11566 assert_eq!(status, ScopeStatus::NotLive);
11567 assert!(!owner_configured);
11568
11569 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
11571 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
11572 let ModuleControlResponseToModule::ScopeDescribe {
11573 status,
11574 scope_epoch,
11575 owner_synced,
11576 scope,
11577 ..
11578 } = describe(&handler, &reader, OWNER, "s").await
11579 else {
11580 panic!("not a describe reply");
11581 };
11582 assert_eq!(status, ScopeStatus::Live);
11583 assert_eq!(scope_epoch, Some(4));
11584 assert!(owner_synced);
11585 let stamp = scope.expect("a live scope carries its stamp");
11586 assert!(
11587 stamp.owner_authorized,
11588 "prefrontal-core is the default authority"
11589 );
11590 assert_eq!(stamp.kind, ScopeKind::Head);
11591 }
11592
11593 #[tokio::test]
11594 async fn scope_authority_owners_decides_owner_authorized() {
11595 let supervisor = SupervisorHandle::new();
11596 supervisor.set_spawn_nonce("broca", "b1".to_string());
11597 let handler = ControlHandler::new(Arc::new(Registry::default()))
11598 .with_supervisor(supervisor)
11599 .with_scope_authority_owners(vec!["broca".to_string()]);
11600 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
11601 let mut gated = head("s", 1);
11602 gated.attributes.agent_id = Some("agent".to_string());
11603 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
11604 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
11605 describe(&handler, &broca, "broca", "s").await
11606 else {
11607 panic!("not a describe reply");
11608 };
11609 assert!(scope.unwrap().owner_authorized);
11610 }
11611
11612 #[tokio::test]
11616 async fn scope_ops_and_the_scopes_capability_are_advertised() {
11617 let handler = ControlHandler::new(Arc::new(Registry::default()));
11618 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
11619 let ack = hello_via_sink(
11620 &handler,
11621 &ctx,
11622 &mut rx,
11623 hello_frame("m", PROTOCOL_VERSION, 1),
11624 )
11625 .await;
11626 let ack = parse_ack(&ack);
11627 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
11628 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
11629 }
11630 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
11631
11632 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
11633 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11634 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11635 let reply = handler
11636 .handle_control_frame(&client, frame)
11637 .await
11638 .unwrap()
11639 .pop()
11640 .unwrap();
11641 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11642 serde_json::from_slice(&reply.body).unwrap()
11643 else {
11644 panic!("not a server.describe reply");
11645 };
11646 assert!(
11647 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
11648 "{capabilities:?}"
11649 );
11650 }
11651
11652 const PLEXUS: &str = "plexus";
11655 const OTHER: &str = "other";
11656 const AFT: &str = "aft";
11657 const BROCA: &str = "broca";
11658 const MAGIC: &str = "magic-context";
11659
11660 fn nonce(module_id: &str) -> String {
11661 format!("nonce-{module_id}")
11662 }
11663
11664 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11665 let (tx, rx) = mpsc::channel(64);
11666 (
11667 RouteCtx {
11668 connection_id: ConnectionId::new(connection),
11669 egress: FrameSink::new(tx),
11670 },
11671 rx,
11672 )
11673 }
11674
11675 struct Rig {
11679 handler: ControlHandler,
11680 forwarding: Arc<ForwardingTable>,
11681 owner: RouteCtx,
11682 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11683 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
11684 generation: u64,
11685 next_connection: u64,
11686 _supervisor: Supervisor,
11687 }
11688
11689 async fn rig() -> Rig {
11690 let registry = Arc::new(Registry::default());
11691 let forwarding = Arc::new(ForwardingTable::default());
11692 let supervisor_handle = SupervisorHandle::new();
11693 let supervisor = Supervisor::new(Arc::clone(®istry), RestartPolicy::default())
11694 .with_handle(supervisor_handle.clone());
11695 supervisor
11696 .supervise_configured(
11697 ModuleSpec {
11698 launch_nonce_env: true,
11699 module_id: OWNER.to_string(),
11700 program: PathBuf::from("/nonexistent/prefrontal-core"),
11701 args: Vec::new(),
11702 env: Vec::new(),
11703 reserved: false,
11704 reserved_prefixes: Vec::new(),
11705 protocol: ModuleProtocol::Subc,
11706 overlap: Default::default(),
11707 },
11708 false,
11709 )
11710 .unwrap();
11711 for module_id in [OWNER, AFT, BROCA, MAGIC] {
11712 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
11713 }
11714 let handler =
11715 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11716 .with_supervisor(supervisor_handle);
11717 let (owner, mut owner_rx) = wide_ctx(1);
11718 hello_via_sink(
11719 &handler,
11720 &owner,
11721 &mut owner_rx,
11722 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
11723 )
11724 .await;
11725 let mut modules = BTreeMap::new();
11726 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
11727 let (ctx, mut rx) = wide_ctx(connection);
11728 hello_via_sink(
11729 &handler,
11730 &ctx,
11731 &mut rx,
11732 hello_frame(module_id, PROTOCOL_VERSION, connection),
11733 )
11734 .await;
11735 modules.insert(module_id.to_string(), (ctx, rx));
11736 }
11737 Rig {
11738 handler,
11739 forwarding,
11740 owner,
11741 _owner_rx: owner_rx,
11742 modules,
11743 generation: 0,
11744 next_connection: 100,
11745 _supervisor: supervisor,
11746 }
11747 }
11748
11749 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
11750 ScopeCarrier {
11751 principal: Principal::Reserved {
11752 module_id: module_id.to_string(),
11753 },
11754 targets: targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()),
11755 }
11756 }
11757
11758 fn session(scope_epoch: u64) -> ScopeRecord {
11761 let mut record = head("s", scope_epoch);
11762 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
11763 record.attributes.agent_id = Some("agent-1".to_string());
11764 record.attributes.delegates = true;
11765 record
11766 }
11767
11768 impl Rig {
11769 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
11770 self.generation += 1;
11771 sync(&self.handler, &self.owner, self.generation, scopes)
11772 .await
11773 .expect("the owner's sync is accepted");
11774 }
11775
11776 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
11777 ScopeSelector {
11778 owner: Principal::Reserved {
11779 module_id: OWNER.to_string(),
11780 },
11781 scope_ref: scope_ref.to_string(),
11782 scope_epoch,
11783 }
11784 }
11785
11786 fn open_frame(
11787 &mut self,
11788 opener: Option<&str>,
11789 target: &str,
11790 scope: Option<ScopeSelector>,
11791 ) -> (
11792 RouteCtx,
11793 mpsc::Receiver<crate::router::OutboundFrame>,
11794 Frame,
11795 ) {
11796 self.next_connection += 1;
11797 let (ctx, rx) = wide_ctx(self.next_connection);
11798 let root = unique_project_root("scoped-open");
11799 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11800 target: RouteTarget::ToolProvider {
11801 module_id: target.to_string(),
11802 },
11803 identity: BindIdentity::new(
11804 root.path().to_path_buf(),
11805 "unit".to_string(),
11806 "session".to_string(),
11807 ),
11808 consumer_identity: opener.map(|module_id| ConsumerIdentity {
11809 module_id: module_id.to_string(),
11810 launch_nonce: nonce(module_id),
11811 }),
11812 consumer_capabilities: None,
11813 admission_facts: None,
11814 scope,
11815 })
11816 .unwrap();
11817 let frame = Frame::build(
11818 FrameType::Request,
11819 control_flags(),
11820 0,
11821 0,
11822 self.next_connection,
11823 body,
11824 )
11825 .unwrap();
11826 (ctx, rx, frame)
11827 }
11828
11829 async fn refused(
11831 &mut self,
11832 opener: Option<&str>,
11833 target: &str,
11834 scope: Option<ScopeSelector>,
11835 ) -> String {
11836 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
11837 let replies = self
11838 .handler
11839 .handle_control_frame(&ctx, frame)
11840 .await
11841 .unwrap();
11842 assert_eq!(replies.len(), 1, "{replies:?}");
11843 assert_eq!(replies[0].header.ty, FrameType::Error);
11844 let (_, module_rx) = self.modules.get_mut(target).unwrap();
11845 assert!(
11846 module_rx.try_recv().is_err(),
11847 "a refused open relays nothing"
11848 );
11849 parse_error(&replies[0])["code"]
11850 .as_str()
11851 .unwrap()
11852 .to_string()
11853 }
11854
11855 async fn relayed(
11857 &mut self,
11858 opener: Option<&str>,
11859 target: &str,
11860 scope: Option<ScopeSelector>,
11861 ) -> Relayed {
11862 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
11863 let handler = self.handler.clone();
11864 let task_ctx = ctx.clone();
11865 let task = tokio::spawn(async move {
11866 handler
11867 .handle_control_frame(&task_ctx, frame)
11868 .await
11869 .unwrap()
11870 });
11871 let (_, module_rx) = self.modules.get_mut(target).unwrap();
11872 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
11873 .await
11874 .expect("the target receives the relayed route.bind")
11875 .unwrap()
11876 .frame;
11877 Relayed {
11878 target: target.to_string(),
11879 client: ctx,
11880 client_rx: rx,
11881 task,
11882 bind,
11883 }
11884 }
11885
11886 async fn ack(&self, relayed: &Relayed) {
11887 let (module, _) = &self.modules[&relayed.target];
11888 self.handler
11889 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
11890 .await
11891 .unwrap();
11892 }
11893
11894 async fn bound(
11896 &mut self,
11897 opener: Option<&str>,
11898 target: &str,
11899 scope: Option<ScopeSelector>,
11900 ) -> Bound {
11901 let relayed = self.relayed(opener, target, scope).await;
11902 self.ack(&relayed).await;
11903 let Relayed {
11904 target,
11905 client,
11906 mut client_rx,
11907 task,
11908 bind,
11909 } = relayed;
11910 assert!(
11911 task.await.unwrap().is_empty(),
11912 "the open is answered by commit"
11913 );
11914 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
11915 Bound {
11916 target,
11917 client,
11918 client_rx,
11919 channel,
11920 epoch,
11921 bind,
11922 }
11923 }
11924
11925 fn live(&self, route: &Bound) -> bool {
11926 matches!(
11927 self.forwarding
11928 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
11929 .unwrap(),
11930 DataRoute::Client(DataRouteState::Bound(_))
11931 )
11932 }
11933 }
11934
11935 struct Relayed {
11936 target: String,
11937 client: RouteCtx,
11938 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11939 task: tokio::task::JoinHandle<Vec<Frame>>,
11940 bind: Frame,
11941 }
11942
11943 struct Bound {
11944 target: String,
11945 client: RouteCtx,
11946 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11947 channel: u16,
11948 epoch: u32,
11949 bind: Frame,
11950 }
11951
11952 impl Bound {
11953 fn closed_reason(&mut self) -> RouteCloseReason {
11956 let mut reason = None;
11957 let mut goodbye = false;
11958 while let Ok(outbound) = self.client_rx.try_recv() {
11959 let frame = outbound.frame;
11960 match frame.header.ty {
11961 FrameType::Goodbye => {
11962 assert_eq!(
11963 (frame.header.channel, frame.header.epoch),
11964 (self.channel, self.epoch)
11965 );
11966 goodbye = true;
11967 }
11968 FrameType::Push => {
11969 let ClientControlPush::RouteClosed {
11970 reason: r,
11971 module_id,
11972 ..
11973 } = serde_json::from_slice(&frame.body).unwrap()
11974 else {
11975 panic!("unexpected push");
11976 };
11977 assert_eq!(module_id, self.target);
11978 reason = Some(r);
11979 }
11980 other => panic!("unexpected frame {other:?}"),
11981 }
11982 }
11983 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
11984 reason.expect("the client is told why the route closed")
11985 }
11986
11987 fn untouched(&mut self) -> bool {
11988 self.client_rx.try_recv().is_err()
11989 }
11990
11991 fn stamp(&self) -> Option<ScopeStamp> {
11992 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
11993 ModuleControlRequest::RouteBind { scope, .. } => scope,
11994 other => panic!("expected a route.bind, got {other:?}"),
11995 }
11996 }
11997 }
11998
11999 #[tokio::test]
12000 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
12001 ) {
12002 let mut rig = rig().await;
12003 let mut record = session(1);
12004 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
12005 record.child_owners = vec![Principal::Reserved {
12006 module_id: MAGIC.to_string(),
12007 }];
12008 rig.sync(vec![record]).await;
12009 let scope = || Some(rig_selector("s", Some(1)));
12010
12011 rig.bound(Some(OWNER), PLEXUS, scope()).await;
12014 rig.bound(Some(AFT), OTHER, scope()).await;
12015 rig.bound(Some(BROCA), PLEXUS, scope()).await;
12016
12017 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
12021 assert_eq!(
12022 rig.refused(opener, target, scope()).await,
12023 error_codes::SCOPE_NOT_CARRIER,
12024 "{opener:?} -> {target}"
12025 );
12026 }
12027 }
12028
12029 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12030 ScopeSelector {
12031 owner: Principal::Reserved {
12032 module_id: OWNER.to_string(),
12033 },
12034 scope_ref: scope_ref.to_string(),
12035 scope_epoch,
12036 }
12037 }
12038
12039 #[tokio::test]
12040 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
12041 let mut rig = rig().await;
12042 rig.sync(vec![session(1)]).await;
12043 for opener in [OWNER, AFT] {
12044 assert_eq!(
12045 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
12046 .await,
12047 error_codes::SCOPE_EPOCH_REQUIRED,
12048 "{opener}"
12049 );
12050 }
12051 }
12052
12053 #[tokio::test]
12054 async fn admission_separates_not_synced_not_live_and_ended() {
12055 let mut rig = rig().await;
12056 let code = rig
12058 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12059 .await;
12060 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
12061 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
12062
12063 let ghost = ScopeSelector {
12065 owner: Principal::Reserved {
12066 module_id: "ghost".to_string(),
12067 },
12068 scope_ref: "s".to_string(),
12069 scope_epoch: Some(1),
12070 };
12071 assert_eq!(
12072 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
12073 error_codes::SCOPE_NOT_LIVE
12074 );
12075
12076 rig.sync(vec![session(2)]).await;
12077 assert_eq!(
12078 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
12079 .await,
12080 error_codes::SCOPE_NOT_LIVE
12081 );
12082 for epoch in [1, 3] {
12083 assert_eq!(
12084 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
12085 .await,
12086 error_codes::SCOPE_ENDED,
12087 "epoch {epoch}"
12088 );
12089 }
12090 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
12092 .await;
12093 }
12094
12095 #[tokio::test]
12096 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
12097 let mut rig = rig().await;
12098 rig.sync(vec![session(1)]).await;
12099 let route = rig
12100 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12101 .await;
12102 let stamp = route.stamp().expect("a scoped bind carries the stamp");
12103 assert_eq!(stamp.scope_ref, "s");
12104 assert_eq!(stamp.scope_epoch, 1);
12105 assert_eq!(stamp.kind, ScopeKind::Head);
12106 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
12107 assert!(stamp.attributes.delegates);
12108 assert!(stamp.owner_authorized);
12109 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
12110 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
12111
12112 let (broca, mut broca_rx) = wide_ctx(50);
12115 hello_via_sink(
12116 &rig.handler,
12117 &broca,
12118 &mut broca_rx,
12119 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
12120 )
12121 .await;
12122 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
12123 .await
12124 .unwrap();
12125 let own = ScopeSelector {
12126 owner: Principal::Reserved {
12127 module_id: BROCA.to_string(),
12128 },
12129 scope_ref: "b".to_string(),
12130 scope_epoch: Some(1),
12131 };
12132 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
12133 assert!(!route.stamp().unwrap().owner_authorized);
12134 }
12135
12136 #[tokio::test]
12141 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
12142 let mut rig = rig().await;
12143 rig.sync(vec![session(1)]).await;
12144 let mut cotenant = rig
12145 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
12146 .await;
12147
12148 let mut changed = session(1);
12149 changed.child_owners.push(Principal::Reserved {
12150 module_id: MAGIC.to_string(),
12151 });
12152 let mut ended = None;
12153 for (code, next) in [
12154 (error_codes::SCOPE_CHANGED, vec![changed]),
12155 (error_codes::SCOPE_ENDED, Vec::new()),
12156 ] {
12157 let relayed = rig
12158 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12159 .await;
12160 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
12161 ended = Some(next.is_empty());
12162 rig.sync(next).await;
12163 rig.ack(&relayed).await;
12164 let replies = relayed.task.await.unwrap();
12165 assert_eq!(replies.len(), 1, "{replies:?}");
12166 assert_eq!(parse_error(&replies[0])["code"], code);
12167 assert_eq!(
12168 subc_protocol::error_codes::is_retryable_route_open(code),
12169 code == error_codes::SCOPE_CHANGED
12170 );
12171 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
12172 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12174 let mut goodbyes = Vec::new();
12177 while let Ok(outbound) = plexus_rx.try_recv() {
12178 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
12179 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
12180 }
12181 assert!(
12182 goodbyes.contains(&(bind_channel, bind_epoch)),
12183 "{goodbyes:?}"
12184 );
12185 assert!(rig
12186 .handler
12187 .registry
12188 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
12189 .unwrap()
12190 .is_some());
12191 }
12192 assert_eq!(ended, Some(true));
12193 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
12196 }
12197
12198 #[tokio::test]
12201 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
12202 struct Case {
12203 name: &'static str,
12204 change: fn(&mut ScopeRecord),
12205 closed: [Option<RouteCloseReason>; 4],
12208 }
12209 use RouteCloseReason::*;
12210 let cases = [
12211 Case {
12212 name: "a carrier entry removed",
12213 change: |r| {
12214 r.carriers.retain(|c| {
12215 c.principal
12216 != Principal::Reserved {
12217 module_id: AFT.to_string(),
12218 }
12219 })
12220 },
12221 closed: [None, Some(ScopeCarrierRemoved), None, None],
12222 },
12223 Case {
12224 name: "a target removed from a carrier",
12225 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
12226 closed: [None, None, None, Some(ScopeCarrierRemoved)],
12227 },
12228 Case {
12229 name: "a bare carrier narrowed to targets",
12230 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
12231 closed: [None, Some(ScopeCarrierRemoved), None, None],
12232 },
12233 Case {
12234 name: "delegates turned off",
12235 change: |r| r.attributes.delegates = false,
12236 closed: [Some(ScopeDelegationChanged); 4],
12237 },
12238 Case {
12239 name: "agent_id changed",
12240 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
12241 closed: [Some(ScopeDelegationChanged); 4],
12242 },
12243 Case {
12244 name: "a carrier added, child owners changed, the record re-sent",
12245 change: |r| {
12246 r.carriers.push(carrier(MAGIC, None));
12247 r.child_owners.push(Principal::Reserved {
12248 module_id: MAGIC.to_string(),
12249 });
12250 },
12251 closed: [None; 4],
12252 },
12253 Case {
12254 name: "a target added",
12255 change: |r| {
12256 r.carriers[1]
12257 .targets
12258 .as_mut()
12259 .unwrap()
12260 .push("third".to_string())
12261 },
12262 closed: [None; 4],
12263 },
12264 Case {
12265 name: "delegates turned on",
12266 change: |r| r.attributes.delegates = true,
12267 closed: [None; 4],
12268 },
12269 ];
12270 for case in cases {
12271 let mut rig = rig().await;
12272 rig.sync(vec![session(1)]).await;
12273 let scope = || Some(rig_selector("s", Some(1)));
12274 let mut routes = [
12275 rig.bound(Some(OWNER), PLEXUS, scope()).await,
12276 rig.bound(Some(AFT), PLEXUS, scope()).await,
12277 rig.bound(Some(BROCA), PLEXUS, scope()).await,
12278 rig.bound(Some(BROCA), OTHER, scope()).await,
12279 ];
12280 let mut record = session(1);
12281 (case.change)(&mut record);
12282 rig.sync(vec![record]).await;
12283 for (index, expected) in case.closed.iter().enumerate() {
12284 let route = &mut routes[index];
12285 match expected {
12286 Some(reason) => {
12287 assert!(!rig.live(route), "{}: route {index} still live", case.name);
12288 assert_eq!(
12289 route.closed_reason(),
12290 *reason,
12291 "{}: route {index}",
12292 case.name
12293 );
12294 }
12295 None => {
12296 assert!(rig.live(route), "{}: route {index} closed", case.name);
12297 assert!(
12298 route.untouched(),
12299 "{}: route {index} was told something",
12300 case.name
12301 );
12302 }
12303 }
12304 }
12305 }
12306 }
12307
12308 #[tokio::test]
12309 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
12310 for next in [Vec::new(), vec![session(2)]] {
12312 let mut rig = rig().await;
12313 rig.sync(vec![session(1)]).await;
12314 let mut route = rig
12315 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12316 .await;
12317 rig.sync(next).await;
12318 assert!(!rig.live(&route));
12319 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
12320 }
12321
12322 let mut rig = rig().await;
12325 let mut child = session(1);
12326 child.scope_ref = "child".to_string();
12327 child.kind = ScopeKind::Worker;
12328 child.parent = Some(ScopeParent {
12329 owner: Principal::Reserved {
12330 module_id: OWNER.to_string(),
12331 },
12332 scope_ref: "s".to_string(),
12333 scope_epoch: 1,
12334 });
12335 rig.sync(vec![session(1), child.clone()]).await;
12336 let mut child_route = rig
12337 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
12338 .await;
12339 assert_eq!(
12340 child_route.stamp().unwrap().parent_state,
12341 Some(ParentState::Linked)
12342 );
12343 rig.sync(vec![child]).await;
12344 assert!(!rig.live(&child_route));
12345 assert_eq!(
12346 child_route.closed_reason(),
12347 RouteCloseReason::ScopeParentEnded
12348 );
12349 }
12350
12351 #[tokio::test]
12352 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
12353 ) {
12354 let mut rig = rig().await;
12355 rig.sync(vec![session(1)]).await;
12356 let mut route = rig
12357 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12358 .await;
12359 let before = rig.forwarding.published_scope_tag(OWNER, "s");
12360 rig.sync(vec![session(1)]).await;
12361 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
12362 assert!(rig.live(&route) && route.untouched());
12363
12364 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
12368 .forwarding
12369 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
12370 .unwrap()
12371 else {
12372 panic!("the route is bound");
12373 };
12374 binding.flow.acquire_tagged(9, false).await.unwrap();
12375 let mut widened = session(1);
12376 widened.carriers.push(carrier(MAGIC, None));
12377 rig.sync(vec![widened]).await;
12378 assert!(rig.live(&route) && route.untouched());
12379 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12380 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
12381 assert_eq!(binding.flow.in_flight(), 1);
12384 binding
12385 .flow
12386 .acquire_tagged(10, false)
12387 .await
12388 .expect("the flow is still open");
12389 }
12390
12391 #[tokio::test]
12394 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
12395 let mut rig = rig().await;
12396 rig.sync(vec![session(1)]).await;
12397 let mut on_incumbent = rig
12398 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12399 .await;
12400
12401 let (candidate, _candidate_rx) = wide_ctx(9);
12404 let registration = rig
12405 .handler
12406 .registry
12407 .register_candidate_with_control_ops(
12408 manifest(PLEXUS, PROTOCOL_VERSION),
12409 PROTOCOL_VERSION,
12410 candidate.connection_id,
12411 module_baseline_control_ops(),
12412 )
12413 .unwrap();
12414 rig.forwarding
12415 .register_candidate_module_connection(
12416 candidate.connection_id,
12417 PLEXUS.to_string(),
12418 PROTOCOL_VERSION,
12419 manifest_concurrency(®istration.manifest),
12420 candidate.egress.clone(),
12421 )
12422 .unwrap();
12423 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
12424 rig.handler
12425 .registry
12426 .promote_candidate(PLEXUS)
12427 .unwrap()
12428 .unwrap();
12429 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
12430
12431 rig.sync(Vec::new()).await;
12432 assert!(!rig.live(&on_incumbent));
12433 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
12434 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12435 let goodbye = incumbent_rx
12436 .try_recv()
12437 .expect("the superseded endpoint is told")
12438 .frame;
12439 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12440 }
12441 }
12442}
12443
12444#[cfg(test)]
12445mod concurrency_default_exposure_tests {
12446 use super::*;
12447
12448 fn hello_body(role_json: &str) -> Vec<u8> {
12449 format!(
12450 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
12451 )
12452 .into_bytes()
12453 }
12454
12455 fn manifest_from(body: &[u8]) -> ModuleManifest {
12456 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
12457 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
12458 .expect("manifest parses")
12459 }
12460
12461 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
12462
12463 #[test]
12464 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
12465 let body = hello_body(&format!(
12466 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
12467 ));
12468 let manifest = manifest_from(&body);
12469 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
12472 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
12473 }
12474
12475 #[test]
12476 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
12477 let body = hello_body(&format!(
12478 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
12479 ));
12480 let manifest = manifest_from(&body);
12481 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
12482 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
12483 }
12484
12485 #[test]
12486 fn non_management_roles_are_never_reported() {
12487 let body = hello_body(
12488 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
12489 );
12490 let manifest = manifest_from(&body);
12491 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
12492 }
12493}