1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_self_signal_declarations,
23 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
24 ProviderRole,
25 },
26 scope::{ScopeRecord, ScopeSelector, CAP_SCOPES_V1, SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP},
27 session::{
28 HealthReport, ModuleControlPush, ModuleControlRequest, ModuleControlRequestFromModule,
29 ModuleControlResponse, ModuleControlResponseToModule, MODULE_CONTROL_OP_HEALTH_CHECK,
30 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
31 },
32 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
33 Priority, RouteTarget, PROTOCOL_VERSION,
34};
35use tokio::time::{timeout_at, Instant};
36use tracing::{debug, info, warn};
37
38use crate::{
39 capability_requirements::{
40 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
41 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
42 RuntimeModule,
43 },
44 daemon_config::RestartRequiredSection,
45 forwarding::{
46 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
47 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
48 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
49 },
50 observability::{
51 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
52 },
53 provenance::{
54 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
55 },
56 registry::{ChannelState, ConnectionId, Registry, RegistryError},
57 router::{RouteCtx, RouterError},
58 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
59 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
60 stderr_tail::{CaptureState, TailEntry},
61 supervise::{
62 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
63 SupervisorHandle, SwapHelloAdmission,
64 },
65 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
66};
67
68pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
74
75const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
76const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
77const CAP_PING_PONG: &str = "ping_pong_v1";
78const CAP_SESSION_ATTACH: &str = "session_attach_v1";
79const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
80
81const SUBC_CONTROL_OPS: &[&str] = &[
82 ops::SERVER_DESCRIBE,
83 ops::CATALOG_LIST,
84 ops::ROUTE_OPEN,
85 ops::ROUTE_POLL,
86 ops::ROUTE_CLOSING,
87 ops::ROUTE_CLOSED,
88 ops::SUPERVISOR_LIST,
89 ops::SUPERVISOR_RESTART,
90 ops::SUPERVISOR_SWAP,
91 ops::SUPERVISOR_RELOAD,
92 ops::SUPERVISOR_RESCAN,
93 ops::SUPERVISOR_RELEASE_RESERVED,
94 ops::SUPERVISOR_SET_ENABLED,
95 ops::SUPERVISOR_HEALTH_PROBE,
96 ops::SUPERVISOR_HEALTH,
97 ops::SUPERVISOR_STDERR_TAIL,
98 ops::SUPERVISOR_TERMINALS,
99 ops::SUPERVISOR_ROUTES,
100 ops::SUPERVISOR_PROVENANCE,
101 ops::SUPERVISOR_SPAWN_SNAPSHOT,
102 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
103];
104
105const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
106 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
107 "supervisor.live_roots",
108 SCOPE_SYNC_OP,
109 SCOPE_DESCRIBE_OP,
110];
111
112const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
116
117const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
118
119pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
127
128pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
143
144pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
162
163const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
164const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
165
166fn reload_verdict(
167 configured: &Path,
168 spawned_from: Option<&Path>,
169 image: subc_control::RunningImageAgreement,
170) -> PendingReloadVerdict {
171 let path = match spawned_from {
172 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
173 Some(spawned_from) => ReloadPathAgreement::Mismatch {
174 configured: configured.to_path_buf(),
175 spawned_from: spawned_from.to_path_buf(),
176 },
177 None => ReloadPathAgreement::Unavailable {
178 reason: if matches!(
179 image,
180 subc_control::RunningImageAgreement::Unavailable {
181 reason: subc_control::RunningImageUnavailableReason::NotRunning
182 }
183 ) {
184 ReloadPathUnavailableReason::NotRunning
185 } else {
186 ReloadPathUnavailableReason::SpawnedPathUnavailable
187 },
188 },
189 };
190 PendingReloadVerdict { path, image }
191}
192
193#[derive(Clone)]
194struct DaemonProvenanceFacts {
195 build: DaemonBuildProvenance,
196 pid: Option<u32>,
197 started_at_ms: Option<u64>,
198 start_clock: Option<crate::clock::StartClock>,
199 executable_path: Option<PathBuf>,
200 executable_identity: Option<SpawnedFileIdentity>,
201 process_start_time: Option<u64>,
202 probe: ExecutableIdentityProbe,
203}
204
205impl Default for DaemonProvenanceFacts {
206 fn default() -> Self {
207 Self {
208 build: DaemonBuildProvenance {
209 build_git_sha: None,
210 build_lock_digest: None,
211 },
212 pid: None,
213 started_at_ms: None,
214 start_clock: None,
215 executable_path: None,
216 executable_identity: None,
217 process_start_time: None,
218 probe: ExecutableIdentityProbe::default(),
219 }
220 }
221}
222
223#[derive(Debug, Clone)]
224struct SupervisorRescanContext {
225 supervisor: Supervisor,
226 config_path: PathBuf,
227 configured_port: Option<u16>,
228 storage_config: Option<crate::daemon_config::StorageConfig>,
229 admission_facts_carrier_module_id: Option<String>,
230 admission_facts_targets: Option<Vec<String>>,
231 scope_authority_owners: Vec<String>,
232}
233
234const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
255 "reloading",
256 "supervisor_not_live",
257 "registration_not_active",
258 "no_forwarding_connection",
259 "relay_send_failed",
260];
261
262#[derive(Clone)]
264pub struct ControlHandler {
265 registry: Arc<Registry>,
266 forwarding: Arc<ForwardingTable>,
267 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
268 supervisor: SupervisorHandle,
269 subc_capabilities: Arc<[String]>,
270 route_bind_relay_timeout: Duration,
274 route_bind_relay_timeouts: BTreeMap<String, Duration>,
278 route_bind_breakers: RouteBindBreakers,
281 route_bind_concurrency: RouteBindConcurrency,
284 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
289 route_bind_breaker_threshold: u32,
291 route_bind_breaker_cooldown: Duration,
293 health_probe_timeout: Duration,
294 storage_config: Option<crate::daemon_config::StorageConfig>,
297 machine_id: Option<crate::machine_id::MachineId>,
301 admission_facts_carrier_module_id: Option<String>,
302 admission_facts_targets: Option<Vec<String>>,
303 scopes: Arc<RwLock<ScopeTable>>,
307 scope_authority_owners: Vec<String>,
310 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
313 rescan: Option<SupervisorRescanContext>,
314 connected_clients: ConnectedClients,
315 counters: DaemonCounters,
316 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
317 daemon_provenance: DaemonProvenanceFacts,
318 #[cfg(test)]
319 control_dispatch_delay: Option<Duration>,
320 #[cfg(test)]
321 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
322}
323
324impl fmt::Debug for ControlHandler {
325 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
326 f.debug_struct("ControlHandler")
327 .field("registry", &self.registry)
328 .field("forwarding", &self.forwarding)
329 .field("process_liveness", &self.process_liveness.is_some())
330 .field("supervisor", &self.supervisor)
331 .field("subc_capabilities", &self.subc_capabilities)
332 .finish()
333 }
334}
335
336struct RouteOpenRequest {
337 target: RouteTarget,
338 identity: BindIdentity,
339 consumer_identity: Option<ConsumerIdentity>,
340 consumer_capabilities: Option<Vec<String>>,
341 admission_facts: Option<serde_json::Value>,
342 scope: Option<ScopeSelector>,
343}
344
345struct RouteBindReservationGuard {
346 forwarding: Arc<ForwardingTable>,
347 endpoint: ModuleEndpointId,
348 relay_corr: u64,
349 armed: bool,
350}
351
352struct ModuleControlRpcGuard {
353 forwarding: Arc<ForwardingTable>,
354 endpoint: ModuleEndpointId,
355 corr: u64,
356 armed: bool,
357}
358
359impl ModuleControlRpcGuard {
360 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
361 Self {
362 forwarding,
363 endpoint,
364 corr,
365 armed: true,
366 }
367 }
368
369 fn disarm(&mut self) {
370 self.armed = false;
371 }
372}
373
374impl Drop for ModuleControlRpcGuard {
375 fn drop(&mut self) {
376 if self.armed {
377 let _ = self
378 .forwarding
379 .cancel_module_control_rpc(self.endpoint, self.corr);
380 }
381 }
382}
383
384impl RouteBindReservationGuard {
385 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
386 Self {
387 forwarding,
388 endpoint,
389 relay_corr,
390 armed: true,
391 }
392 }
393
394 fn release_and_disarm(&mut self) {
395 if !self.armed {
396 return;
397 }
398 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
399 self.endpoint,
400 self.relay_corr,
401 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
402 ) {
403 send_goodbye_target_best_effort(
404 &self.forwarding.counters(),
405 &target,
406 "canceled route.bind",
407 );
408 }
409 self.armed = false;
410 }
411
412 fn disarm(&mut self) {
413 self.armed = false;
414 }
415}
416
417impl Drop for RouteBindReservationGuard {
418 fn drop(&mut self) {
419 self.release_and_disarm();
420 }
421}
422
423#[derive(Debug, Clone, Default)]
453pub(crate) struct RouteBindBreakers {
454 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
455}
456
457#[derive(Debug, Clone, Default)]
458pub(crate) struct RouteBindConcurrency {
459 modules: Arc<Mutex<HashMap<String, usize>>>,
460}
461
462struct RouteBindConcurrencyGuard {
463 concurrency: RouteBindConcurrency,
464 module_id: String,
465}
466
467impl RouteBindConcurrency {
468 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
471 let mut modules = self
472 .modules
473 .lock()
474 .expect("route.bind concurrency mutex poisoned");
475 let in_flight = modules.entry(module_id.to_string()).or_default();
476 if *in_flight >= limit {
477 return Err(*in_flight);
478 }
479 *in_flight += 1;
480 Ok(RouteBindConcurrencyGuard {
481 concurrency: self.clone(),
482 module_id: module_id.to_string(),
483 })
484 }
485}
486
487impl Drop for RouteBindConcurrencyGuard {
488 fn drop(&mut self) {
489 let mut modules = self
490 .concurrency
491 .modules
492 .lock()
493 .expect("route.bind concurrency mutex poisoned");
494 let remove = {
495 let in_flight = modules
496 .get_mut(&self.module_id)
497 .expect("admitted route.bind has a concurrency entry");
498 *in_flight -= 1;
499 *in_flight == 0
500 };
501 if remove {
502 modules.remove(&self.module_id);
503 }
504 }
505}
506
507#[derive(Debug, Default)]
508struct ModuleBreakerState {
509 consecutive_timeouts: u32,
511 cooldown_until: Option<Instant>,
514 probe_in_flight: bool,
519}
520
521enum RouteBindAdmission<'a> {
523 Admitted {
524 guard: RouteBindBreakerGuard<'a>,
525 probe: bool,
528 },
529 Refused {
530 consecutive_timeouts: u32,
531 retry_in: Duration,
535 probe_in_flight: bool,
536 },
537}
538
539struct RouteBindBreakerGuard<'a> {
546 breakers: RouteBindBreakers,
547 module_id: &'a str,
548 settled: bool,
549}
550
551impl RouteBindBreakerGuard<'_> {
552 fn record_accepted(&mut self) -> bool {
556 self.settled = true;
557 self.breakers.record_accepted(self.module_id)
558 }
559
560 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
563 self.settled = true;
564 self.breakers
565 .record_timeout(self.module_id, threshold, cooldown)
566 }
567
568 fn record_inconclusive(&mut self) {
577 self.settled = true;
578 self.breakers.record_inconclusive(self.module_id);
579 }
580}
581
582impl Drop for RouteBindBreakerGuard<'_> {
583 fn drop(&mut self) {
584 if !self.settled {
585 self.breakers.record_inconclusive(self.module_id);
586 }
587 }
588}
589
590struct BreakerOpened {
594 consecutive_timeouts: u32,
595 reopened_after_probe: bool,
598}
599
600impl RouteBindBreakers {
601 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
602 self.modules
603 .lock()
604 .expect("route.bind breaker mutex poisoned")
605 }
606
607 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
610 let admitted = |probe| RouteBindAdmission::Admitted {
611 guard: RouteBindBreakerGuard {
612 breakers: self.clone(),
613 module_id,
614 settled: false,
615 },
616 probe,
617 };
618
619 let mut modules = self.lock();
620 let Some(state) = modules.get_mut(module_id) else {
621 return admitted(false);
622 };
623 let Some(cooldown_until) = state.cooldown_until else {
624 return admitted(false);
625 };
626 if state.probe_in_flight {
627 return RouteBindAdmission::Refused {
628 consecutive_timeouts: state.consecutive_timeouts,
629 retry_in: Duration::ZERO,
630 probe_in_flight: true,
631 };
632 }
633 let now = Instant::now();
634 if now < cooldown_until {
635 return RouteBindAdmission::Refused {
636 consecutive_timeouts: state.consecutive_timeouts,
637 retry_in: cooldown_until - now,
638 probe_in_flight: false,
639 };
640 }
641 state.probe_in_flight = true;
642 admitted(true)
643 }
644
645 fn record_accepted(&self, module_id: &str) -> bool {
646 self.lock()
647 .remove(module_id)
648 .is_some_and(|state| state.cooldown_until.is_some())
649 }
650
651 fn record_timeout(
652 &self,
653 module_id: &str,
654 threshold: u32,
655 cooldown: Duration,
656 ) -> Option<BreakerOpened> {
657 let mut modules = self.lock();
658 let state = modules.entry(module_id.to_string()).or_default();
659 let was_open = state.cooldown_until.is_some();
660 let was_probe = state.probe_in_flight;
661 state.probe_in_flight = false;
662 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
663 if state.consecutive_timeouts < threshold {
664 return None;
665 }
666 state.cooldown_until = Some(Instant::now() + cooldown);
667 Some(BreakerOpened {
668 consecutive_timeouts: state.consecutive_timeouts,
669 reopened_after_probe: was_open && was_probe,
670 })
671 }
672
673 fn record_inconclusive(&self, module_id: &str) {
674 if let Some(state) = self.lock().get_mut(module_id) {
675 state.probe_in_flight = false;
676 }
677 }
678
679 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
694 self.lock()
695 .remove(module_id)
696 .map(|state| state.consecutive_timeouts)
697 .filter(|discarded| *discarded > 0)
698 }
699
700 fn open_snapshot(&self) -> Option<serde_json::Value> {
707 let now = Instant::now();
708 let modules = self.lock();
709 let open = modules
710 .iter()
711 .filter_map(|(module_id, state)| {
712 let cooldown_until = state.cooldown_until?;
713 Some((
714 module_id.clone(),
715 serde_json::json!({
716 "consecutive_timeouts": state.consecutive_timeouts,
717 "cooldown_remaining_ms":
718 cooldown_until.saturating_duration_since(now).as_millis() as u64,
719 "probe_in_flight": state.probe_in_flight,
720 }),
721 ))
722 })
723 .collect::<serde_json::Map<String, serde_json::Value>>();
724 (!open.is_empty()).then_some(serde_json::Value::Object(open))
725 }
726}
727
728impl ControlHandler {
729 pub fn new(registry: Arc<Registry>) -> Self {
730 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
731 }
732
733 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
734 let counters = forwarding.counters();
735 let route_bind_breakers = forwarding.route_bind_breakers();
739 let route_bind_concurrency = forwarding.route_bind_concurrency();
740 let route_outages = forwarding.route_outages();
741 Self {
742 registry,
743 forwarding,
744 process_liveness: None,
745 supervisor: SupervisorHandle::new(),
746 subc_capabilities: Arc::from([
747 CAP_MANIFEST_REGISTRATION.to_string(),
748 CAP_CHANNEL_LIFECYCLE.to_string(),
749 CAP_PING_PONG.to_string(),
750 CAP_SESSION_ATTACH.to_string(),
751 CAP_ADMISSION_FACTS_RELAY.to_string(),
752 CAP_SCOPES_V1.to_string(),
753 ]),
754 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
755 route_bind_relay_timeouts: BTreeMap::new(),
756 route_bind_breakers,
757 route_bind_concurrency,
758 route_outages,
759 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
760 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
761 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
762 storage_config: None,
763 machine_id: None,
764 admission_facts_carrier_module_id: None,
765 admission_facts_targets: None,
766 scopes: Arc::new(RwLock::new(ScopeTable::new(
767 crate::daemon_config::default_scope_authority_owners(),
768 ))),
769 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
770 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
771 rescan: None,
772 connected_clients: ConnectedClients::new(),
773 counters,
774 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
775 daemon_provenance: DaemonProvenanceFacts::default(),
776 #[cfg(test)]
777 control_dispatch_delay: None,
778 #[cfg(test)]
779 provenance_probe_override: None,
780 }
781 }
782
783 pub fn with_storage_config(
786 mut self,
787 storage_config: Option<crate::daemon_config::StorageConfig>,
788 ) -> Self {
789 self.storage_config = storage_config;
790 self
791 }
792
793 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
796 self.machine_id = machine_id;
797 self
798 }
799
800 pub fn with_admission_facts_config(
804 mut self,
805 carrier_module_id: Option<String>,
806 targets: Option<Vec<String>>,
807 ) -> Self {
808 self.admission_facts_carrier_module_id = carrier_module_id;
809 self.admission_facts_targets = targets;
810 self
811 }
812
813 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
817 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
818 self.scope_authority_owners = owners;
819 self
820 }
821
822 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
825 self.route_bind_relay_timeout = timeout;
826 self
827 }
828
829 pub fn with_route_bind_relay_timeouts(
835 mut self,
836 timeouts: impl IntoIterator<Item = (String, Duration)>,
837 ) -> Self {
838 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
839 self
840 }
841
842 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
848 self.route_bind_relay_timeouts
849 .get(module_id)
850 .copied()
851 .unwrap_or(self.route_bind_relay_timeout)
852 }
853
854 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
862 self.route_bind_breaker_threshold = threshold.max(1);
863 self.route_bind_breaker_cooldown = cooldown;
864 self
865 }
866
867 #[cfg(test)]
868 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
869 self.health_probe_timeout = timeout;
870 self
871 }
872
873 #[cfg(test)]
874 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
875 self.control_dispatch_delay = Some(delay);
876 self
877 }
878
879 pub fn with_process_liveness(
880 mut self,
881 process_liveness: Arc<dyn ModuleProcessLiveness>,
882 ) -> Self {
883 self.process_liveness = Some(process_liveness);
884 self
885 }
886
887 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
888 self.supervisor = supervisor;
889 self
890 }
891
892 pub fn with_daemon_provenance(
893 mut self,
894 pid: u32,
895 started_at_ms: u64,
896 executable_path: Option<PathBuf>,
897 build_git_sha: Option<String>,
898 build_lock_digest: Option<String>,
899 ) -> Self {
900 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
901 let process_start_time = process_start_time(pid);
902 self.daemon_provenance = DaemonProvenanceFacts {
903 build: DaemonBuildProvenance {
904 build_git_sha,
905 build_lock_digest,
906 },
907 pid: Some(pid),
908 started_at_ms: Some(started_at_ms),
909 start_clock: None,
910 executable_path,
911 executable_identity,
912 process_start_time,
913 probe: ExecutableIdentityProbe::default(),
914 };
915 self
916 }
917
918 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
919 self.daemon_provenance.start_clock = Some(clock);
920 self
921 }
922
923 #[cfg(test)]
924 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
925 self.provenance_probe_override = Some(result);
926 self
927 }
928
929 pub fn with_capability_config(
933 self,
934 modules: impl IntoIterator<Item = (String, bool)>,
935 reserved_capabilities: BTreeMap<String, String>,
936 ) -> Self {
937 self.capability_evaluator
938 .configure(modules, reserved_capabilities);
939 self
940 }
941
942 pub fn with_supervisor_rescan(
943 mut self,
944 supervisor: Supervisor,
945 config_path: impl Into<PathBuf>,
946 configured_port: Option<u16>,
947 ) -> Self {
948 self.rescan = Some(SupervisorRescanContext {
949 supervisor,
950 config_path: config_path.into(),
951 configured_port,
952 storage_config: self.storage_config.clone(),
953 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
954 admission_facts_targets: self.admission_facts_targets.clone(),
955 scope_authority_owners: self.scope_authority_owners.clone(),
956 });
957 self
958 }
959
960 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
961 self.connected_clients = connected_clients;
962 self
963 }
964
965 pub fn forwarding(&self) -> Arc<ForwardingTable> {
966 Arc::clone(&self.forwarding)
967 }
968
969 pub(crate) fn counters(&self) -> DaemonCounters {
970 self.counters.clone()
971 }
972
973 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
976 tokio::spawn(async move {
977 loop {
978 self.capability_evaluator
979 .wait_for_change_or_deadline()
980 .await;
981 self.refresh_capability_requirements();
982 }
983 });
984 }
985
986 fn runtime_capability_snapshot(
987 &self,
988 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
989 let runtime = self
990 .supervisor
991 .list()
992 .into_iter()
993 .map(|module| {
994 let status = module.status().map_err(|err| {
995 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
996 })?;
997 Ok(RuntimeModule {
998 module_id: status.module_id,
999 state: status.state,
1000 enabled: status.enabled,
1001 })
1002 })
1003 .collect::<Result<Vec<_>, RouterError>>()?;
1004 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1005 RouterError::backend(
1006 0,
1007 0,
1008 format!("failed to list capability registrations: {err}"),
1009 )
1010 })?;
1011 let registrations = registrations
1012 .into_iter()
1013 .map(|registration| RegisteredModule {
1014 module_id: registration.manifest.module_id,
1015 module_version: registration.manifest.module_version,
1016 capabilities: registration.manifest.capabilities,
1017 })
1018 .collect();
1019 Ok((runtime, registrations))
1020 }
1021
1022 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1029 let cached_registration = RegisteredModule {
1030 module_id: registration.manifest.module_id.clone(),
1031 module_version: registration.manifest.module_version.clone(),
1032 capabilities: registration.manifest.capabilities.clone(),
1033 };
1034 if self.capability_evaluator.record_hello(&cached_registration) {
1035 warn!(
1036 module_id = %cached_registration.module_id,
1037 "capability claims drifted from the cached manifest"
1038 );
1039 }
1040 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1041 self.enforce_capability_denies();
1042 }
1043 self.refresh_capability_requirements();
1044 }
1045
1046 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1050 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1051 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1052 self.supervisor.set_swap_promotion_observer(observer);
1053 }
1054
1055 pub fn refresh_capability_requirements(&self) {
1056 match self.runtime_capability_snapshot() {
1057 Ok((runtime, registrations)) => {
1058 log_requirement_events(
1059 self.capability_evaluator
1060 .evaluate_now(&runtime, ®istrations),
1061 );
1062 }
1063 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1064 }
1065 }
1066
1067 fn enforce_capability_denies(&self) {
1071 let (_, registrations) = match self.registry.list_modules() {
1072 Ok(snapshot) => snapshot,
1073 Err(err) => {
1074 warn!(error = %err, "failed to read registrations for capability deny census");
1075 return;
1076 }
1077 };
1078 let manifests = registrations
1079 .into_iter()
1080 .map(|registration| {
1081 (
1082 registration.manifest.module_id.clone(),
1083 registration.manifest,
1084 )
1085 })
1086 .collect::<BTreeMap<_, _>>();
1087 let census = match self.forwarding.route_census(None) {
1088 Ok(census) => census,
1089 Err(err) => {
1090 warn!(error = %err, "failed to read route census for capability deny enforcement");
1091 return;
1092 }
1093 };
1094
1095 for (target_module_id, routes) in census {
1096 let Some(target_manifest) = manifests.get(&target_module_id) else {
1097 continue;
1098 };
1099 let mut closed_routes = Vec::new();
1100 let mut module_goodbyes = Vec::new();
1101 for route in routes {
1102 let Principal::Reserved {
1103 module_id: opening_module_id,
1104 } = &route.principal
1105 else {
1106 continue;
1107 };
1108 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1109 continue;
1110 };
1111 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1112 continue;
1113 };
1114
1115 match self.forwarding.release_client_route(
1116 route.goodbye_target.connection_id,
1117 route.goodbye_target.channel,
1118 route.goodbye_target.epoch,
1119 ) {
1120 Ok(RouteRelease::Removed(module_goodbye)) => {
1121 warn!(
1122 opening_module_id,
1123 target_module_id,
1124 capability,
1125 "force-closing route because an attested capability deny edge now matches"
1126 );
1127 closed_routes.push(route);
1128 module_goodbyes.push(module_goodbye);
1129 }
1130 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1131 Err(err) => warn!(
1132 opening_module_id,
1133 target_module_id,
1134 capability,
1135 error = %err,
1136 "failed to force-close capability-denied route"
1137 ),
1138 }
1139 }
1140
1141 if closed_routes.is_empty() {
1142 continue;
1143 }
1144 send_route_control_pushes(
1145 &self.forwarding,
1146 closed_routes,
1147 ClientControlPush::RouteClosed {
1148 module_id: target_module_id,
1149 reason: RouteCloseReason::CapabilityDenied,
1150 drained: false,
1151 abandoned: 0,
1152 excluded_subscriptions: 0,
1153 terminal: Some(false),
1154 },
1155 );
1156 self.emit_route_goodbyes(module_goodbyes);
1157 }
1158 }
1159
1160 fn not_ready_reason(
1166 &self,
1167 registration: &crate::registry::ModuleRegistration,
1168 ) -> Option<NotReadyReason> {
1169 if !registration.ready {
1170 return Some(NotReadyReason {
1171 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1172 capability: None,
1173 });
1174 }
1175 self.first_unprovided_required_capability(registration)
1176 .map(|capability| NotReadyReason {
1177 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1178 capability: Some(capability),
1179 })
1180 }
1181
1182 fn first_unprovided_required_capability(
1200 &self,
1201 registration: &crate::registry::ModuleRegistration,
1202 ) -> Option<String> {
1203 let required = registration
1204 .manifest
1205 .capabilities
1206 .iter()
1207 .flat_map(|declarations| declarations.requires.iter())
1208 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1209 .map(|requirement| requirement.capability.as_str())
1210 .collect::<BTreeSet<_>>();
1211 if required.is_empty() {
1212 return None;
1213 }
1214 let module_id = registration.manifest.module_id.as_str();
1215 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1216 if required
1217 .iter()
1218 .any(|capability| verdict(capability).is_none())
1219 {
1220 self.refresh_capability_requirements();
1221 }
1222 required
1223 .into_iter()
1224 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1225 .map(str::to_string)
1226 }
1227
1228 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1229 self.capability_evaluator
1230 .statuses()
1231 .into_iter()
1232 .map(capability_requirement_status)
1233 .collect()
1234 }
1235
1236 fn deregister_connection(
1243 &self,
1244 connection_id: ConnectionId,
1245 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1246 self.registry.deregister_connection(connection_id)
1247 }
1248
1249 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1250 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1251 return None;
1252 }
1253 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1254 parse_client_control_request(&frame.body)
1255 else {
1256 return None;
1257 };
1258 Some(target_module_id(&target).to_string())
1259 }
1260
1261 pub(crate) fn route_open_capacity_refusal(
1262 &self,
1263 ctx: &RouteCtx,
1264 frame: &Frame,
1265 target_module_id: &str,
1266 in_flight: usize,
1267 limit: usize,
1268 ) -> Result<Frame, RouterError> {
1269 self.route_open_admission_refusal_frame(
1270 ctx,
1271 frame,
1272 target_module_id,
1273 "open_admission_full",
1274 (in_flight, limit),
1275 format!(
1276 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1277 ),
1278 )
1279 }
1280
1281 fn route_open_target_capacity_refusal(
1282 &self,
1283 ctx: &RouteCtx,
1284 frame: &Frame,
1285 target_module_id: &str,
1286 in_flight: usize,
1287 ) -> Result<Frame, RouterError> {
1288 self.route_open_admission_refusal_frame(
1289 ctx,
1290 frame,
1291 target_module_id,
1292 "target_binds_full",
1293 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1294 format!(
1295 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1296 ),
1297 )
1298 }
1299
1300 fn route_open_admission_refusal_frame(
1308 &self,
1309 ctx: &RouteCtx,
1310 frame: &Frame,
1311 target_module_id: &str,
1312 reason: &'static str,
1313 (in_flight, limit): (usize, usize),
1314 message: impl Into<String>,
1315 ) -> Result<Frame, RouterError> {
1316 let code = error_codes::TARGET_UNAVAILABLE;
1317 self.counters.increment_route_open_refused(code);
1318 info!(
1319 target: "control",
1320 code,
1321 reason,
1322 module_id = ?target_module_id,
1323 connection_id = ctx.connection_id.get(),
1324 in_flight,
1325 limit,
1326 "route.open refused"
1327 );
1328 control_error_frame(frame, code, message.into())
1329 }
1330
1331 #[cfg(test)]
1337 pub fn handle_control(
1338 &self,
1339 connection_id: ConnectionId,
1340 frame: Frame,
1341 ) -> Result<Vec<Frame>, RouterError> {
1342 match frame.header.ty {
1343 FrameType::Ping => Ok(vec![pong(&frame)?]),
1344 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1345 FrameType::Goodbye => self.handle_goodbye(connection_id),
1346 ty => Ok(vec![control_error_frame(
1347 &frame,
1348 "unsupported_control_frame",
1349 format!("unsupported channel-0 frame {ty:?}"),
1350 )?]),
1351 }
1352 }
1353
1354 pub async fn handle_control_frame(
1355 &self,
1356 ctx: &RouteCtx,
1357 frame: Frame,
1358 ) -> Result<Vec<Frame>, RouterError> {
1359 self.handle_control_frame_timed(ctx, frame, None).await
1360 }
1361
1362 pub(crate) async fn handle_control_frame_timed(
1363 &self,
1364 ctx: &RouteCtx,
1365 frame: Frame,
1366 dispatch_started_at: Option<StdInstant>,
1367 ) -> Result<Vec<Frame>, RouterError> {
1368 match frame.header.ty {
1369 FrameType::Ping => Ok(vec![pong(&frame)?]),
1370 FrameType::Hello => {
1371 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1372 }
1373 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1374 FrameType::Cancel => {
1375 if self
1376 .supervisor
1377 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1378 {
1379 Ok(Vec::new())
1380 } else {
1381 Ok(vec![control_error_frame(
1382 &frame,
1383 "unknown_subscription",
1384 "no supervisor spawn subscription has this correlation id",
1385 )?])
1386 }
1387 }
1388 FrameType::Request => {
1389 if self
1390 .forwarding
1391 .module_endpoint_for_connection(ctx.connection_id)
1392 .map_err(RouterError::Forwarding)?
1393 .is_some()
1394 {
1395 if !is_known_module_request_op(&frame.body) {
1396 return Ok(vec![control_error_frame(
1397 &frame,
1398 "unsupported_control_frame",
1399 "module-originated channel-0 REQUEST is not supported",
1400 )?]);
1401 }
1402 let request = match parse_module_control_request_from_module(&frame.body) {
1403 Ok(request) => request,
1404 Err((err, ControlRequestBodyError::UnknownOp)) => {
1405 return Ok(vec![control_error_frame(
1406 &frame,
1407 "unsupported_control_frame",
1408 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1409 )?])
1410 }
1411 Err((err, ControlRequestBodyError::InvalidBody)) => {
1412 return Ok(vec![control_error_frame(
1413 &frame,
1414 "invalid_control_body",
1415 format!("malformed module control body: {err}"),
1416 )?])
1417 }
1418 };
1419 let op = module_control_request_op(&request);
1420 let corr = frame.header.corr;
1421 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1422 let result =
1423 self.handle_module_control_request(ctx.connection_id, frame, request);
1424 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1425 return result;
1426 }
1427
1428 if is_known_module_request_op(&frame.body) {
1429 return Ok(vec![control_error_frame(
1430 &frame,
1431 "not_registered",
1432 "catalog.update requires an active module registration owned by this connection",
1433 )?]);
1434 }
1435
1436 let request = match parse_client_control_request(&frame.body) {
1437 Ok(request) => request,
1438 Err((err, ControlRequestBodyError::UnknownOp)) => {
1439 return Ok(vec![control_error_frame(
1440 &frame,
1441 "unknown_control_op",
1442 format!("unknown client control op: {err}"),
1443 )?])
1444 }
1445 Err((err, ControlRequestBodyError::InvalidBody)) => {
1446 return Ok(vec![control_error_frame(
1447 &frame,
1448 "invalid_control_body",
1449 format!("malformed client control body: {err}"),
1450 )?])
1451 }
1452 };
1453 let op = client_control_request_op(&request);
1454 let corr = frame.header.corr;
1455 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1456 #[cfg(test)]
1457 if let Some(delay) = self.control_dispatch_delay {
1458 tokio::time::sleep(delay).await;
1459 }
1460 let result = self
1461 .handle_client_control_request(ctx, frame, request)
1462 .await;
1463 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1464 result
1465 }
1466 FrameType::Push => {
1467 let Some(endpoint) = self
1468 .forwarding
1469 .module_endpoint_for_connection(ctx.connection_id)
1470 .map_err(RouterError::Forwarding)?
1471 else {
1472 return Ok(vec![control_error_frame(
1473 &frame,
1474 "unsupported_control_frame",
1475 "client-originated channel-0 PUSH is not supported",
1476 )?]);
1477 };
1478 self.handle_status_update(endpoint, frame)
1479 }
1480 FrameType::Response | FrameType::Error
1481 if self
1482 .forwarding
1483 .module_endpoint_for_connection(ctx.connection_id)
1484 .map_err(RouterError::Forwarding)?
1485 .is_some() =>
1486 {
1487 self.handle_module_relay_response(ctx.connection_id, frame)
1488 }
1489 ty => Ok(vec![control_error_frame(
1490 &frame,
1491 "unsupported_control_frame",
1492 format!("unsupported channel-0 frame {ty:?}"),
1493 )?]),
1494 }
1495 }
1496
1497 pub fn cleanup_connection(
1498 &self,
1499 connection_id: ConnectionId,
1500 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1501 let crash_closed = self
1502 .registry
1503 .get_module_by_connection(connection_id)?
1504 .and_then(|registration| {
1505 self.forwarding
1506 .module_endpoint_for_connection(connection_id)
1507 .ok()
1508 .flatten()
1509 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1510 .map(|routes| (registration.manifest.module_id, routes))
1511 });
1512 let crash_closed = crash_closed.map(|(module_id, routes)| {
1513 let terminal = match self.supervisor.get(&module_id) {
1514 None => false,
1515 Some(module) => match module.will_recover_after_connection_loss() {
1516 Ok(will_recover) => !will_recover,
1517 Err(err) => {
1518 warn!(
1519 %module_id,
1520 error = %err,
1521 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1522 );
1523 false
1524 }
1525 },
1526 };
1527 let reason = match self.forwarding.is_daemon_draining() {
1531 Ok(true) => RouteCloseReason::Restart,
1532 Ok(false) => RouteCloseReason::Crash,
1533 Err(err) => {
1534 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1535 RouteCloseReason::Crash
1536 }
1537 };
1538 (module_id, routes, reason, terminal)
1539 });
1540 let registrations = self.deregister_connection(connection_id);
1541 let cleanup = self.forwarding.cleanup_connection_counted(connection_id);
1542 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1547 let abandoned = cleanup
1548 .as_ref()
1549 .map_or(0, |cleanup| cleanup.abandoned_relays);
1550 send_route_control_pushes(
1551 &self.forwarding,
1552 routes,
1553 ClientControlPush::RouteClosed {
1554 module_id,
1555 reason,
1556 drained: false,
1557 abandoned,
1558 excluded_subscriptions: 0,
1559 terminal: Some(terminal),
1560 },
1561 );
1562 }
1563 if let Ok(cleanup) = cleanup {
1564 self.emit_route_goodbyes(cleanup.released);
1565 }
1566 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1570 crate::supervise::notify_registration_release();
1571 self.capability_evaluator.wake_deadline_loop();
1572 self.refresh_capability_requirements();
1573 }
1574 self.supervisor.remove_spawn_subscribers(connection_id);
1575 self.hello_launch_nonces
1578 .lock()
1579 .unwrap_or_else(|poisoned| poisoned.into_inner())
1580 .forget(connection_id);
1581 self.scopes
1582 .write()
1583 .unwrap_or_else(|poisoned| poisoned.into_inner())
1584 .release_connection(connection_id);
1585 registrations
1586 }
1587
1588 pub(crate) fn handle_route_goodbye(
1589 &self,
1590 connection_id: ConnectionId,
1591 route_channel: u16,
1592 route_epoch: u32,
1593 ) -> Result<bool, RouterError> {
1594 debug!(
1595 connection_id = connection_id.get(),
1596 route_channel, route_epoch, "handling route GOODBYE"
1597 );
1598 let RouteRelease::Removed(released_route) = self
1599 .forwarding
1600 .release_client_route(connection_id, route_channel, route_epoch)
1601 .map_err(RouterError::Forwarding)?
1602 else {
1603 return Ok(false);
1604 };
1605 self.emit_route_goodbyes(vec![released_route]);
1606 Ok(true)
1607 }
1608
1609 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1610 for released in released_routes {
1611 let frame = match Frame::build_with_version(
1612 released.negotiated_ver,
1613 FrameType::Goodbye,
1614 control_flags(),
1615 released.channel,
1616 released.epoch,
1617 0,
1618 Vec::new(),
1619 ) {
1620 Ok(frame) => frame,
1621 Err(err) => {
1622 warn!(
1623 route_channel = released.channel,
1624 error = %err,
1625 "failed to build route GOODBYE frame"
1626 );
1627 continue;
1628 }
1629 };
1630 if !released.close_on_delivery_failure() {
1631 crate::forwarding::send_module_route_goodbye(
1632 &self.counters,
1633 &released.sink,
1634 frame,
1635 released.module_id.as_deref(),
1636 "client route released",
1637 );
1638 continue;
1639 }
1640 if let Err(err) = released.sink.try_send(frame) {
1641 warn!(
1642 target_connection_id = released.connection_id.get(),
1643 route_channel = released.channel,
1644 error = %err,
1645 "route GOODBYE was not delivered to client; closing target connection"
1646 );
1647 if self
1648 .forwarding
1649 .escalate_client_delivery_failure(
1650 released.connection_id,
1651 released.channel,
1652 released.epoch,
1653 CloseReason::new(
1654 "route_goodbye_delivery_failed",
1655 format!(
1656 "failed to enqueue route GOODBYE for channel {}: {err}",
1657 released.channel
1658 ),
1659 ),
1660 crate::forwarding::UndeliveredFrame {
1661 module_id: released.module_id.as_deref(),
1662 sink: &released.sink,
1663 },
1664 )
1665 .unwrap_or(false)
1666 {
1667 self.counters.increment_goodbye_relay_client_failed();
1668 }
1669 }
1670 }
1671 }
1672
1673 fn send_abandoned_route_bind_goodbye(
1685 &self,
1686 module_sink: &crate::FrameSink,
1687 negotiated_ver: u8,
1688 module_channel: u16,
1689 module_epoch: u32,
1690 ) {
1691 let frame = match Frame::build_with_version(
1692 negotiated_ver,
1693 FrameType::Goodbye,
1694 control_flags(),
1695 module_channel,
1696 module_epoch,
1697 0,
1698 Vec::new(),
1699 ) {
1700 Ok(frame) => frame,
1701 Err(err) => {
1702 warn!(
1703 route_channel = module_channel,
1704 error = %err,
1705 "failed to build GOODBYE for abandoned route.bind"
1706 );
1707 return;
1708 }
1709 };
1710 crate::forwarding::send_module_route_goodbye(
1711 &self.counters,
1712 module_sink,
1713 frame,
1714 None,
1715 "abandoned route.bind",
1716 );
1717 }
1718
1719 fn handle_hello(
1720 &self,
1721 connection_id: ConnectionId,
1722 sink: Option<crate::FrameSink>,
1723 frame: Frame,
1724 ) -> Result<Vec<Frame>, RouterError> {
1725 debug!(
1726 connection_id = connection_id.get(),
1727 corr = frame.header.corr,
1728 "handling HELLO"
1729 );
1730 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1731 Ok(value) => value,
1732 Err(err) => {
1733 return Ok(vec![control_error_frame(
1734 &frame,
1735 "invalid_hello",
1736 format!("malformed HELLO body: {err}"),
1737 )?])
1738 }
1739 };
1740 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1741 return Ok(vec![control_error_frame(
1742 &frame,
1743 "invalid_capability_grammar",
1744 err.to_string(),
1745 )?]);
1746 }
1747 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1748 return Ok(vec![control_error_frame(
1749 &frame,
1750 "invalid_manifest",
1751 err.to_string(),
1752 )?]);
1753 }
1754 if let Some(provenance) = hello_value
1755 .get("manifest")
1756 .and_then(|manifest| manifest.get("provenance"))
1757 {
1758 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1759 return Ok(vec![control_error_frame(
1760 &frame,
1761 "invalid_manifest",
1762 format!("malformed manifest provenance: {err}"),
1763 )?]);
1764 }
1765 }
1766 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1767 Ok(hello) => hello,
1768 Err(err) => {
1769 return Ok(vec![control_error_frame(
1770 &frame,
1771 "invalid_hello",
1772 format!("malformed HELLO body: {err}"),
1773 )?])
1774 }
1775 };
1776
1777 if hello.protocol_ver != hello.manifest.protocol_ver {
1778 return Ok(vec![control_error_frame(
1779 &frame,
1780 "invalid_manifest",
1781 format!(
1782 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1783 hello.protocol_ver, hello.manifest.protocol_ver
1784 ),
1785 )?]);
1786 }
1787
1788 if hello.manifest.module_id.trim().is_empty() {
1789 return Ok(vec![control_error_frame(
1790 &frame,
1791 "invalid_manifest",
1792 "manifest module_id must not be empty",
1793 )?]);
1794 }
1795
1796 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1797 Ok(negotiated_ver) => negotiated_ver,
1798 Err(message) => {
1799 return Ok(vec![control_error_frame(
1800 &frame,
1801 "version_unsupported",
1802 message,
1803 )?])
1804 }
1805 };
1806
1807 let swap_admission = self
1815 .supervisor
1816 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1817 if swap_admission == SwapHelloAdmission::Refused {
1818 warn!(
1819 module_id = %hello.manifest.module_id,
1820 connection_id = connection_id.get(),
1821 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1822 );
1823 return Ok(vec![control_error_frame(
1824 &frame,
1825 "swap_token_invalid",
1826 format!(
1827 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1828 hello.manifest.module_id
1829 ),
1830 )?]);
1831 }
1832 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1833
1834 if let Some(rejection) = (!swap_candidate)
1842 .then(|| {
1843 self.supervisor.reserved_hello_rejection(
1844 &hello.manifest.module_id,
1845 hello.launch_nonce.as_deref(),
1846 )
1847 })
1848 .flatten()
1849 {
1850 let message = match rejection {
1851 ReservedHelloRejection::Exact { module_id } => format!(
1852 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1853 ),
1854 ReservedHelloRejection::Prefix {
1855 prefix,
1856 owner_module_id,
1857 } => format!(
1858 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1859 hello.manifest.module_id
1860 ),
1861 };
1862 return Ok(vec![control_error_frame(
1863 &frame,
1864 "reserved_module",
1865 message,
1866 )?]);
1867 }
1868
1869 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1870 &hello.manifest.module_id,
1871 hello.manifest.capabilities.as_ref(),
1872 );
1873 if let Some(refusal) = reserved_capability_refusals.first() {
1874 let capability = refusal.capability.clone();
1875 let bound_module = refusal.claimants[0].clone();
1876 log_duplicate_claim_events(reserved_capability_refusals);
1877 return Ok(vec![control_error_frame(
1878 &frame,
1879 "reserved_capability",
1880 format!(
1881 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1882 capability, bound_module, hello.manifest.module_id
1883 ),
1884 )?]);
1885 }
1886
1887 if self
1890 .forwarding
1891 .connection_has_client_routes(connection_id)
1892 .map_err(RouterError::Forwarding)?
1893 {
1894 return Ok(vec![control_error_frame(
1895 &frame,
1896 "invalid_hello",
1897 "connection has open client routes and cannot also register as a module",
1898 )?]);
1899 }
1900
1901 self.hello_launch_nonces
1906 .lock()
1907 .unwrap_or_else(|poisoned| poisoned.into_inner())
1908 .record(connection_id, hello.launch_nonce.as_deref());
1909 let control_ops = effective_module_control_ops(hello.control_ops);
1910 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
1913 if swap_candidate {
1914 return self.register_swap_candidate(
1915 connection_id,
1916 sink,
1917 &frame,
1918 hello.manifest,
1919 negotiated_ver,
1920 control_ops,
1921 hello_ack,
1922 );
1923 }
1924 let registration = match self.registry.register_with_control_ops(
1925 hello.manifest,
1926 negotiated_ver,
1927 connection_id,
1928 control_ops,
1929 ) {
1930 Ok(registration) => registration,
1931 Err(RegistryError::DuplicateModuleId { module_id }) => {
1932 return Ok(vec![control_error_frame(
1933 &frame,
1934 "duplicate_module_id",
1935 format!(
1936 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
1937 ),
1938 )?])
1939 }
1940 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
1941 return Ok(vec![control_error_frame(
1942 &frame,
1943 "invalid_module_id",
1944 err.to_string(),
1945 )?])
1946 }
1947 Err(err) => {
1948 return Ok(vec![control_error_frame(
1949 &frame,
1950 "registry_error",
1951 err.to_string(),
1952 )?])
1953 }
1954 };
1955
1956 let reply = if let Some(sink) = sink {
1957 let concurrency = manifest_concurrency(®istration.manifest);
1973 if let Err(err) = self.forwarding.register_module_connection_acked(
1974 connection_id,
1975 registration.manifest.module_id.clone(),
1976 negotiated_ver,
1977 concurrency,
1978 sink,
1979 hello_ack,
1980 ) {
1981 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
1985 crate::supervise::notify_registration_release();
1986 }
1987 return Ok(vec![control_error_frame(
1988 &frame,
1989 forwarding_error_code(&err),
1990 err.to_string(),
1991 )?]);
1992 }
1993 Vec::new()
1994 } else {
1995 vec![hello_ack]
1998 };
1999
2000 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2009 info!(
2010 module_id = %registration.manifest.module_id,
2011 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2012 );
2013 }
2014
2015 self.apply_registration_capabilities(®istration);
2016
2017 info!(
2018 module_id = %registration.manifest.module_id,
2019 module_version = %registration.manifest.module_version,
2020 negotiated_ver,
2021 routable_provider = manifest_provides_routable_role(®istration.manifest),
2022 connection_id = connection_id.get(),
2023 "module registered"
2024 );
2025
2026 Ok(reply)
2027 }
2028
2029 #[allow(clippy::too_many_arguments)]
2039 fn register_swap_candidate(
2040 &self,
2041 connection_id: ConnectionId,
2042 sink: Option<crate::FrameSink>,
2043 frame: &Frame,
2044 manifest: ModuleManifest,
2045 negotiated_ver: u8,
2046 control_ops: Vec<String>,
2047 hello_ack: Frame,
2048 ) -> Result<Vec<Frame>, RouterError> {
2049 let module_id = manifest.module_id.clone();
2050 let registration = match self.registry.register_candidate_with_control_ops(
2051 manifest,
2052 negotiated_ver,
2053 connection_id,
2054 control_ops,
2055 ) {
2056 Ok(registration) => registration,
2057 Err(RegistryError::DuplicateModuleId { module_id }) => {
2058 return Ok(vec![control_error_frame(
2059 frame,
2060 "duplicate_module_id",
2061 format!(
2062 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2063 ),
2064 )?])
2065 }
2066 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2067 return Ok(vec![control_error_frame(
2068 frame,
2069 "invalid_module_id",
2070 err.to_string(),
2071 )?])
2072 }
2073 Err(err) => {
2074 return Ok(vec![control_error_frame(
2075 frame,
2076 "registry_error",
2077 err.to_string(),
2078 )?])
2079 }
2080 };
2081 let reply = if let Some(sink) = sink {
2082 let concurrency = manifest_concurrency(®istration.manifest);
2086 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2087 connection_id,
2088 module_id.clone(),
2089 negotiated_ver,
2090 concurrency,
2091 sink,
2092 hello_ack,
2093 ) {
2094 if matches!(self.deregister_connection(connection_id), Ok(r) if !r.is_empty()) {
2095 crate::supervise::notify_registration_release();
2096 }
2097 return Ok(vec![control_error_frame(
2098 frame,
2099 forwarding_error_code(&err),
2100 err.to_string(),
2101 )?]);
2102 }
2103 Vec::new()
2104 } else {
2105 vec![hello_ack]
2106 };
2107 self.supervisor.mark_swap_candidate_admitted(&module_id);
2108 info!(
2109 module_id = %module_id,
2110 module_version = %registration.manifest.module_version,
2111 negotiated_ver,
2112 ready = registration.ready,
2113 connection_id = connection_id.get(),
2114 "swap candidate registered; not routable until cutover"
2115 );
2116 Ok(reply)
2117 }
2118
2119 fn build_hello_ack(
2120 &self,
2121 frame: &Frame,
2122 negotiated_ver: u8,
2123 module_id: &str,
2124 ) -> Result<Frame, RouterError> {
2125 let ack = ModuleHelloAckBody {
2126 negotiated_ver,
2127 subc_ops: module_subc_ops(),
2128 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2129 storage: self
2130 .storage_config
2131 .as_ref()
2132 .map(|cfg| cfg.descriptor_for(module_id)),
2133 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2134 };
2135 let body = serde_json::to_vec(&ack).map_err(|err| {
2136 RouterError::backend(
2137 0,
2138 frame.header.corr,
2139 format!("failed to encode HELLO_ACK: {err}"),
2140 )
2141 })?;
2142
2143 Frame::build_with_version(
2144 negotiated_ver,
2145 FrameType::HelloAck,
2146 control_flags(),
2147 0,
2148 0,
2149 frame.header.corr,
2150 body,
2151 )
2152 .map_err(RouterError::FrameBuild)
2153 }
2154
2155 async fn handle_client_control_request(
2156 &self,
2157 ctx: &RouteCtx,
2158 frame: Frame,
2159 request: ClientControlRequest,
2160 ) -> Result<Vec<Frame>, RouterError> {
2161 match request {
2162 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2163 ClientControlRequest::CatalogList { module_id } => {
2164 self.handle_catalog_list(frame, module_id)
2165 }
2166 ClientControlRequest::RouteOpen {
2167 target,
2168 identity,
2169 consumer_identity,
2170 consumer_capabilities,
2171 admission_facts,
2172 scope,
2173 } => {
2174 self.handle_route_open(
2175 ctx,
2176 frame,
2177 RouteOpenRequest {
2178 target,
2179 identity,
2180 consumer_identity,
2181 consumer_capabilities,
2182 admission_facts,
2183 scope,
2184 },
2185 )
2186 .await
2187 }
2188 ClientControlRequest::RoutePoll {
2189 route_channel,
2190 route_epoch,
2191 kind,
2192 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2193 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2194 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2195 self.handle_supervisor_spawn_snapshot(frame)
2196 }
2197 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2198 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2199 }
2200 ClientControlRequest::SupervisorRestart {
2201 module_id,
2202 drain_timeout_ms,
2203 } => {
2204 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2205 .await
2206 }
2207 ClientControlRequest::SupervisorSwap {
2208 module_id,
2209 ready_timeout_ms,
2210 } => {
2211 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2212 .await
2213 }
2214 ClientControlRequest::SupervisorReload { module_id } => {
2215 self.handle_supervisor_reload(frame, module_id).await
2216 }
2217 ClientControlRequest::SupervisorRescan { preview } => {
2218 self.handle_supervisor_rescan(frame, preview).await
2219 }
2220 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2221 self.handle_supervisor_release_reserved(frame, module_id)
2222 .await
2223 }
2224 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2225 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2226 .await
2227 }
2228 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2229 self.handle_supervisor_health_probe(frame, module_id).await
2230 }
2231 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2232 ClientControlRequest::SupervisorRoutes { module_id } => {
2233 self.handle_supervisor_routes(frame, module_id)
2234 }
2235 ClientControlRequest::SupervisorProvenance { module_id } => {
2236 self.handle_supervisor_provenance(frame, module_id).await
2237 }
2238 ClientControlRequest::SupervisorStderrTail {
2239 module_id,
2240 max_lines,
2241 max_bytes,
2242 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2243 ClientControlRequest::SupervisorTerminals { module_id } => {
2244 self.handle_supervisor_terminals(frame, module_id).await
2245 }
2246 }
2247 }
2248
2249 fn handle_module_control_request(
2250 &self,
2251 connection_id: ConnectionId,
2252 frame: Frame,
2253 request: ModuleControlRequestFromModule,
2254 ) -> Result<Vec<Frame>, RouterError> {
2255 match request {
2256 ModuleControlRequestFromModule::CatalogUpdate {
2257 provides,
2258 capabilities,
2259 ready,
2260 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2261 ModuleControlRequestFromModule::LiveRoots {} => {
2262 let registered = self
2263 .registry
2264 .get_module_by_connection(connection_id)
2265 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2266 let Some(registration) = registered else {
2267 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2268 };
2269 let response = self
2270 .forwarding
2271 .live_roots(®istration.manifest.module_id)
2272 .map_err(RouterError::Forwarding)?;
2273 Ok(vec![control_response_body_frame(
2274 &frame,
2275 &response,
2276 "ModuleControlResponseToModule::LiveRoots",
2277 )?])
2278 }
2279 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2280 self.handle_scope_sync(connection_id, frame, generation, scopes)
2281 }
2282 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2283 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2284 }
2285 }
2286 }
2287
2288 fn handle_scope_sync(
2292 &self,
2293 connection_id: ConnectionId,
2294 frame: Frame,
2295 generation: u64,
2296 scopes: Vec<ScopeRecord>,
2297 ) -> Result<Vec<Frame>, RouterError> {
2298 let Some(registration) = self
2299 .registry
2300 .get_module_by_connection(connection_id)
2301 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2302 else {
2303 return Ok(vec![control_error_frame(
2304 &frame,
2305 "not_registered",
2306 "scope.sync requires an active module registration owned by this connection",
2307 )?]);
2308 };
2309 let owner = registration.manifest.module_id;
2310 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2311 let is_current_launch = |connection: ConnectionId| {
2312 self.hello_launch_nonces
2313 .lock()
2314 .unwrap_or_else(|poisoned| poisoned.into_inner())
2315 .presented(connection, current_nonce.as_deref())
2316 };
2317 let mut table = self
2322 .scopes
2323 .write()
2324 .unwrap_or_else(|poisoned| poisoned.into_inner());
2325 let outcome = table.sync(&owner, connection_id, is_current_launch, generation, scopes);
2326 let drained = match &outcome {
2327 Ok(applied) => self
2328 .forwarding
2329 .publish_scope_changes(&applied.tag_changes)
2330 .map_err(RouterError::Forwarding)?,
2331 Err(_) => Vec::new(),
2332 };
2333 drop(table);
2334 match outcome {
2335 Ok(applied) => {
2336 info!(
2337 owner = %owner,
2338 generation,
2339 records = applied.results.len(),
2340 ended = applied.ended.len(),
2341 tag_changes = applied.tag_changes.len(),
2342 routes_closed = drained.len(),
2343 "scope sync accepted"
2344 );
2345 self.close_scope_drained_routes(drained);
2346 let response = ModuleControlResponseToModule::ScopeSync {
2347 generation,
2348 results: applied.results,
2349 ended: applied.ended,
2350 };
2351 Ok(vec![control_response_body_frame(
2352 &frame,
2353 &response,
2354 "ModuleControlResponseToModule::ScopeSync",
2355 )?])
2356 }
2357 Err(refusal) => {
2358 info!(
2359 owner = %owner,
2360 generation,
2361 code = refusal.code,
2362 "scope sync refused"
2363 );
2364 Ok(vec![control_error_frame(
2365 &frame,
2366 refusal.code,
2367 refusal.message,
2368 )?])
2369 }
2370 }
2371 }
2372
2373 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2379 if drained.is_empty() {
2380 return;
2381 }
2382 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2383 BTreeMap::new();
2384 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2385 for route in drained {
2386 warn!(
2387 module_id = %route.module_id,
2388 reason = ?route.reason,
2389 client_connection_id = route.client.connection_id.get(),
2390 route_channel = route.client.channel,
2391 "closing route because its scope changed"
2392 );
2393 pushes
2394 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2395 .or_insert_with(|| (route.reason, Vec::new()))
2396 .1
2397 .push(EndpointRoute {
2398 goodbye_target: route.client.clone(),
2399 principal: Principal::Unverified,
2400 bound_at: Instant::now(),
2401 draining: false,
2402 drain_reason: None,
2403 });
2404 goodbyes.push(route.module);
2405 goodbyes.push(route.client);
2406 }
2407 for ((module_id, _), (reason, routes)) in pushes {
2408 send_route_control_pushes(
2409 &self.forwarding,
2410 routes,
2411 ClientControlPush::RouteClosed {
2412 module_id,
2413 reason,
2414 drained: false,
2415 abandoned: 0,
2416 excluded_subscriptions: 0,
2417 terminal: Some(false),
2418 },
2419 );
2420 }
2421 self.emit_route_goodbyes(goodbyes);
2422 }
2423
2424 fn handle_scope_describe(
2427 &self,
2428 connection_id: ConnectionId,
2429 frame: Frame,
2430 owner: Principal,
2431 scope_ref: String,
2432 ) -> Result<Vec<Frame>, RouterError> {
2433 let registered = self
2434 .registry
2435 .get_module_by_connection(connection_id)
2436 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2437 if registered.is_none() {
2438 return Ok(vec![control_error_frame(
2439 &frame,
2440 "not_registered",
2441 "scope.describe requires an active module registration owned by this connection",
2442 )?]);
2443 }
2444 let description = self
2445 .scopes
2446 .read()
2447 .unwrap_or_else(|poisoned| poisoned.into_inner())
2448 .describe(&owner, &scope_ref);
2449 let owner_configured = match &owner {
2450 Principal::Reserved { module_id } => self.supervisor.get(module_id).is_some(),
2451 _ => false,
2452 };
2453 let response = ModuleControlResponseToModule::ScopeDescribe {
2454 status: description.status,
2455 scope_epoch: description.scope_epoch,
2456 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2457 owner_synced: description.owner_synced,
2458 owner_configured,
2459 scope: description.stamp,
2460 };
2461 Ok(vec![control_response_body_frame(
2462 &frame,
2463 &response,
2464 "ModuleControlResponseToModule::ScopeDescribe",
2465 )?])
2466 }
2467
2468 fn handle_catalog_update(
2469 &self,
2470 connection_id: ConnectionId,
2471 frame: Frame,
2472 provides: Vec<ProviderRole>,
2473 capabilities: Option<CapabilityDeclarations>,
2474 ready: Option<bool>,
2475 ) -> Result<Vec<Frame>, RouterError> {
2476 self.refresh_capability_requirements();
2477 let Some(registration) = self
2478 .registry
2479 .get_module_by_connection(connection_id)
2480 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2481 else {
2482 return Ok(vec![control_error_frame(
2483 &frame,
2484 "not_registered",
2485 "catalog.update requires an active module registration owned by this connection",
2486 )?]);
2487 };
2488
2489 if let Some(message) =
2490 catalog_update_frozen_field_message(®istration.manifest, &provides)
2491 {
2492 return Ok(vec![control_error_frame(
2493 &frame,
2494 "catalog_update_frozen_field",
2495 message,
2496 )?]);
2497 }
2498
2499 let mut candidate = registration.manifest.clone();
2500 candidate.provides = provides.clone();
2501 candidate.capabilities = capabilities
2502 .clone()
2503 .or_else(|| registration.manifest.capabilities.clone());
2504 if let Err(err) = candidate.validate_capability_grammar() {
2505 return Ok(vec![control_error_frame(
2506 &frame,
2507 "invalid_capability_grammar",
2508 err.to_string(),
2509 )?]);
2510 }
2511
2512 let updated = self
2513 .registry
2514 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2515 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2516 if updated.is_none() {
2517 return Ok(vec![control_error_frame(
2518 &frame,
2519 "not_registered",
2520 "catalog.update requires an active module registration owned by this connection",
2521 )?]);
2522 }
2523 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2524 log_duplicate_claim_events(
2525 self.capability_evaluator
2526 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2527 );
2528 }
2529 if capability_census_trigger(
2530 registration.manifest.capabilities.as_ref(),
2531 updated
2532 .as_ref()
2533 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2534 ) {
2535 self.enforce_capability_denies();
2536 }
2537 self.refresh_capability_requirements();
2538
2539 let response = ModuleControlResponseToModule::CatalogUpdate {};
2540 control_response_body_frame(
2541 &frame,
2542 &response,
2543 "ModuleControlResponseToModule::CatalogUpdate",
2544 )
2545 .map(|frame| vec![frame])
2546 }
2547
2548 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2549 self.refresh_capability_requirements();
2550 let mut counters = self.counters.snapshot();
2556 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2557 self.forwarding.client_route_concentration(),
2558 counters.as_object_mut(),
2559 ) {
2560 obj.insert(
2561 "client_connections_with_routes".into(),
2562 connections_with_routes.into(),
2563 );
2564 obj.insert("max_routes_on_one_connection".into(), max.into());
2565 }
2566 if let (Some(open_breakers), Some(obj)) = (
2572 self.route_bind_breakers.open_snapshot(),
2573 counters.as_object_mut(),
2574 ) {
2575 obj.insert("route_bind_breakers_open".into(), open_breakers);
2576 }
2577 let response = ClientControlResponse::ServerDescribe {
2578 protocol_ver: PROTOCOL_VERSION,
2579 subc_ops: subc_ops(),
2580 capabilities: self.subc_capabilities.as_ref().to_vec(),
2581 connected_clients: self.connected_clients.count(),
2582 counters: Some(counters),
2583 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2584 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2585 capability_requirements: self.capability_requirement_statuses(),
2586 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2587 };
2588 Ok(vec![control_response_body_frame(
2589 &frame,
2590 &response,
2591 "ClientControlResponse::ServerDescribe",
2592 )?])
2593 }
2594
2595 fn handle_catalog_list(
2596 &self,
2597 frame: Frame,
2598 module_id: Option<String>,
2599 ) -> Result<Vec<Frame>, RouterError> {
2600 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2601 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2602 })?;
2603 let entries = modules
2604 .into_iter()
2605 .filter(|registration| {
2606 module_id
2607 .as_deref()
2608 .map(|wanted| registration.manifest.module_id == wanted)
2609 .unwrap_or(true)
2610 })
2611 .map(|registration| {
2612 let not_ready = self.not_ready_reason(®istration);
2613 let roles = registration.manifest.provides;
2614 CatalogEntry {
2615 module_id: registration.manifest.module_id,
2616 ready: not_ready.is_none(),
2617 not_ready,
2618 module_version: Some(registration.manifest.module_version),
2619 roles,
2620 control_ops: registration.control_ops,
2621 capabilities: registration.manifest.capabilities,
2622 self_signals: registration.manifest.self_signals,
2623 }
2624 })
2625 .collect();
2626 let response = ClientControlResponse::CatalogList {
2627 generation,
2628 modules: entries,
2629 subc_ops: subc_ops(),
2630 };
2631 Ok(vec![control_response_body_frame(
2632 &frame,
2633 &response,
2634 "ClientControlResponse::CatalogList",
2635 )?])
2636 }
2637
2638 fn route_open_principal(
2639 &self,
2640 frame: &Frame,
2641 consumer_identity: Option<ConsumerIdentity>,
2642 ) -> Result<Result<Principal, Frame>, RouterError> {
2643 let Some(consumer_identity) = consumer_identity else {
2644 return Ok(Ok(Principal::Direct));
2645 };
2646
2647 if self.supervisor.spawned_consumer_authorized(
2648 &consumer_identity.module_id,
2649 &consumer_identity.launch_nonce,
2650 ) {
2651 return Ok(Ok(Principal::Reserved {
2652 module_id: consumer_identity.module_id,
2653 }));
2654 }
2655
2656 Ok(Err(control_error_frame(
2657 frame,
2658 "bad_consumer_identity",
2659 format!(
2660 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2661 consumer_identity.module_id
2662 ),
2663 )?))
2664 }
2665
2666 fn route_open_refusal_frame(
2673 &self,
2674 ctx: &RouteCtx,
2675 frame: &Frame,
2676 module_id: &str,
2677 reason: &'static str,
2678 code: &'static str,
2679 message: impl Into<String>,
2680 ) -> Result<Frame, RouterError> {
2681 self.observe_route_open_refusal(ctx, module_id, reason, code);
2682 control_error_frame(frame, code, message.into())
2683 }
2684
2685 fn route_open_breaker_refusal_frame(
2703 &self,
2704 ctx: &RouteCtx,
2705 frame: &Frame,
2706 module_id: &str,
2707 consecutive_timeouts: u32,
2708 retry_in: Duration,
2709 probe_in_flight: bool,
2710 ) -> Result<Frame, RouterError> {
2711 self.counters
2712 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
2713 debug!(
2714 target: "control",
2715 code = "module_timeout",
2716 module_id = ?module_id,
2717 connection_id = ctx.connection_id.get(),
2718 consecutive_timeouts,
2719 retry_in_ms = retry_in.as_millis() as u64,
2720 probe_in_flight,
2721 "route.open refused by open bind-relay breaker"
2722 );
2723 let detail = if probe_in_flight {
2724 "one probe bind is already in flight; retry once it settles".to_string()
2725 } else {
2726 format!("not relaying for another {retry_in:?}")
2727 };
2728 control_error_frame(
2729 frame,
2730 "module_timeout",
2731 format!(
2732 "module_id '{module_id}' failed {consecutive_timeouts} consecutive route.bind \
2733 relays; {detail}"
2734 ),
2735 )
2736 }
2737
2738 fn observe_route_open_refusal(
2751 &self,
2752 ctx: &RouteCtx,
2753 module_id: &str,
2754 reason: &'static str,
2755 code: &'static str,
2756 ) {
2757 self.counters.increment_route_open_refused(code);
2758 info!(
2759 target: "control",
2760 code,
2761 reason,
2762 module_id = ?module_id,
2763 connection_id = ctx.connection_id.get(),
2764 "route.open refused"
2765 );
2766 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
2767 self.route_outages.record_not_serving(module_id, reason);
2768 }
2769 }
2770
2771 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
2816 self.route_outages.record_accepted(module_id);
2817 self.counters.increment_route_open_accepted(principal);
2818 info!(
2819 target: "control",
2820 principal,
2821 module_id,
2822 connection_id = ctx.connection_id.get(),
2823 "route.open accepted"
2824 );
2825 }
2826
2827 fn supervised_absent_route_open_refusal_frame(
2828 &self,
2829 ctx: &RouteCtx,
2830 frame: &Frame,
2831 module_id: &str,
2832 code: &'static str,
2833 status: &crate::supervise::ModuleStatus,
2834 ) -> Result<Frame, RouterError> {
2835 self.counters.increment_route_open_refused(code);
2836 info!(
2837 target: "control",
2838 code,
2839 reason = "supervised_not_registered",
2840 module_id = ?module_id,
2841 connection_id = ctx.connection_id.get(),
2842 state = %status.state,
2843 enabled = status.enabled,
2844 live = status.live,
2845 "route.open refused"
2846 );
2847 self.route_outages
2851 .record_not_serving(module_id, "supervised_not_registered");
2852 control_error_frame(
2853 frame,
2854 code,
2855 format!(
2856 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
2857 status.state, status.enabled, status.live
2858 ),
2859 )
2860 }
2861
2862 async fn handle_route_open(
2863 &self,
2864 ctx: &RouteCtx,
2865 frame: Frame,
2866 request: RouteOpenRequest,
2867 ) -> Result<Vec<Frame>, RouterError> {
2868 let RouteOpenRequest {
2869 target,
2870 mut identity,
2871 consumer_identity,
2872 consumer_capabilities,
2873 admission_facts,
2874 scope,
2875 } = request;
2876 let target_module_id = target_module_id(&target).to_string();
2877 debug!(
2878 connection_id = ctx.connection_id.get(),
2879 corr = frame.header.corr,
2880 module_id = %target_module_id,
2881 "handling route.open"
2882 );
2883
2884 let Some(registration) = self
2903 .registry
2904 .get_module(&target_module_id)
2905 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2906 else {
2907 if let Some((status, warming)) =
2908 self.supervisor_status(&target_module_id, frame.header.corr)?
2909 {
2910 if status.protocol == ModuleProtocol::None {
2917 return Ok(vec![self.route_open_refusal_frame(
2918 ctx,
2919 &frame,
2920 &target_module_id,
2921 "protocol_none",
2922 error_codes::MODULE_NO_PROTOCOL,
2923 format!(
2924 "module_id '{target_module_id}' is declared protocol: none; \
2925 it speaks no subc wire and serves no routes"
2926 ),
2927 )?]);
2928 }
2929 let code = if warming {
2930 "module_warming"
2931 } else {
2932 "target_unavailable"
2933 };
2934 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
2935 ctx,
2936 &frame,
2937 &target_module_id,
2938 code,
2939 &status,
2940 )?]);
2941 }
2942 if let Some(removed_ago_ms) =
2943 self.supervisor.removal_tombstone_age_ms(&target_module_id)
2944 {
2945 return Ok(vec![self.route_open_refusal_frame(
2946 ctx,
2947 &frame,
2948 &target_module_id,
2949 "removed",
2950 error_codes::MODULE_REMOVED,
2951 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
2952 )?]);
2953 }
2954 return Ok(vec![self.route_open_refusal_frame(
2955 ctx,
2956 &frame,
2957 &target_module_id,
2958 "not_registered",
2959 error_codes::UNKNOWN_MODULE,
2960 format!("module_id '{target_module_id}' is not registered"),
2961 )?]);
2962 };
2963
2964 if !registration.ready {
2968 self.counters
2969 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
2970 info!(
2971 target: "control",
2972 code = error_codes::MODULE_WARMING,
2973 module_id = ?target_module_id,
2974 connection_id = ctx.connection_id.get(),
2975 reason = "declared_not_ready",
2976 "route.open refused"
2977 );
2978 self.route_outages
2981 .record_not_serving(&target_module_id, "declared_not_ready");
2982 return Ok(vec![control_error_body_frame(
2983 &frame,
2984 ErrorBody {
2985 code: error_codes::MODULE_WARMING.to_string(),
2986 message: format!(
2987 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
2988 ),
2989 detail: Some(serde_json::json!({
2990 "reason": "declared_not_ready"
2991 })),
2992 },
2993 )?]);
2994 }
2995
2996 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3015 self.counters
3016 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3017 info!(
3018 target: "control",
3019 code = error_codes::MODULE_WARMING,
3020 module_id = ?target_module_id,
3021 connection_id = ctx.connection_id.get(),
3022 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3023 capability = %capability,
3024 "route.open refused"
3025 );
3026 return Ok(vec![control_error_body_frame(
3027 &frame,
3028 ErrorBody {
3029 code: error_codes::MODULE_WARMING.to_string(),
3030 message: format!(
3031 "module_id '{target_module_id}' requires capability '{capability}', \
3032 which no registered module provides; retry"
3033 ),
3034 detail: Some(serde_json::json!({
3035 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3036 "capability": capability,
3037 })),
3038 },
3039 )?]);
3040 }
3041
3042 if !target_has_required_role(&target, ®istration.manifest.provides) {
3043 return Ok(vec![self.route_open_refusal_frame(
3044 ctx,
3045 &frame,
3046 &target_module_id,
3047 "role_not_provided",
3048 "target_unavailable",
3049 format!("module_id '{target_module_id}' does not provide the requested target"),
3050 )?]);
3051 }
3052
3053 if registration.state != ChannelState::Active {
3054 return Ok(vec![self.route_open_refusal_frame(
3055 ctx,
3056 &frame,
3057 &target_module_id,
3058 "registration_not_active",
3059 "target_unavailable",
3060 format!("module_id '{target_module_id}' is not active"),
3061 )?]);
3062 }
3063
3064 if self
3065 .forwarding
3066 .module_is_draining(&target_module_id)
3067 .map_err(RouterError::Forwarding)?
3068 {
3069 return Ok(vec![self.route_open_refusal_frame(
3070 ctx,
3071 &frame,
3072 &target_module_id,
3073 "reloading",
3074 "module_reloading",
3075 format!("module_id '{target_module_id}' is reloading"),
3076 )?]);
3077 }
3078
3079 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3080 process_liveness.process_live(&target_module_id) == Some(false)
3081 }) {
3082 if process_liveness.process_replacing(&target_module_id) {
3089 return Ok(vec![self.route_open_refusal_frame(
3090 ctx,
3091 &frame,
3092 &target_module_id,
3093 "reloading",
3094 "module_reloading",
3095 format!("module_id '{target_module_id}' is reloading"),
3096 )?]);
3097 }
3098 return Ok(vec![self.route_open_refusal_frame(
3099 ctx,
3100 &frame,
3101 &target_module_id,
3102 "supervisor_not_live",
3103 "target_unavailable",
3104 format!("module_id '{target_module_id}' is not live"),
3105 )?]);
3106 }
3107
3108 if !self
3109 .forwarding
3110 .has_live_module_connection(&target_module_id)
3111 .map_err(RouterError::Forwarding)?
3112 {
3113 return Ok(vec![self.route_open_refusal_frame(
3114 ctx,
3115 &frame,
3116 &target_module_id,
3117 "no_forwarding_connection",
3118 "target_unavailable",
3119 format!("module_id '{target_module_id}' has no live forwarding connection"),
3120 )?]);
3121 }
3122
3123 if let Some(error) =
3124 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3125 {
3126 self.observe_route_open_refusal(
3127 ctx,
3128 &target_module_id,
3129 "op_not_allowed",
3130 "op_not_allowed",
3131 );
3132 return Ok(vec![error]);
3133 }
3134
3135 let principal = match self.route_open_principal(&frame, consumer_identity)? {
3136 Ok(principal) => principal,
3137 Err(error) => {
3138 self.observe_route_open_refusal(
3139 ctx,
3140 &target_module_id,
3141 "bad_consumer_identity",
3142 "bad_consumer_identity",
3143 );
3144 return Ok(vec![error]);
3145 }
3146 };
3147
3148 if let Principal::Reserved {
3152 module_id: opening_module_id,
3153 } = &principal
3154 {
3155 if let Some(opening_registration) = self
3156 .registry
3157 .get_module(opening_module_id)
3158 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3159 {
3160 if let Some(capability) =
3161 denied_capability(&opening_registration.manifest, ®istration.manifest)
3162 {
3163 warn!(
3164 opening_module_id,
3165 target_module_id,
3166 capability,
3167 "refusing route.open because an attested capability deny edge matches"
3168 );
3169 return Ok(vec![self.route_open_refusal_frame(
3170 ctx,
3171 &frame,
3172 &target_module_id,
3173 "capability_deny_edge",
3174 "capability_forbidden",
3175 format!(
3176 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3177 ),
3178 )?]);
3179 }
3180 }
3181 }
3182
3183 if admission_facts.is_some() {
3184 let carrier_matches = matches!(
3185 &principal,
3186 Principal::Reserved { module_id }
3187 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3188 );
3189 if !carrier_matches {
3190 return Ok(vec![self.route_open_refusal_frame(
3191 ctx,
3192 &frame,
3193 &target_module_id,
3194 "admission_facts_carrier_not_permitted",
3195 "admission_facts_not_permitted",
3196 "admission facts may only be carried by the configured reserved module",
3197 )?]);
3198 }
3199
3200 let target_allowed = self
3201 .admission_facts_targets
3202 .as_ref()
3203 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3204 if !target_allowed {
3205 return Ok(vec![self.route_open_refusal_frame(
3206 ctx,
3207 &frame,
3208 &target_module_id,
3209 "admission_facts_target_not_listed",
3210 "admission_facts_target_not_allowed",
3211 format!(
3212 "admission facts are not permitted for target module_id '{target_module_id}'"
3213 ),
3214 )?]);
3215 }
3216
3217 }
3221
3222 let (bound_scope, scope_stamp) = match scope {
3228 None => (None, None),
3229 Some(selector) => {
3230 let owner_configured = match &selector.owner {
3231 Principal::Reserved { module_id } => self.supervisor.get(module_id).is_some(),
3232 _ => false,
3233 };
3234 let admitted = self
3235 .scopes
3236 .read()
3237 .unwrap_or_else(|poisoned| poisoned.into_inner())
3238 .admit(&principal, &target_module_id, &selector, owner_configured);
3239 match admitted {
3240 Ok(admission) => (
3241 Some(BoundScope {
3242 owner: admission.owner,
3243 scope_ref: admission.stamp.scope_ref.clone(),
3244 tag: admission.tag,
3245 }),
3246 Some(admission.stamp),
3247 ),
3248 Err(refusal) => {
3249 return Ok(vec![self.route_open_refusal_frame(
3250 ctx,
3251 &frame,
3252 &target_module_id,
3253 refusal.code,
3254 refusal.code,
3255 refusal.message,
3256 )?]);
3257 }
3258 }
3259 }
3260 };
3261
3262 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3284 Ok(project_root) => project_root,
3285 Err(err) => {
3286 return Ok(vec![control_error_frame(
3287 &frame,
3288 "invalid_project_root",
3289 err.to_string(),
3290 )?])
3291 }
3292 };
3293 identity.project_root = project_root.as_path().to_path_buf();
3294
3295 let _concurrency_guard = match self
3306 .route_bind_concurrency
3307 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3308 {
3309 Ok(guard) => guard,
3310 Err(in_flight) => {
3311 return Ok(vec![self.route_open_target_capacity_refusal(
3312 ctx,
3313 &frame,
3314 &target_module_id,
3315 in_flight,
3316 )?]);
3317 }
3318 };
3319
3320 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3323 RouteBindAdmission::Admitted { guard, probe } => {
3324 if probe {
3325 info!(
3326 module_id = %target_module_id,
3327 connection_id = ctx.connection_id.get(),
3328 "route.bind breaker half-open: admitting one probe"
3329 );
3330 }
3331 guard
3332 }
3333 RouteBindAdmission::Refused {
3334 consecutive_timeouts,
3335 retry_in,
3336 probe_in_flight,
3337 } => {
3338 return Ok(vec![self.route_open_breaker_refusal_frame(
3339 ctx,
3340 &frame,
3341 &target_module_id,
3342 consecutive_timeouts,
3343 retry_in,
3344 probe_in_flight,
3345 )?]);
3346 }
3347 };
3348
3349 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3354 let relay_deadline = Instant::now() + route_bind_relay_timeout;
3355 let pending = match self
3356 .forwarding
3357 .begin_route_bind_relay_for(
3358 ctx.connection_id,
3359 ctx.egress.clone(),
3360 response_version(&frame),
3361 frame.header.corr,
3362 &target_module_id,
3363 principal.clone(),
3364 bound_scope,
3365 Some(project_root),
3366 relay_deadline,
3367 )
3368 .await
3369 {
3370 Ok(pending) => pending,
3371 Err(err) => {
3372 return Ok(vec![self.route_open_refusal_frame(
3373 ctx,
3374 &frame,
3375 &target_module_id,
3376 "relay_reservation_failed",
3377 forwarding_error_code(&err),
3378 err.to_string(),
3379 )?])
3380 }
3381 };
3382 let crate::forwarding::PendingRouteBindRelay {
3383 endpoint,
3384 module_sink,
3385 negotiated_ver,
3386 client_channel,
3387 client_epoch,
3388 module_channel,
3389 module_epoch,
3390 corr: relay_corr,
3391 receiver,
3392 } = pending;
3393 let mut reservation =
3394 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3395
3396 debug!(
3397 connection_id = ctx.connection_id.get(),
3398 client_channel,
3399 client_epoch,
3400 module_channel,
3401 module_epoch,
3402 "reserved route handle pair"
3403 );
3404 let principal_label = match &principal {
3407 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
3408 Principal::Direct => "direct".to_string(),
3409 other => format!("{other:?}"),
3410 };
3411 let relay = ModuleControlRequest::RouteBind {
3412 route_channel: module_channel,
3413 epoch: module_epoch,
3414 target,
3415 identity,
3416 principal: Some(principal),
3417 consumer_capabilities,
3418 admission_facts,
3419 scope: scope_stamp,
3420 };
3421 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3422 RouterError::backend(
3423 0,
3424 frame.header.corr,
3425 format!("failed to encode route.bind request: {err}"),
3426 )
3427 })?;
3428 let relay_frame = Frame::build_with_version(
3429 negotiated_ver,
3430 FrameType::Request,
3431 control_flags(),
3432 0,
3433 0,
3434 relay_corr,
3435 relay_body,
3436 )
3437 .map_err(RouterError::FrameBuild)?;
3438
3439 if let Err(err) = module_sink.send(relay_frame).await {
3440 reservation.release_and_disarm();
3441 return Ok(vec![self.route_open_refusal_frame(
3442 ctx,
3443 &frame,
3444 &target_module_id,
3445 "relay_send_failed",
3446 "target_unavailable",
3447 err.to_string(),
3448 )?]);
3449 }
3450
3451 if !self
3452 .forwarding
3453 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3454 .map_err(RouterError::Forwarding)?
3455 {
3456 self.send_abandoned_route_bind_goodbye(
3457 &module_sink,
3458 negotiated_ver,
3459 module_channel,
3460 module_epoch,
3461 );
3462 }
3463
3464 match timeout_at(relay_deadline, receiver).await {
3465 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3466 reservation.disarm();
3467 if breaker.record_accepted() {
3468 info!(
3469 module_id = %target_module_id,
3470 "route.bind breaker closed: the probe was accepted"
3471 );
3472 }
3473 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3474 Ok(Vec::new())
3475 }
3476 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3477 reservation.release_and_disarm();
3478 breaker.record_inconclusive();
3482 let scope_code = match body.code.as_str() {
3486 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
3487 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
3488 _ => None,
3489 };
3490 if let Some(code) = scope_code {
3491 self.observe_route_open_refusal(
3492 ctx,
3493 &target_module_id,
3494 "scope_changed_before_commit",
3495 code,
3496 );
3497 return Ok(vec![control_error_body_frame(&frame, body)?]);
3498 }
3499 self.counters
3500 .increment_route_open_refused("module_rejected");
3501 info!(
3502 target: "control",
3503 code = "module_rejected",
3504 module_code = ?body.code,
3505 module_id = ?target_module_id,
3506 connection_id = ctx.connection_id.get(),
3507 "route.open refused"
3508 );
3509 Ok(vec![control_error_body_frame(&frame, body)?])
3510 }
3511 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3512 reservation.release_and_disarm();
3513 breaker.record_inconclusive();
3514 tracing::warn!(
3521 module_id = %target_module_id,
3522 "route.bind relay abandoned: {message}"
3523 );
3524 Ok(vec![self.route_open_refusal_frame(
3525 ctx,
3526 &frame,
3527 &target_module_id,
3528 "relay_abandoned",
3529 "target_unavailable",
3530 message,
3531 )?])
3532 }
3533 Ok(Err(_)) => {
3534 reservation.release_and_disarm();
3535 breaker.record_inconclusive();
3536 Ok(vec![self.route_open_refusal_frame(
3537 ctx,
3538 &frame,
3539 &target_module_id,
3540 "relay_waiter_canceled",
3541 "target_unavailable",
3542 "route.bind relay waiter was canceled before the module responded",
3543 )?])
3544 }
3545 Err(_) => {
3546 reservation.release_and_disarm();
3547 if let Some(opened) = breaker.record_timeout(
3551 self.route_bind_breaker_threshold,
3552 self.route_bind_breaker_cooldown,
3553 ) {
3554 warn!(
3555 module_id = %target_module_id,
3556 consecutive_timeouts = opened.consecutive_timeouts,
3557 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3558 reopened_after_probe = opened.reopened_after_probe,
3559 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3560 );
3561 }
3562 tracing::warn!(
3570 module_id = %target_module_id,
3571 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3572 "route.bind relay timed out: module did not ack within budget"
3573 );
3574 Ok(vec![self.route_open_refusal_frame(
3575 ctx,
3576 &frame,
3577 &target_module_id,
3578 "relay_timed_out",
3579 "module_timeout",
3580 format!(
3581 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3582 route_bind_relay_timeout
3583 ),
3584 )?])
3585 }
3586 }
3587 }
3588
3589 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3590 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3591 snapshot: self.supervisor.spawn_snapshot(),
3592 };
3593 Ok(vec![control_response_body_frame(
3594 &frame,
3595 &response,
3596 "ClientControlResponse::SupervisorSpawnSnapshot",
3597 )?])
3598 }
3599
3600 fn handle_supervisor_spawn_subscribe(
3601 &self,
3602 ctx: &RouteCtx,
3603 frame: Frame,
3604 since: Option<SpawnCursor>,
3605 ) -> Result<Vec<Frame>, RouterError> {
3606 match self.supervisor.subscribe_spawns(
3607 ctx.connection_id,
3608 frame.header.corr,
3609 response_version(&frame),
3610 since,
3611 ctx.egress.clone(),
3612 ) {
3613 Ok(()) => Ok(Vec::new()),
3614 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3615 Ok(vec![control_error_body_frame(
3616 &frame,
3617 ErrorBody {
3618 code: "spawn_cursor_incarnation_mismatch".to_string(),
3619 message: "spawn cursor belongs to a different daemon incarnation"
3620 .to_string(),
3621 detail: Some(serde_json::json!({
3622 "current_daemon_incarnation": current
3623 })),
3624 },
3625 )?])
3626 }
3627 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
3628 &frame,
3629 ErrorBody {
3630 code: "spawn_cursor_too_old".to_string(),
3631 message: "spawn cursor predates the retained event ring".to_string(),
3632 detail: Some(serde_json::json!({
3633 "oldest_retained_cursor": oldest
3634 })),
3635 },
3636 )?]),
3637 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
3638 0,
3639 frame.header.corr,
3640 format!("failed to open supervisor spawn subscription: {error}"),
3641 )),
3642 }
3643 }
3644
3645 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3646 let generation = self
3647 .registry
3648 .generation()
3649 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3650 let mut modules = Vec::new();
3651 for module in self.supervisor.list() {
3652 let status = module.status_for_control("list").map_err(|err| {
3653 RouterError::backend(
3654 0,
3655 frame.header.corr,
3656 format!("failed to read supervisor status: {err}"),
3657 )
3658 })?;
3659 let (configured, _) = module.configuration().map_err(|err| {
3660 RouterError::backend(
3661 0,
3662 frame.header.corr,
3663 format!("failed to read module configuration: {err}"),
3664 )
3665 })?;
3666 let image = module.running_image_agreement().await;
3668 let resources = Some(module.child_resource_usage());
3671 let pending_reload = Some(reload_verdict(
3672 &configured.program,
3673 status.spawned_from.as_deref(),
3674 image,
3675 ));
3676 modules.push(SupervisorEntry {
3677 module_id: status.module_id,
3678 state: status.state.to_string(),
3679 enabled: status.enabled,
3680 live: status.live,
3681 protocol: status.protocol,
3682 health: status.health.status,
3683 pending_reload,
3684 last_probe_ms: status.health.last_probe_ms,
3685 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
3686 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
3687 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
3688 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
3689 restart_count: Some(status.restart_count),
3690 max_restarts: Some(status.max_restarts),
3691 lifetime_restarts: Some(status.lifetime_restarts),
3692 spawn_generation: Some(status.spawn_generation),
3693 restart_window_secs: Some(status.restart_window.as_secs()),
3694 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
3695 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
3696 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
3697 resources,
3698 });
3699 }
3700 let response = ClientControlResponse::SupervisorList {
3701 generation,
3702 modules,
3703 };
3704 Ok(vec![control_response_body_frame(
3705 &frame,
3706 &response,
3707 "ClientControlResponse::SupervisorList",
3708 )?])
3709 }
3710
3711 fn handle_supervisor_stderr_tail(
3712 &self,
3713 frame: Frame,
3714 module_id: String,
3715 max_lines: Option<u32>,
3716 max_bytes: Option<u32>,
3717 ) -> Result<Vec<Frame>, RouterError> {
3718 let Some(module) = self.supervisor.get(&module_id) else {
3719 return Ok(vec![control_error_frame(
3720 &frame,
3721 "unknown_module",
3722 format!("module_id '{module_id}' is not supervised"),
3723 )?]);
3724 };
3725
3726 let snapshot = module.stderr_tail(
3727 max_lines.map(|value| value as usize),
3728 max_bytes.map(|value| value as usize),
3729 );
3730
3731 let response = ClientControlResponse::SupervisorStderrTail {
3732 module_id,
3733 tail: StderrTail {
3734 capture: match snapshot.capture {
3735 CaptureState::Captured => StderrCaptureState::Captured,
3736 CaptureState::Incomplete { reason } => {
3737 StderrCaptureState::Incomplete { reason }
3738 }
3739 CaptureState::NotCaptured { reason } => {
3740 StderrCaptureState::NotCaptured { reason }
3741 }
3742 },
3743 entries: snapshot
3744 .entries
3745 .into_iter()
3746 .map(|entry| match entry {
3747 TailEntry::Line {
3748 text,
3749 truncated,
3750 at_ms,
3751 } => StderrTailEntry::Line {
3752 text,
3753 truncated,
3754 at_ms,
3755 },
3756 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
3757 })
3758 .collect(),
3759 dropped_lines: snapshot.dropped_lines,
3760 },
3761 };
3762 Ok(vec![control_response_body_frame(
3763 &frame,
3764 &response,
3765 "ClientControlResponse::SupervisorStderrTail",
3766 )?])
3767 }
3768
3769 async fn handle_supervisor_terminals(
3770 &self,
3771 frame: Frame,
3772 module_id: String,
3773 ) -> Result<Vec<Frame>, RouterError> {
3774 let Some(module) = self.supervisor.get(&module_id) else {
3775 return Ok(vec![control_error_frame(
3776 &frame,
3777 "unknown_module",
3778 format!("module_id '{module_id}' is not supervised"),
3779 )?]);
3780 };
3781
3782 let terminals = module
3785 .read_durable_terminal_history()
3786 .await
3787 .map_err(|error| {
3788 RouterError::backend(
3789 0,
3790 frame.header.corr,
3791 format!("failed to read terminal history: {error}"),
3792 )
3793 })?;
3794 let response = ClientControlResponse::SupervisorTerminals {
3795 module_id,
3796 terminals,
3797 };
3798 Ok(vec![control_response_body_frame(
3799 &frame,
3800 &response,
3801 "ClientControlResponse::SupervisorTerminals",
3802 )?])
3803 }
3804
3805 fn handle_supervisor_routes(
3806 &self,
3807 frame: Frame,
3808 module_id: Option<String>,
3809 ) -> Result<Vec<Frame>, RouterError> {
3810 let modules = self
3811 .forwarding
3812 .route_census(module_id.as_deref())
3813 .map_err(RouterError::Forwarding)?
3814 .into_iter()
3815 .map(|(module_id, routes)| SupervisorRouteModule {
3816 module_id,
3817 routes: routes
3818 .into_iter()
3819 .map(|route| SupervisorRoute {
3820 consumer: match route.principal {
3821 Principal::Reserved { module_id } => {
3822 SupervisorRouteConsumer::Reserved { module_id }
3823 }
3824 Principal::Direct | Principal::Unverified => {
3825 SupervisorRouteConsumer::Direct {
3826 connection_id: route.goodbye_target.connection_id.get(),
3827 }
3828 }
3829 },
3830 age_ms: Instant::now()
3831 .saturating_duration_since(route.bound_at)
3832 .as_millis()
3833 .try_into()
3834 .unwrap_or(u64::MAX),
3835 draining: route.draining,
3836 drain_reason: route.drain_reason,
3837 })
3838 .collect(),
3839 })
3840 .collect();
3841 let response = ClientControlResponse::SupervisorRoutes { modules };
3842 Ok(vec![control_response_body_frame(
3843 &frame,
3844 &response,
3845 "ClientControlResponse::SupervisorRoutes",
3846 )?])
3847 }
3848
3849 async fn handle_supervisor_provenance(
3850 &self,
3851 frame: Frame,
3852 module_id: Option<String>,
3853 ) -> Result<Vec<Frame>, RouterError> {
3854 let mut selected = if let Some(module_id) = module_id {
3855 let Some(module) = self.supervisor.get(&module_id) else {
3856 return Ok(vec![control_error_frame(
3857 &frame,
3858 "unknown_module",
3859 format!("module_id '{module_id}' is not supervised"),
3860 )?]);
3861 };
3862 vec![module]
3863 } else {
3864 self.supervisor.list()
3865 };
3866
3867 let mut modules = Vec::with_capacity(selected.len());
3868 for module in selected.drain(..) {
3869 let status = module.status().map_err(|err| {
3870 RouterError::backend(
3871 0,
3872 frame.header.corr,
3873 format!("failed to read supervisor status: {err}"),
3874 )
3875 })?;
3876 let module_declared = self
3877 .registry
3878 .get_module(&status.module_id)
3879 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3880 .and_then(|registration| registration.manifest.provenance)
3881 .map(|build| ModuleDeclaredProvenance::Reported { build })
3882 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
3883 #[cfg(test)]
3884 let running_image = match &self.provenance_probe_override {
3885 Some(result) => result.clone(),
3886 None => module.running_image_agreement().await,
3887 };
3888 #[cfg(not(test))]
3889 let running_image = module.running_image_agreement().await;
3890 modules.push(SupervisorModuleProvenance {
3891 module_id: status.module_id,
3892 module_declared,
3893 daemon_observed: SupervisorObservedProcess {
3894 pid: status.pid,
3895 spawned_at_ms: status.spawned_at_ms,
3896 spawned_from: status.spawned_from,
3897 running_image,
3898 },
3899 });
3900 }
3901 let daemon = SupervisorDaemonProvenance {
3902 daemon_build: self.daemon_provenance.build.clone(),
3903 daemon_observed: DaemonObservedProcess {
3904 pid: self.daemon_provenance.pid,
3905 started_at_ms: self
3906 .daemon_provenance
3907 .start_clock
3908 .map(|clock| clock.started_at_ms())
3909 .or(self.daemon_provenance.started_at_ms),
3910 running_image: self
3911 .daemon_provenance
3912 .probe
3913 .observe(
3914 self.daemon_provenance.pid,
3915 self.daemon_provenance.executable_path.as_deref(),
3916 self.daemon_provenance.executable_identity,
3917 self.daemon_provenance.process_start_time,
3918 )
3919 .await,
3920 },
3921 };
3922 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
3923 Ok(vec![control_response_body_frame(
3924 &frame,
3925 &response,
3926 "ClientControlResponse::SupervisorProvenance",
3927 )?])
3928 }
3929
3930 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3931 self.refresh_capability_requirements();
3932 let generation = self
3933 .registry
3934 .generation()
3935 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3936 let modules = self
3937 .supervisor
3938 .list()
3939 .into_iter()
3940 .map(|module| {
3941 let status = module.status_for_control("health").map_err(|err| {
3942 RouterError::backend(
3943 0,
3944 frame.header.corr,
3945 format!("failed to read supervisor health: {err}"),
3946 )
3947 })?;
3948 let module_id = status.module_id;
3949 let capability_detail = self
3950 .capability_evaluator
3951 .required_problem_detail(&module_id);
3952 Ok(SupervisorHealthEntry {
3953 module_id,
3954 status: status.health.status,
3955 detail: append_capability_problem_detail(
3956 status.health.detail,
3957 capability_detail,
3958 ),
3959 metrics: status.health.metrics,
3960 consecutive_failures: status.health.consecutive_failures,
3961 late_answer_count: status.health.late_answer_count,
3962 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
3963 last_action: status.health.last_action,
3964 last_action_ms: status.health.last_action_ms,
3965 last_probe_ms: status.health.last_probe_ms,
3966 })
3967 })
3968 .collect::<Result<Vec<_>, RouterError>>()?;
3969 let response = ClientControlResponse::SupervisorHealth {
3970 generation,
3971 modules,
3972 };
3973 Ok(vec![control_response_body_frame(
3974 &frame,
3975 &response,
3976 "ClientControlResponse::SupervisorHealth",
3977 )?])
3978 }
3979
3980 async fn handle_supervisor_restart(
3981 &self,
3982 frame: Frame,
3983 module_id: String,
3984 drain_timeout_ms: Option<u64>,
3985 ) -> Result<Vec<Frame>, RouterError> {
3986 let operation_lock = self.supervisor.operation_lock();
3987 let _operation_guard = operation_lock.lock().await;
3988 let Some(module) = self.supervisor.get(&module_id) else {
3989 return Ok(vec![control_error_frame(
3990 &frame,
3991 "unknown_module",
3992 format!("module_id '{module_id}' is not supervised"),
3993 )?]);
3994 };
3995
3996 self.route_outages.mark_operator_action(&module_id);
3997 if let Err(err) = module.restart(drain_timeout_ms).await {
3998 self.route_outages
3999 .operator_action_ended_unrefused(&module_id);
4000 let (code, message) = match err {
4001 crate::supervise::SuperviseError::Disabled { .. } => {
4002 ("module_disabled", err.to_string())
4003 }
4004 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4005 ("swap_in_progress", err.to_string())
4006 }
4007 _ => (
4008 "target_unavailable",
4009 format!("failed to restart module_id '{module_id}': {err}"),
4010 ),
4011 };
4012 return Ok(vec![control_error_frame(&frame, code, message)?]);
4013 }
4014
4015 let response = ClientControlResponse::SupervisorAck {
4016 module_id,
4017 applied: true,
4018 };
4019 Ok(vec![control_response_body_frame(
4020 &frame,
4021 &response,
4022 "ClientControlResponse::SupervisorAck",
4023 )?])
4024 }
4025
4026 async fn handle_supervisor_swap(
4030 &self,
4031 frame: Frame,
4032 module_id: String,
4033 ready_timeout_ms: Option<u64>,
4034 ) -> Result<Vec<Frame>, RouterError> {
4035 let module = {
4042 let operation_lock = self.supervisor.operation_lock();
4043 let _operation_guard = operation_lock.lock().await;
4044 self.supervisor.get(&module_id)
4045 };
4046 let Some(module) = module else {
4047 return Ok(vec![control_error_frame(
4048 &frame,
4049 "unknown_module",
4050 format!("module_id '{module_id}' is not supervised"),
4051 )?]);
4052 };
4053
4054 self.route_outages.mark_operator_action(&module_id);
4055 if let Err(err) = module
4056 .swap(ready_timeout_ms.map(Duration::from_millis))
4057 .await
4058 {
4059 self.route_outages
4060 .operator_action_ended_unrefused(&module_id);
4061 use crate::supervise::SuperviseError;
4062 let message = err.to_string();
4063 let error = match err {
4064 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4065 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4066 code: "swap_refused".to_string(),
4067 message,
4068 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4069 },
4070 SuperviseError::SwapFailed {
4071 arm,
4072 candidate_exit,
4073 ..
4074 } => ErrorBody {
4075 code: "swap_failed".to_string(),
4076 message,
4077 detail: Some(serde_json::json!({
4078 "arm": arm.as_str(),
4079 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4080 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4081 })),
4082 },
4083 _ => ErrorBody::new(
4084 "target_unavailable",
4085 format!("failed to swap module_id '{module_id}': {message}"),
4086 ),
4087 };
4088 return Ok(vec![control_error_body_frame(&frame, error)?]);
4089 }
4090 self.route_outages
4094 .operator_action_ended_unrefused(&module_id);
4095
4096 let response = ClientControlResponse::SupervisorAck {
4097 module_id,
4098 applied: true,
4099 };
4100 Ok(vec![control_response_body_frame(
4101 &frame,
4102 &response,
4103 "ClientControlResponse::SupervisorAck",
4104 )?])
4105 }
4106
4107 async fn handle_supervisor_reload(
4108 &self,
4109 frame: Frame,
4110 module_id: String,
4111 ) -> Result<Vec<Frame>, RouterError> {
4112 let operation_lock = self.supervisor.operation_lock();
4113 let _operation_guard = operation_lock.lock().await;
4114 let Some(module) = self.supervisor.get(&module_id) else {
4115 return Ok(vec![control_error_frame(
4116 &frame,
4117 "unknown_module",
4118 format!("module_id '{module_id}' is not supervised"),
4119 )?]);
4120 };
4121
4122 self.route_outages.mark_operator_action(&module_id);
4123 if let Err(err) = module.reload().await {
4124 self.route_outages
4125 .operator_action_ended_unrefused(&module_id);
4126 let (code, message) = match err {
4127 crate::supervise::SuperviseError::Disabled { .. } => {
4128 ("module_disabled", err.to_string())
4129 }
4130 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4131 ("swap_in_progress", err.to_string())
4132 }
4133 _ => (
4134 "reload_failed",
4135 format!("failed to reload module_id '{module_id}': {err}"),
4136 ),
4137 };
4138 return Ok(vec![control_error_frame(&frame, code, message)?]);
4139 }
4140
4141 let response = ClientControlResponse::SupervisorAck {
4142 module_id,
4143 applied: true,
4144 };
4145 Ok(vec![control_response_body_frame(
4146 &frame,
4147 &response,
4148 "ClientControlResponse::SupervisorAck",
4149 )?])
4150 }
4151
4152 async fn handle_supervisor_rescan(
4153 &self,
4154 frame: Frame,
4155 preview: bool,
4156 ) -> Result<Vec<Frame>, RouterError> {
4157 let Some(context) = self.rescan.clone() else {
4158 return Ok(vec![control_error_frame(
4159 &frame,
4160 "rescan_unavailable",
4161 "the daemon was not started with a reloadable config path".to_string(),
4162 )?]);
4163 };
4164
4165 let operation_lock = self.supervisor.operation_lock();
4166 let _operation_guard = operation_lock.lock().await;
4167 let loaded = match crate::daemon_config::load(&context.config_path) {
4168 Ok(config) => config,
4169 Err(err) => {
4170 return Ok(vec![control_error_frame(
4171 &frame,
4172 "invalid_daemon_config",
4173 format!("supervisor rescan rejected daemon config: {err}"),
4174 )?])
4175 }
4176 };
4177 let Some(config) = loaded else {
4185 return Ok(vec![control_error_frame(
4186 &frame,
4187 "invalid_daemon_config",
4188 format!(
4189 "daemon config not found at {}; refusing to rescan (an absent config would \
4190 retire every supervised module)",
4191 context.config_path.display()
4192 ),
4193 )?]);
4194 };
4195 let (
4196 configured_port,
4197 storage_config,
4198 admission_facts_carrier_module_id,
4199 admission_facts_targets,
4200 scope_authority_owners,
4201 modules,
4202 reserved_capabilities,
4203 ) = (
4204 config.port,
4205 config.storage,
4206 config.admission_facts_carrier_module_id,
4207 config.admission_facts_targets,
4208 config.scope_authority_owners,
4209 config.modules,
4210 config.reserved_capabilities,
4211 );
4212
4213 let mut restart_required = Vec::new();
4222 for section in RestartRequiredSection::ALL {
4223 let changed = match section {
4224 RestartRequiredSection::Port => configured_port != context.configured_port,
4225 RestartRequiredSection::Storage => storage_config != context.storage_config,
4226 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4227 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4228 }
4229 RestartRequiredSection::AdmissionFactsTargets => {
4230 admission_facts_targets != context.admission_facts_targets
4231 }
4232 RestartRequiredSection::ScopeAuthorityOwners => {
4233 scope_authority_owners != context.scope_authority_owners
4234 }
4235 };
4236 if changed {
4237 restart_required.push(section.label().to_string());
4238 }
4239 }
4240 if !restart_required.is_empty() {
4241 warn!(
4242 config_path = %context.config_path.display(),
4243 sections = %restart_required.join(", "),
4244 "daemon config changed outside the modules section; restart the daemon to apply those changes"
4245 );
4246 }
4247
4248 for configured in &modules {
4249 if let Err(err) = validate_spec(&configured.module_spec()) {
4250 return Ok(vec![control_error_frame(
4251 &frame,
4252 "invalid_daemon_config",
4253 format!("supervisor rescan rejected daemon config: {err}"),
4254 )?]);
4255 }
4256 }
4257
4258 let configured_capabilities = modules
4259 .iter()
4260 .map(|module| (module.module_id.clone(), module.enabled))
4261 .collect::<Vec<_>>();
4262 let preview_capability_warnings = if preview {
4263 let (_, registrations) = self.runtime_capability_snapshot()?;
4264 let current_modules = self
4265 .supervisor
4266 .list()
4267 .into_iter()
4268 .map(|module| module.module_id().to_string())
4269 .collect::<BTreeSet<_>>();
4270 let resulting_modules = configured_capabilities.clone();
4271 let removed = current_modules
4272 .into_iter()
4273 .filter(|module_id| {
4274 !resulting_modules
4275 .iter()
4276 .any(|(configured_id, _)| configured_id == module_id)
4277 })
4278 .collect::<Vec<_>>();
4279 self.capability_evaluator.preview_removal_warnings(
4280 resulting_modules,
4281 &removed,
4282 ®istrations,
4283 )
4284 } else {
4285 Vec::new()
4286 };
4287 let result = match self
4288 .reconcile_supervised_modules(&context.supervisor, modules, preview)
4289 .await
4290 {
4291 Ok(result) => result,
4292 Err(message) => {
4293 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4294 }
4295 };
4296 if !preview {
4297 self.capability_evaluator
4298 .configure(configured_capabilities, reserved_capabilities);
4299 self.capability_evaluator.wake_deadline_loop();
4300 self.refresh_capability_requirements();
4301 }
4302 let mut result = result;
4303 result.restart_required = restart_required;
4304 result.capability_warnings = preview_capability_warnings;
4305 let response = ClientControlResponse::SupervisorRescan { result };
4306 Ok(vec![control_response_body_frame(
4307 &frame,
4308 &response,
4309 "ClientControlResponse::SupervisorRescan",
4310 )?])
4311 }
4312
4313 async fn handle_supervisor_release_reserved(
4314 &self,
4315 frame: Frame,
4316 module_id: String,
4317 ) -> Result<Vec<Frame>, RouterError> {
4318 let Some(context) = self.rescan.clone() else {
4319 return Ok(vec![control_error_frame(
4320 &frame,
4321 "release_unavailable",
4322 "reserved-id release requires a daemon started with a reloadable config path",
4323 )?]);
4324 };
4325 let operation_lock = self.supervisor.operation_lock();
4326 let _operation_guard = operation_lock.lock().await;
4327 let loaded = match crate::daemon_config::load(&context.config_path) {
4328 Ok(Some(config)) => config,
4329 Ok(None) => {
4330 return Ok(vec![control_error_frame(
4331 &frame,
4332 "invalid_daemon_config",
4333 format!(
4334 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4335 context.config_path.display()
4336 ),
4337 )?])
4338 }
4339 Err(err) => {
4340 return Ok(vec![control_error_frame(
4341 &frame,
4342 "invalid_daemon_config",
4343 format!("unable to verify reserved-id release against daemon config: {err}"),
4344 )?])
4345 }
4346 };
4347 if loaded
4348 .modules
4349 .iter()
4350 .any(|configured| configured.module_id == module_id)
4351 {
4352 return Ok(vec![control_error_frame(
4353 &frame,
4354 "reserved_module_configured",
4355 format!(
4356 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4357 ),
4358 )?]);
4359 }
4360 if !self.supervisor.release_retained_reserved_gate(&module_id) {
4361 return Ok(vec![control_error_frame(
4362 &frame,
4363 "reserved_gate_not_retained",
4364 format!(
4365 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4366 ),
4367 )?]);
4368 }
4369
4370 let response = ClientControlResponse::SupervisorAck {
4371 module_id,
4372 applied: true,
4373 };
4374 Ok(vec![control_response_body_frame(
4375 &frame,
4376 &response,
4377 "ClientControlResponse::SupervisorAck",
4378 )?])
4379 }
4380
4381 async fn reconcile_supervised_modules(
4390 &self,
4391 supervisor: &Supervisor,
4392 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4393 preview: bool,
4394 ) -> Result<SupervisorRescanResult, String> {
4395 let mut current = BTreeMap::new();
4396 for module in self.supervisor.list() {
4397 let (spec, health) = module.configuration().map_err(|err| {
4398 format!(
4399 "failed to read configuration for module_id '{}': {err}",
4400 module.module_id()
4401 )
4402 })?;
4403 let enabled = module
4404 .status()
4405 .map_err(|err| {
4406 format!(
4407 "failed to read status for module_id '{}': {err}",
4408 module.module_id()
4409 )
4410 })?
4411 .enabled;
4412 current.insert(
4413 module.module_id().to_string(),
4414 (module, spec, health, enabled),
4415 );
4416 }
4417 let configured = configured_modules
4418 .into_iter()
4419 .map(|module| (module.module_id.clone(), module))
4420 .collect::<BTreeMap<_, _>>();
4421
4422 let added = configured
4423 .keys()
4424 .filter(|module_id| !current.contains_key(*module_id))
4425 .cloned()
4426 .collect::<Vec<_>>();
4427 let removed = current
4428 .keys()
4429 .filter(|module_id| !configured.contains_key(*module_id))
4430 .cloned()
4431 .collect::<Vec<_>>();
4432 let mut changed_pending_reload = Vec::new();
4433 let mut configuration_changes = BTreeSet::new();
4434 let mut enabled_changes = BTreeSet::new();
4435 let mut unchanged = 0_u32;
4436
4437 for (module_id, configured_module) in &configured {
4438 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4439 else {
4440 continue;
4441 };
4442 let configuration_changed = *current_spec != configured_module.module_spec()
4443 || *current_health != configured_module.health;
4444 let enabled_changed = *current_enabled != configured_module.enabled;
4445 if configuration_changed {
4446 configuration_changes.insert(module_id.clone());
4447 changed_pending_reload.push(module_id.clone());
4448 }
4449 if enabled_changed {
4450 enabled_changes.insert(module_id.clone());
4451 }
4452 if !configuration_changed && !enabled_changed {
4453 unchanged = unchanged.saturating_add(1);
4454 }
4455 }
4456
4457 if preview {
4463 return Ok(SupervisorRescanResult {
4464 added,
4465 removed,
4466 changed_pending_reload,
4467 enabled_changes: enabled_changes.iter().cloned().collect(),
4468 unchanged,
4469 preview: true,
4470 restart_required: Vec::new(),
4474 capability_warnings: Vec::new(),
4475 });
4476 }
4477
4478 for module_id in &removed {
4479 let module = ¤t
4480 .get(module_id)
4481 .expect("removed module came from current supervisor state")
4482 .0;
4483 module.retire().await.map_err(|err| {
4484 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4485 })?;
4486 self.supervisor.record_rescan_removal(module_id);
4520 self.supervisor.retire(module_id);
4521 self.route_outages.forget(module_id);
4522 }
4523
4524 for module_id in configured.keys() {
4525 let Some((module, _, _, _)) = current.get(module_id) else {
4526 continue;
4527 };
4528 let configured_module = configured
4529 .get(module_id)
4530 .expect("configured module id came from configured map");
4531 if configuration_changes.contains(module_id) {
4532 module
4533 .update_configuration(
4534 configured_module.module_spec(),
4535 configured_module.health,
4536 configured_module.drain_timeout_ms,
4537 )
4538 .await
4539 .map_err(|err| {
4540 format!(
4541 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4542 )
4543 })?;
4544 }
4545 if enabled_changes.contains(module_id) {
4546 self.route_outages.mark_operator_action(module_id);
4549 module
4550 .set_enabled(configured_module.enabled)
4551 .await
4552 .map_err(|err| {
4553 self.route_outages.operator_action_ended_unrefused(module_id);
4554 format!(
4555 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4556 configured_module.enabled
4557 )
4558 })?;
4559 }
4560 }
4561
4562 for module_id in &added {
4563 let configured_module = configured
4564 .get(module_id)
4565 .expect("added module id came from configured map");
4566 supervisor
4567 .supervise_configured_with_health(
4568 configured_module.module_spec(),
4569 configured_module.enabled,
4570 configured_module.health,
4571 configured_module.drain_timeout_ms,
4572 configured_module.restart,
4573 )
4574 .map_err(|err| {
4575 format!("failed to add module_id '{module_id}' during rescan: {err}")
4576 })?;
4577 }
4578
4579 Ok(SupervisorRescanResult {
4580 added,
4581 removed,
4582 changed_pending_reload,
4583 enabled_changes: enabled_changes.iter().cloned().collect(),
4584 unchanged,
4585 preview: false,
4586 restart_required: Vec::new(),
4589 capability_warnings: Vec::new(),
4590 })
4591 }
4592
4593 async fn handle_supervisor_set_enabled(
4594 &self,
4595 frame: Frame,
4596 module_id: String,
4597 enabled: bool,
4598 ) -> Result<Vec<Frame>, RouterError> {
4599 let operation_lock = self.supervisor.operation_lock();
4600 let _operation_guard = operation_lock.lock().await;
4601 let Some(module) = self.supervisor.get(&module_id) else {
4602 return Ok(vec![control_error_frame(
4603 &frame,
4604 "unknown_module",
4605 format!("module_id '{module_id}' is not supervised"),
4606 )?]);
4607 };
4608
4609 self.route_outages.mark_operator_action(&module_id);
4612 let applied = match module.set_enabled(enabled).await {
4613 Ok(applied) => applied,
4614 Err(err) => {
4615 self.route_outages
4616 .operator_action_ended_unrefused(&module_id);
4617 return Ok(vec![control_error_frame(
4618 &frame,
4619 "target_unavailable",
4620 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
4621 )?]);
4622 }
4623 };
4624 if !applied {
4625 self.route_outages
4628 .operator_action_ended_unrefused(&module_id);
4629 }
4630
4631 self.capability_evaluator.wake_deadline_loop();
4632 self.refresh_capability_requirements();
4633 let response = ClientControlResponse::SupervisorAck { module_id, applied };
4634 Ok(vec![control_response_body_frame(
4635 &frame,
4636 &response,
4637 "ClientControlResponse::SupervisorAck",
4638 )?])
4639 }
4640
4641 async fn handle_supervisor_health_probe(
4642 &self,
4643 frame: Frame,
4644 module_id: String,
4645 ) -> Result<Vec<Frame>, RouterError> {
4646 self.refresh_capability_requirements();
4647 let Some(registration) = self
4648 .registry
4649 .get_module(&module_id)
4650 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4651 else {
4652 return Ok(vec![control_error_frame(
4653 &frame,
4654 "unknown_module",
4655 format!("module_id '{module_id}' is not registered"),
4656 )?]);
4657 };
4658
4659 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
4672 {
4673 return Ok(vec![control_error_frame(
4674 &frame,
4675 "health_not_advertised",
4676 format!("module_id '{module_id}' did not advertise health.check"),
4677 )?]);
4678 }
4679
4680 let deadline = Instant::now() + self.health_probe_timeout;
4681 let pending = match self.forwarding.begin_module_control_rpc_for(
4682 &module_id,
4683 MODULE_CONTROL_OP_HEALTH_CHECK,
4684 deadline,
4685 ) {
4686 Ok(pending) => pending,
4687 Err(err) => {
4688 return Ok(vec![control_error_frame(
4689 &frame,
4690 forwarding_error_code(&err),
4691 err.to_string(),
4692 )?])
4693 }
4694 };
4695
4696 let PendingModuleControlRpc {
4697 endpoint,
4698 module_sink,
4699 negotiated_ver,
4700 corr: probe_corr,
4701 receiver,
4702 } = pending;
4703 let mut guard =
4704 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
4705 let probe_body =
4706 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
4707 RouterError::backend(
4708 0,
4709 frame.header.corr,
4710 format!("failed to encode health.check request: {err}"),
4711 )
4712 })?;
4713 let probe_frame = Frame::build_with_version(
4714 negotiated_ver,
4715 FrameType::Request,
4716 control_flags(),
4717 0,
4718 0,
4719 probe_corr,
4720 probe_body,
4721 )
4722 .map_err(RouterError::FrameBuild)?;
4723
4724 if let Err(err) = module_sink.send(probe_frame).await {
4725 return Ok(vec![control_error_frame(
4726 &frame,
4727 "target_unavailable",
4728 err.to_string(),
4729 )?]);
4730 }
4731
4732 match timeout_at(deadline, receiver).await {
4733 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
4734 guard.disarm();
4735 let Some(report) = response.health_report() else {
4736 return Ok(vec![control_error_frame(
4737 &frame,
4738 "invalid_control_body",
4739 "health.check RPC returned a non-health response",
4740 )?]);
4741 };
4742 let HealthReport {
4747 status,
4748 detail,
4749 metrics,
4750 } = report;
4751 let capability_detail = self
4752 .capability_evaluator
4753 .required_problem_detail(&module_id);
4754 let response = ClientControlResponse::SupervisorHealthProbe {
4755 module_id,
4756 status,
4757 detail: append_capability_problem_detail(detail, capability_detail),
4758 metrics,
4759 };
4760 Ok(vec![control_response_body_frame(
4761 &frame,
4762 &response,
4763 "ClientControlResponse::SupervisorHealthProbe",
4764 )?])
4765 }
4766 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
4767 guard.disarm();
4768 Ok(vec![control_error_body_frame(&frame, body)?])
4769 }
4770 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
4771 guard.disarm();
4772 Ok(vec![control_error_frame(
4773 &frame,
4774 "target_unavailable",
4775 message,
4776 )?])
4777 }
4778 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
4779 guard.disarm();
4780 Ok(vec![control_error_frame(
4781 &frame,
4782 "invalid_control_body",
4783 message,
4784 )?])
4785 }
4786 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
4787 guard.disarm();
4788 Ok(vec![control_error_frame(
4789 &frame,
4790 "invalid_control_body",
4791 format!("expected module-control op '{expected}', got '{actual}'"),
4792 )?])
4793 }
4794 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
4795 guard.disarm();
4796 Ok(vec![control_error_frame(
4797 &frame,
4798 "module_timeout",
4799 format!(
4800 "module_id '{module_id}' answered health.check after {:?}",
4801 self.health_probe_timeout
4802 ),
4803 )?])
4804 }
4805 Ok(Err(_)) => Ok(vec![control_error_frame(
4806 &frame,
4807 "target_unavailable",
4808 "health.check waiter was canceled before the module responded",
4809 )?]),
4810 Err(_) => Ok(vec![control_error_frame(
4811 &frame,
4812 "module_timeout",
4813 format!(
4814 "module_id '{module_id}' did not answer health.check within {:?}",
4815 self.health_probe_timeout
4816 ),
4817 )?]),
4818 }
4819 }
4820
4821 fn supervisor_status(
4822 &self,
4823 module_id: &str,
4824 corr: u64,
4825 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
4826 self.supervisor
4827 .get(module_id)
4828 .map(|module| {
4829 let warming = module.is_warming_for_control("status").map_err(|err| {
4830 RouterError::backend(
4831 0,
4832 corr,
4833 format!(
4834 "failed to read supervisor warming state for module_id '{module_id}': {err}"
4835 ),
4836 )
4837 })?;
4838 module.status_for_control("status").map_err(|err| {
4839 RouterError::backend(
4840 0,
4841 corr,
4842 format!(
4843 "failed to read supervisor status for module_id '{module_id}': {err}"
4844 ),
4845 )
4846 }).map(|status| (status, warming))
4847 })
4848 .transpose()
4849 }
4850
4851 fn guard_module_control_op(
4852 &self,
4853 frame: &Frame,
4854 module_id: &str,
4855 op: &str,
4856 ) -> Result<Option<Frame>, RouterError> {
4857 if self.module_grants_op(module_id, op, frame.header.corr)? {
4858 return Ok(None);
4859 }
4860
4861 Ok(Some(control_error_frame(
4862 frame,
4863 "op_not_allowed",
4864 format!("module_id '{module_id}' did not grant control op '{op}'"),
4865 )?))
4866 }
4867
4868 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
4869 let Some(registration) = self
4870 .registry
4871 .get_module(module_id)
4872 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
4873 else {
4874 return Ok(false);
4875 };
4876 Ok(module_registration_grants_op(®istration.control_ops, op))
4877 }
4878
4879 fn handle_status_update(
4880 &self,
4881 endpoint: ModuleEndpointId,
4882 frame: Frame,
4883 ) -> Result<Vec<Frame>, RouterError> {
4884 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
4885 Ok(update) => update,
4886 Err(err) => {
4887 if is_known_module_push_op(&frame.body) {
4892 return Ok(vec![control_error_frame(
4893 &frame,
4894 "invalid_control_body",
4895 format!("malformed module control push body: {err}"),
4896 )?]);
4897 }
4898 return Ok(Vec::new());
4899 }
4900 };
4901
4902 match update {
4903 ModuleControlPush::RouteStatus {
4904 route_channel,
4905 route_epoch,
4906 status,
4907 } => {
4908 self.forwarding
4909 .cache_status(endpoint, route_channel, route_epoch, status)
4910 .map_err(RouterError::Forwarding)?;
4911 }
4912 }
4913 Ok(Vec::new())
4914 }
4915
4916 fn handle_route_poll(
4917 &self,
4918 ctx: &RouteCtx,
4919 frame: Frame,
4920 route_channel: u16,
4921 route_epoch: u32,
4922 kind: PollKind,
4923 ) -> Result<Vec<Frame>, RouterError> {
4924 let snapshot = self
4925 .forwarding
4926 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
4927 .map_err(RouterError::Forwarding)?;
4928 let response = match (kind, snapshot) {
4929 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
4930 ClientControlResponse::RoutePoll {
4931 route_channel,
4932 route_epoch,
4933 status,
4934 live: None,
4935 }
4936 }
4937 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4938 route_channel,
4939 route_epoch,
4940 status: None,
4941 live: None,
4942 },
4943 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
4944 let live = self
4970 .process_liveness
4971 .as_ref()
4972 .and_then(|source| source.process_live(&module_id))
4973 .unwrap_or(true);
4974 ClientControlResponse::RoutePoll {
4975 route_channel,
4976 route_epoch,
4977 status: None,
4978 live: Some(live),
4979 }
4980 }
4981 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
4982 route_channel,
4983 route_epoch,
4984 status: None,
4985 live: Some(false),
4986 },
4987 };
4988
4989 Ok(vec![control_response_body_frame(
4990 &frame,
4991 &response,
4992 "ClientControlResponse::RoutePoll",
4993 )?])
4994 }
4995
4996 pub(crate) fn observe_module_control_completion(
4997 &self,
4998 completion: ModuleControlRpcCompletion,
4999 ) -> bool {
5000 match completion {
5001 ModuleControlRpcCompletion::Unknown => false,
5002 ModuleControlRpcCompletion::Settled => true,
5003 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5004 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5005 info!(
5006 module_id = %module_id,
5007 latency_ms,
5008 "late health.check answer proves the module is alive"
5009 );
5010 match self
5011 .supervisor
5012 .record_late_health_answer(&module_id, latency_ms)
5013 {
5014 Ok(true) => {}
5015 Ok(false) => debug!(
5016 module_id = %module_id,
5017 latency_ms,
5018 "late health.check answer has no active supervisor snapshot"
5019 ),
5020 Err(err) => warn!(
5021 module_id = %module_id,
5022 latency_ms,
5023 error = %err,
5024 "failed to record late health.check answer"
5025 ),
5026 }
5027 true
5028 }
5029 }
5030 }
5031
5032 fn refuse_to_end_module_connection_for_a_client(
5053 &self,
5054 module_connection_id: ConnectionId,
5055 corr: u64,
5056 err: ForwardingError,
5057 ) -> Result<(), RouterError> {
5058 if let ForwardingError::ConnectionClosing { connection_id } = err {
5059 if connection_id != module_connection_id {
5060 warn!(
5061 module_connection_id = module_connection_id.get(),
5062 client_connection_id = connection_id.get(),
5063 corr,
5064 "dropping a route.bind response for a closing client; the module connection keeps serving"
5065 );
5066 return Ok(());
5067 }
5068 }
5069 Err(RouterError::Forwarding(err))
5070 }
5071
5072 fn handle_module_relay_response(
5073 &self,
5074 connection_id: ConnectionId,
5075 frame: Frame,
5076 ) -> Result<Vec<Frame>, RouterError> {
5077 let mut secondary_error = None;
5078 let outcome = match frame.header.ty {
5079 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5080 Ok(probe) if probe.op == "route.bind" => {
5081 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5082 Ok(ModuleControlResponse::RouteBindAck {}) => {
5083 RouteBindRelayOutcome::Accepted
5084 }
5085 Ok(other) => {
5086 let message =
5087 format!("route.bind response carried unexpected body: {other:?}");
5088 secondary_error = Some(control_error_frame(
5089 &frame,
5090 "invalid_control_body",
5091 message.clone(),
5092 )?);
5093 RouteBindRelayOutcome::ModuleGone(message)
5094 }
5095 Err(err) => {
5096 let message = format!("malformed route.bind response body: {err}");
5097 secondary_error = Some(control_error_frame(
5098 &frame,
5099 "invalid_control_body",
5100 message.clone(),
5101 )?);
5102 RouteBindRelayOutcome::ModuleGone(message)
5103 }
5104 }
5105 }
5106 Ok(probe) => {
5107 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5108 {
5109 Ok(response) => ModuleControlRpcOutcome::Response(response),
5110 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5111 "malformed {} response body: {err}",
5112 probe.op
5113 )),
5114 };
5115 let completion = self
5116 .forwarding
5117 .complete_module_control_rpc(
5118 connection_id,
5119 frame.header.corr,
5120 Some(&probe.op),
5121 outcome,
5122 )
5123 .map_err(RouterError::Forwarding)?;
5124 if !self.observe_module_control_completion(completion) {
5125 debug!(
5126 connection_id = connection_id.get(),
5127 corr = frame.header.corr,
5128 op = %probe.op,
5129 "dropping late or unknown module-control RPC response"
5130 );
5131 }
5132 return Ok(Vec::new());
5133 }
5134 Err(err) => {
5135 if let Some(expected_op) = self
5136 .forwarding
5137 .pending_module_control_op(connection_id, frame.header.corr)
5138 .map_err(RouterError::Forwarding)?
5139 {
5140 let completion = self
5141 .forwarding
5142 .complete_module_control_rpc(
5143 connection_id,
5144 frame.header.corr,
5145 None,
5146 ModuleControlRpcOutcome::MalformedResponse(format!(
5147 "malformed {expected_op} response body: {err}"
5148 )),
5149 )
5150 .map_err(RouterError::Forwarding)?;
5151 if !self.observe_module_control_completion(completion) {
5152 debug!(
5153 connection_id = connection_id.get(),
5154 corr = frame.header.corr,
5155 "dropping late malformed module-control RPC response"
5156 );
5157 }
5158 return Ok(Vec::new());
5159 }
5160 let message = format!("malformed route.bind response body: {err}");
5161 secondary_error = Some(control_error_frame(
5162 &frame,
5163 "invalid_control_body",
5164 message.clone(),
5165 )?);
5166 RouteBindRelayOutcome::ModuleGone(message)
5167 }
5168 },
5169 FrameType::Error => {
5170 if self
5171 .forwarding
5172 .pending_module_control_op(connection_id, frame.header.corr)
5173 .map_err(RouterError::Forwarding)?
5174 .is_some()
5175 {
5176 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5177 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5178 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5179 "malformed module-control ERROR body: {err}"
5180 )),
5181 };
5182 let completion = self
5183 .forwarding
5184 .complete_module_control_rpc(
5185 connection_id,
5186 frame.header.corr,
5187 None,
5188 outcome,
5189 )
5190 .map_err(RouterError::Forwarding)?;
5191 if !self.observe_module_control_completion(completion) {
5192 debug!(
5193 connection_id = connection_id.get(),
5194 corr = frame.header.corr,
5195 "dropping late or unknown module-control RPC error"
5196 );
5197 }
5198 return Ok(Vec::new());
5199 }
5200 match serde_json::from_slice::<ErrorBody>(&frame.body) {
5201 Ok(body) => RouteBindRelayOutcome::Rejected(body),
5202 Err(err) => {
5203 let message = format!("malformed route.bind ERROR body: {err}");
5204 secondary_error = Some(control_error_frame(
5205 &frame,
5206 "invalid_control_body",
5207 message.clone(),
5208 )?);
5209 RouteBindRelayOutcome::ModuleGone(message)
5210 }
5211 }
5212 }
5213 ty => {
5214 return Ok(vec![control_error_frame(
5215 &frame,
5216 "unsupported_control_frame",
5217 format!("unsupported module channel-0 frame {ty:?}"),
5218 )?])
5219 }
5220 };
5221
5222 let settled =
5223 self.forwarding
5224 .complete_pending_relay(connection_id, frame.header.corr, outcome);
5225 let completion = match settled {
5226 Ok(completion) => completion,
5227 Err(err) => {
5228 self.refuse_to_end_module_connection_for_a_client(
5229 connection_id,
5230 frame.header.corr,
5231 err,
5232 )?;
5233 return Ok(secondary_error.into_iter().collect());
5234 }
5235 };
5236 if let Some(target) = completion.abandoned.as_ref() {
5237 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5238 }
5239 if !completion.settled {
5240 debug!(
5241 connection_id = connection_id.get(),
5242 corr = frame.header.corr,
5243 frame_type = ?frame.header.ty,
5244 "dropping late or unknown route.bind relay response"
5245 );
5246 }
5247 Ok(secondary_error.into_iter().collect())
5248 }
5249
5250 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5251 debug!(connection_id = connection_id.get(), "handling GOODBYE");
5252 let registrations = self
5253 .deregister_connection(connection_id)
5254 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5255 let released_routes = self
5256 .forwarding
5257 .cleanup_connection(connection_id)
5258 .map_err(RouterError::Forwarding)?;
5259 self.emit_route_goodbyes(released_routes);
5260 if !registrations.is_empty() {
5262 crate::supervise::notify_registration_release();
5263 }
5264 Ok(Vec::new())
5265 }
5266}
5267
5268impl Default for ControlHandler {
5269 fn default() -> Self {
5270 Self::new(Arc::new(Registry::default()))
5271 }
5272}
5273
5274impl crate::supervise::SwapPromotionObserver for ControlHandler {
5275 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5276 self.apply_registration_capabilities(registration);
5277 }
5278}
5279
5280fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5281 CapabilityRequirementStatus {
5282 consumer: status.consumer,
5283 capability: status.capability,
5284 need: match status.need {
5285 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5286 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5287 },
5288 verdict: status.verdict.as_str().to_string(),
5289 episode_seq: status.episode_seq,
5290 config_satisfiable: status.config_satisfiable,
5291 runtime_available: status.runtime_available,
5292 detail: status.detail,
5293 }
5294}
5295
5296fn append_capability_problem_detail(
5297 detail: Option<String>,
5298 capability_detail: Option<String>,
5299) -> Option<String> {
5300 match (detail, capability_detail) {
5301 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5302 (Some(detail), None) => Some(detail),
5303 (None, Some(capability_detail)) => Some(capability_detail),
5304 (None, None) => None,
5305 }
5306}
5307
5308fn subc_ops() -> Vec<String> {
5309 SUBC_CONTROL_OPS
5310 .iter()
5311 .map(|op| (*op).to_string())
5312 .collect()
5313}
5314
5315fn module_subc_ops() -> Vec<String> {
5316 SUBC_CONTROL_OPS
5317 .iter()
5318 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5319 .map(|op| (*op).to_string())
5320 .collect()
5321}
5322
5323#[cfg(test)]
5324fn module_baseline_control_ops() -> Vec<String> {
5325 MODULE_BASELINE_CONTROL_OPS
5326 .iter()
5327 .map(|op| (*op).to_string())
5328 .collect()
5329}
5330
5331fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5332 let mut seen = HashSet::new();
5333 let mut effective = Vec::new();
5334 for op in MODULE_BASELINE_CONTROL_OPS {
5335 if seen.insert((*op).to_string()) {
5336 effective.push((*op).to_string());
5337 }
5338 }
5339 for op in declared.unwrap_or_default() {
5340 if seen.insert(op.clone()) {
5341 effective.push(op);
5342 }
5343 }
5344 effective
5345}
5346
5347fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5348 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5349}
5350
5351fn target_module_id(target: &RouteTarget) -> &str {
5352 match target {
5353 RouteTarget::ToolProvider { module_id }
5354 | RouteTarget::ManagementSurface { module_id }
5355 | RouteTarget::InternalService { module_id, .. } => module_id,
5356 }
5357}
5358
5359fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5360 roles.iter().any(|role| match (target, role) {
5361 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5362 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5363 (
5364 RouteTarget::InternalService { service_id, .. },
5365 ProviderRole::InternalService {
5366 service_id: provided,
5367 ..
5368 },
5369 ) => service_id == provided,
5370 _ => false,
5371 })
5372}
5373
5374fn is_routable_role(role: &ProviderRole) -> bool {
5375 matches!(
5376 role,
5377 ProviderRole::ToolProvider { .. }
5378 | ProviderRole::ManagementSurface { .. }
5379 | ProviderRole::InternalService { .. }
5380 )
5381}
5382
5383#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5384enum ControlRequestBodyError {
5385 UnknownOp,
5386 InvalidBody,
5387}
5388
5389#[derive(Debug, Deserialize)]
5390struct ControlOpProbe {
5391 op: String,
5392}
5393
5394const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5397
5398fn is_known_module_push_op(body: &[u8]) -> bool {
5399 serde_json::from_slice::<ControlOpProbe>(body)
5400 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5401 .unwrap_or(false)
5402}
5403
5404fn is_known_module_request_op(body: &[u8]) -> bool {
5405 serde_json::from_slice::<ControlOpProbe>(body)
5406 .map(|probe| is_module_to_subc_op(&probe.op))
5407 .unwrap_or(false)
5408}
5409
5410fn is_module_to_subc_op(op: &str) -> bool {
5411 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5412}
5413
5414fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5415 debug!(
5416 op = %op,
5417 connection_id = connection_id.get(),
5418 corr,
5419 "control dispatch"
5420 );
5421}
5422
5423fn log_slow_control_dispatch(
5424 dispatch_started_at: Option<StdInstant>,
5425 op: &'static str,
5426 connection_id: ConnectionId,
5427 corr: u64,
5428) {
5429 let Some(dispatch_started_at) = dispatch_started_at else {
5430 return;
5431 };
5432 let elapsed = dispatch_started_at.elapsed();
5433 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5434 warn!(
5435 op = %op,
5436 connection_id = connection_id.get(),
5437 corr,
5438 elapsed_ms = elapsed.as_millis() as u64,
5439 "slow control dispatch"
5440 );
5441 }
5442}
5443
5444fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5445 match request {
5446 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5447 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5448 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5449 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5450 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5451 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5452 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5453 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5454 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5455 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5456 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5457 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5458 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5459 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5460 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5461 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5462 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5463 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5464 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5465 }
5466}
5467
5468fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5469 match request {
5470 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5471 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5472 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
5473 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
5474 }
5475}
5476
5477fn parse_client_control_request(
5478 body: &[u8],
5479) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5480 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5481 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5482 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5483 ControlRequestBodyError::InvalidBody
5484 }
5485 Ok(_) => ControlRequestBodyError::UnknownOp,
5486 Err(_) => ControlRequestBodyError::InvalidBody,
5487 };
5488 (err, classification)
5489 })
5490}
5491
5492fn parse_module_control_request_from_module(
5493 body: &[u8],
5494) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5495 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5496 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5497 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
5498 Ok(_) => ControlRequestBodyError::UnknownOp,
5499 Err(_) => ControlRequestBodyError::InvalidBody,
5500 };
5501 (err, classification)
5502 })
5503}
5504
5505#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5506enum ProviderRoleKind {
5507 ToolProvider,
5508 PipelineStage,
5509 ManagementSurface,
5510 InternalService,
5511}
5512
5513fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5514 match role {
5515 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5516 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5517 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5518 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5519 }
5520}
5521
5522fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5523 roles.iter().map(provider_role_kind).collect()
5524}
5525
5526fn capability_census_trigger(
5530 old: Option<&CapabilityDeclarations>,
5531 new: Option<&CapabilityDeclarations>,
5532) -> bool {
5533 let old_provides = old
5534 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
5535 .unwrap_or_default();
5536 let old_denies = old
5537 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
5538 .unwrap_or_default();
5539 let new = new.cloned().unwrap_or(CapabilityDeclarations {
5540 provides: Vec::new(),
5541 requires: Vec::new(),
5542 must_never_reach: Vec::new(),
5543 });
5544
5545 new.provides
5546 .iter()
5547 .any(|capability| !old_provides.contains(capability))
5548 || new
5549 .must_never_reach
5550 .iter()
5551 .any(|capability| !old_denies.contains(capability))
5552}
5553
5554fn denied_capability<'a>(
5557 opening_manifest: &'a ModuleManifest,
5558 target_manifest: &ModuleManifest,
5559) -> Option<&'a str> {
5560 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
5561 let target_capabilities = target_manifest.capabilities.as_ref()?;
5562 opening_capabilities
5563 .must_never_reach
5564 .iter()
5565 .find(|denied| {
5566 target_capabilities
5567 .provides
5568 .iter()
5569 .any(|provided| provided == *denied)
5570 })
5571 .map(String::as_str)
5572}
5573
5574fn catalog_update_frozen_field_message(
5575 registered: &ModuleManifest,
5576 provides: &[ProviderRole],
5577) -> Option<String> {
5578 let old_has_provides = !registered.provides.is_empty();
5579 let new_has_provides = !provides.is_empty();
5580 if old_has_provides != new_has_provides {
5581 return Some(format!(
5582 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
5583 registered.module_id
5584 ));
5585 }
5586
5587 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
5588 return Some(format!(
5589 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
5590 registered.module_id
5591 ));
5592 }
5593
5594 let registered_concurrency = manifest_concurrency(registered);
5595 let mut candidate = registered.clone();
5596 candidate.provides = provides.to_vec();
5597 let candidate_concurrency = manifest_concurrency(&candidate);
5598 if candidate_concurrency != registered_concurrency {
5599 return Some(format!(
5600 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
5601 registered.module_id, registered_concurrency, candidate_concurrency
5602 ));
5603 }
5604
5605 None
5608}
5609
5610fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
5611 manifest.provides.iter().any(is_routable_role)
5612}
5613
5614fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
5620 manifest
5621 .provides
5622 .iter()
5623 .find_map(|provider| match provider {
5624 ProviderRole::ToolProvider { concurrency, .. }
5625 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
5626 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
5627 })
5628 .unwrap_or(Concurrency::ModuleManaged)
5629}
5630
5631fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
5638 let has_management_surface = manifest
5639 .provides
5640 .iter()
5641 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
5642 if !has_management_surface {
5643 return false;
5644 }
5645 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
5646 return false;
5647 };
5648 let Some(provides) = raw
5649 .get("manifest")
5650 .and_then(|manifest| manifest.get("provides"))
5651 .and_then(serde_json::Value::as_array)
5652 else {
5653 return false;
5654 };
5655 provides.iter().any(|role| {
5660 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
5661 && role.get("concurrency").is_none()
5662 })
5663}
5664
5665fn negotiate_version(peer_version: u8) -> Result<u8, String> {
5666 if peer_version != PROTOCOL_VERSION {
5667 return Err(format!(
5668 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
5669 ));
5670 }
5671 Ok(PROTOCOL_VERSION)
5672}
5673
5674fn pong(frame: &Frame) -> Result<Frame, RouterError> {
5675 Frame::build_with_version(
5676 response_version(frame),
5677 FrameType::Pong,
5678 frame.header.flags,
5679 0,
5680 0,
5681 frame.header.corr,
5682 Vec::new(),
5683 )
5684 .map_err(RouterError::FrameBuild)
5685}
5686
5687fn control_error_frame(
5688 frame: &Frame,
5689 code: &'static str,
5690 message: impl Into<String>,
5691) -> Result<Frame, RouterError> {
5692 control_error_body_frame(
5693 frame,
5694 ErrorBody {
5695 code: code.to_string(),
5696 message: message.into(),
5697 detail: None,
5698 },
5699 )
5700}
5701
5702fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
5703 let body = serde_json::to_vec(&error).map_err(|err| {
5704 RouterError::backend(
5705 0,
5706 frame.header.corr,
5707 format!("failed to encode control ERROR: {err}"),
5708 )
5709 })?;
5710
5711 Frame::build_with_version(
5712 response_version(frame),
5713 FrameType::Error,
5714 control_flags(),
5715 0,
5716 0,
5717 frame.header.corr,
5718 body,
5719 )
5720 .map_err(RouterError::FrameBuild)
5721}
5722
5723fn control_response_body_frame<T: Serialize>(
5724 frame: &Frame,
5725 reply: &T,
5726 label: &'static str,
5727) -> Result<Frame, RouterError> {
5728 let body = serde_json::to_vec(reply).map_err(|err| {
5729 RouterError::backend(
5730 0,
5731 frame.header.corr,
5732 format!("failed to encode {label}: {err}"),
5733 )
5734 })?;
5735
5736 Frame::build_with_version(
5737 response_version(frame),
5738 FrameType::Response,
5739 control_flags(),
5740 0,
5741 0,
5742 frame.header.corr,
5743 body,
5744 )
5745 .map_err(RouterError::FrameBuild)
5746}
5747
5748fn forwarding_error_code(err: &ForwardingError) -> &'static str {
5778 match err {
5779 ForwardingError::NoModuleConnection => "target_unavailable",
5780 ForwardingError::ModuleReloading { .. } => "module_reloading",
5781 ForwardingError::ClientRouteChannelExhausted { .. }
5782 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
5783 ForwardingError::StaleModuleEndpoint
5784 | ForwardingError::UnknownReservation { .. }
5785 | ForwardingError::ConnectionClosing { .. }
5786 | ForwardingError::ClientEgressClosed { .. }
5787 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
5788 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
5791 ForwardingError::RelayCorrelationExhausted
5792 | ForwardingError::RouteOpenBuild(_)
5793 | ForwardingError::Poisoned => "forwarding_error",
5794 }
5795}
5796
5797fn response_version(frame: &Frame) -> u8 {
5798 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
5799 frame.header.ver
5800 } else {
5801 PROTOCOL_VERSION
5802 }
5803}
5804
5805fn control_flags() -> Flags {
5806 Flags::new(false, Priority::Passive, false)
5807}
5808
5809fn send_goodbye_target_best_effort(
5814 counters: &DaemonCounters,
5815 target: &GoodbyeTarget,
5816 context: &'static str,
5817) {
5818 let Ok(frame) = Frame::build_with_version(
5819 target.negotiated_ver,
5820 FrameType::Goodbye,
5821 control_flags(),
5822 target.channel,
5823 target.epoch,
5824 0,
5825 Vec::new(),
5826 ) else {
5827 return;
5828 };
5829 crate::forwarding::send_module_route_goodbye(
5830 counters,
5831 &target.sink,
5832 frame,
5833 target.module_id.as_deref(),
5834 context,
5835 );
5836}
5837
5838pub(crate) fn send_route_control_pushes(
5839 forwarding: &ForwardingTable,
5840 routes: Vec<EndpointRoute>,
5841 push: ClientControlPush,
5842) {
5843 let body = match serde_json::to_vec(&push) {
5844 Ok(body) => body,
5845 Err(err) => {
5846 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
5847 return;
5848 }
5849 };
5850 let mut targets = Vec::new();
5851 for route in routes {
5852 let target = route.goodbye_target;
5853 if let Some(existing) = targets
5854 .iter()
5855 .find(|existing: &&GoodbyeTarget| existing.connection_id == target.connection_id)
5856 {
5857 debug_assert_eq!(
5858 existing.negotiated_ver, target.negotiated_ver,
5859 "one connection cannot negotiate multiple frame versions"
5860 );
5861 continue;
5862 }
5863 targets.push(target);
5864 }
5865 for target in targets {
5866 let frame = match Frame::build_with_version(
5867 target.negotiated_ver,
5868 FrameType::Push,
5869 control_flags(),
5870 0,
5871 0,
5872 0,
5873 body.clone(),
5874 ) {
5875 Ok(frame) => frame,
5876 Err(err) => {
5877 warn!(
5878 route_channel = target.channel,
5879 error = %err,
5880 "failed to build route lifecycle control PUSH frame"
5881 );
5882 continue;
5883 }
5884 };
5885 if let Err(err) = target.sink.try_send(frame) {
5886 if target.close_on_delivery_failure() {
5887 warn!(
5888 target_connection_id = target.connection_id.get(),
5889 route_channel = target.channel,
5890 error = %err,
5891 "route lifecycle control PUSH was not delivered to client; closing target connection"
5892 );
5893 let _ = forwarding.escalate_client_delivery_failure(
5894 target.connection_id,
5895 target.channel,
5896 target.epoch,
5897 CloseReason::new(
5898 "route_lifecycle_push_delivery_failed",
5899 format!(
5900 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
5901 target.channel
5902 ),
5903 ),
5904 crate::forwarding::UndeliveredFrame {
5905 module_id: target.module_id.as_deref(),
5906 sink: &target.sink,
5907 },
5908 );
5909 }
5910 }
5911 }
5912}
5913
5914#[cfg(test)]
5915mod tests {
5916 use std::{
5917 collections::BTreeMap,
5918 fmt,
5919 path::PathBuf,
5920 sync::{Arc, Mutex},
5921 time::Duration,
5922 };
5923 use subc_test_support::TestTempDir;
5924
5925 use serde_json::{json, Value};
5926 use subc_protocol::{
5927 manifest::{
5928 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
5929 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
5930 },
5931 session::HealthStatus,
5932 FrameType,
5933 };
5934
5935 use super::*;
5936 use crate::{
5937 forwarding::{DataRoute, DataRouteState},
5938 registry::ChannelState,
5939 router::FrameSink,
5940 stderr_tail::DEFAULT_MAX_LINE_BYTES,
5941 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
5942 RouteCtx, Router,
5943 };
5944 use tokio::{
5945 sync::mpsc,
5946 time::{sleep, Instant},
5947 };
5948 use tracing::{
5949 field::{Field, Visit},
5950 Event, Subscriber,
5951 };
5952 use tracing_subscriber::{layer::Context, prelude::*, Layer};
5953
5954 fn fake_aft_stub_path() -> PathBuf {
5973 let mut path = std::env::current_exe().expect("current_exe available in tests");
5974 path.pop(); path.pop(); path.push(if cfg!(windows) {
5977 "fake-aft-stub.exe"
5978 } else {
5979 "fake-aft-stub"
5980 });
5981 assert!(
5982 path.exists(),
5983 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
5984 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
5985 path.display()
5986 );
5987 path
5988 }
5989
5990 fn client_retries(code: &str) -> bool {
5995 subc_protocol::error_codes::is_retryable_route_open(code)
5996 }
5997
5998 #[test]
6009 fn retryability_of_forwarding_codes_matches_the_failure() {
6010 let transient = [
6013 ForwardingError::NoModuleConnection,
6014 ForwardingError::ModuleReloading {
6015 module_id: "m".into(),
6016 },
6017 ForwardingError::StaleModuleEndpoint,
6018 ForwardingError::UnknownReservation {
6019 client_channel: 1,
6020 module_channel: 1,
6021 },
6022 ForwardingError::ConnectionClosing {
6023 connection_id: ConnectionId::new(1),
6024 },
6025 ForwardingError::ClientEgressClosed {
6026 connection_id: ConnectionId::new(1),
6027 },
6028 ForwardingError::ModuleEgressUnavailable {
6029 connection_id: ConnectionId::new(1),
6030 },
6031 ];
6032 for err in transient {
6033 let code = forwarding_error_code(&err);
6034 assert!(
6035 client_retries(code),
6036 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6037 );
6038 }
6039
6040 let permanent = [
6045 ForwardingError::ClientRouteChannelExhausted {
6046 connection_id: ConnectionId::new(1),
6047 },
6048 ForwardingError::ModuleRouteChannelExhausted {
6049 endpoint: ModuleEndpointId {
6050 connection_id: ConnectionId::new(1),
6051 generation: 1,
6052 },
6053 },
6054 ForwardingError::RelayCorrelationExhausted,
6055 ForwardingError::RouteOpenBuild("x".into()),
6056 ForwardingError::Poisoned,
6057 ];
6058 for err in permanent {
6059 let code = forwarding_error_code(&err);
6060 assert!(
6061 !client_retries(code),
6062 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6063 );
6064 }
6065 }
6066
6067 #[tokio::test]
6095 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6096 let handler = ControlHandler::default();
6097 let frame =
6098 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6099
6100 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6104 assert_eq!(
6105 stamped,
6106 Principal::Direct,
6107 "a caller that proved nothing must not be stamped as a supervised module"
6108 );
6109
6110 let forged = handler
6115 .route_open_principal(
6116 &frame,
6117 Some(ConsumerIdentity {
6118 module_id: "aft".to_string(),
6119 launch_nonce: "not-a-real-nonce".to_string(),
6120 }),
6121 )
6122 .unwrap();
6123 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6124 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6125 }
6126
6127 #[test]
6144 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6145 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6146 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6147 let ClientControlRequest::RouteOpen {
6148 consumer_identity, ..
6149 } = parsed
6150 else {
6151 panic!("route.open body must parse as RouteOpen");
6152 };
6153 assert_eq!(
6154 consumer_identity,
6155 Some(ConsumerIdentity {
6156 module_id: "aft".to_string(),
6157 launch_nonce: "n".to_string(),
6158 }),
6159 "the wire field name must reach the value route_open_principal reads"
6160 );
6161 }
6162
6163 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6164 ModuleManifest::builder(module_id, "0.1.0")
6165 .protocol_ver(protocol_ver)
6166 .provides(vec![ProviderRole::ToolProvider {
6167 tools: vec![Tool {
6168 name: "read".to_string(),
6169 description: None,
6170 execution_mode: ExecutionMode::Pure,
6171 schema: json!({"type": "object"}),
6172 }],
6173 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6174 concurrency: Concurrency::ModuleManaged,
6175 emits_push: true,
6176 sub_supervises: true,
6177 }])
6178 .build()
6179 }
6180
6181 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6182 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6183 }
6184
6185 fn hello_frame_with_control_ops(
6186 module_id: &str,
6187 protocol_ver: u8,
6188 corr: u64,
6189 control_ops: Option<Vec<String>>,
6190 ) -> Frame {
6191 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6192 }
6193
6194 fn hello_frame_with_nonce(
6195 module_id: &str,
6196 protocol_ver: u8,
6197 corr: u64,
6198 launch_nonce: Option<&str>,
6199 ) -> Frame {
6200 hello_frame_full(
6201 module_id,
6202 protocol_ver,
6203 corr,
6204 None,
6205 launch_nonce.map(ToOwned::to_owned),
6206 )
6207 }
6208
6209 fn hello_frame_full(
6210 module_id: &str,
6211 protocol_ver: u8,
6212 corr: u64,
6213 control_ops: Option<Vec<String>>,
6214 launch_nonce: Option<String>,
6215 ) -> Frame {
6216 let body = serde_json::to_vec(&ModuleHelloBody {
6217 manifest: manifest(module_id, protocol_ver),
6218 protocol_ver,
6219 control_ops,
6220 launch_nonce,
6221 })
6222 .unwrap();
6223 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6224 }
6225
6226 fn non_routable_hello_frame_with_control_ops(
6227 module_id: &str,
6228 corr: u64,
6229 control_ops: Option<Vec<String>>,
6230 ) -> Frame {
6231 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6232 manifest.provides.clear();
6233 let body = serde_json::to_vec(&ModuleHelloBody {
6234 manifest,
6235 protocol_ver: PROTOCOL_VERSION,
6236 control_ops,
6237 launch_nonce: None,
6238 })
6239 .unwrap();
6240 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6241 }
6242
6243 fn capability_grammar_hello_frame(
6244 capabilities: Value,
6245 runtime_computed: Option<Value>,
6246 corr: u64,
6247 ) -> Frame {
6248 let mut body = serde_json::to_value(ModuleHelloBody {
6249 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6250 protocol_ver: PROTOCOL_VERSION,
6251 control_ops: None,
6252 launch_nonce: None,
6253 })
6254 .expect("HELLO body serializes");
6255 body["manifest"]["capabilities"] = capabilities;
6256 if let Some(runtime_computed) = runtime_computed {
6257 body["runtime_computed"] = runtime_computed;
6258 }
6259 Frame::build(
6260 FrameType::Hello,
6261 control_flags(),
6262 0,
6263 0,
6264 corr,
6265 serde_json::to_vec(&body).expect("HELLO body reserializes"),
6266 )
6267 .expect("HELLO frame builds")
6268 }
6269
6270 fn channel_request(channel: u16, corr: u64) -> Frame {
6271 Frame::build(
6272 FrameType::Request,
6273 Flags::new(true, Priority::Interactive, false),
6274 channel,
6275 0,
6276 corr,
6277 b"opaque".to_vec(),
6278 )
6279 .unwrap()
6280 }
6281
6282 fn route_ctx(
6283 connection_id: ConnectionId,
6284 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
6285 let (tx, rx) = mpsc::channel(8);
6286 (
6287 RouteCtx {
6288 connection_id,
6289 egress: FrameSink::new(tx),
6290 },
6291 rx,
6292 )
6293 }
6294
6295 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
6296 serde_json::from_slice(&frame.body).unwrap()
6297 }
6298
6299 async fn hello_via_sink(
6304 handler: &ControlHandler,
6305 ctx: &RouteCtx,
6306 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6307 hello: Frame,
6308 ) -> Frame {
6309 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
6310 assert!(
6311 replies.is_empty(),
6312 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
6313 );
6314 let ack = rx
6315 .try_recv()
6316 .expect("HELLO_ACK is queued on the module sink")
6317 .frame;
6318 assert_eq!(ack.header.ty, FrameType::HelloAck);
6319 ack
6320 }
6321
6322 fn parse_error(frame: &Frame) -> Value {
6323 serde_json::from_slice(&frame.body).unwrap()
6324 }
6325
6326 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
6327 serde_json::from_slice(&frame.body).unwrap()
6328 }
6329
6330 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
6331 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
6332 route_channel,
6333 route_epoch: 0,
6334 kind,
6335 })
6336 .unwrap();
6337 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6338 }
6339
6340 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
6341 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
6342 module_id: module_id.to_string(),
6343 })
6344 .unwrap();
6345 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6346 }
6347
6348 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
6349 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
6350 }
6351
6352 fn route_open_frame_with_consumer_capabilities(
6353 corr: u64,
6354 module_id: &str,
6355 project_root: TestTempDir,
6356 consumer_capabilities: Option<Vec<String>>,
6357 ) -> Frame {
6358 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6359 target: RouteTarget::ToolProvider {
6360 module_id: module_id.to_string(),
6361 },
6362 identity: BindIdentity::new(
6363 project_root.path().to_path_buf(),
6364 "unit".to_string(),
6365 "session".to_string(),
6366 ),
6367 consumer_identity: None,
6368 consumer_capabilities,
6369 admission_facts: None,
6370 scope: None,
6371 })
6372 .unwrap();
6373 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6374 }
6375
6376 fn route_open_frame_with_admission_facts(
6377 corr: u64,
6378 module_id: &str,
6379 project_root: TestTempDir,
6380 consumer_identity: Option<subc_control::ConsumerIdentity>,
6381 facts: Option<Value>,
6382 ) -> Frame {
6383 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6384 target: RouteTarget::ToolProvider {
6385 module_id: module_id.to_string(),
6386 },
6387 identity: BindIdentity::new(
6388 project_root.path().to_path_buf(),
6389 "unit".to_string(),
6390 format!("session-{corr}"),
6391 ),
6392 consumer_identity,
6393 consumer_capabilities: None,
6394 admission_facts: facts,
6395 scope: None,
6396 })
6397 .unwrap();
6398 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6399 }
6400
6401 #[derive(Clone, Default)]
6402 struct EventCapture {
6403 events: Arc<Mutex<Vec<CapturedEvent>>>,
6404 }
6405
6406 #[derive(Clone, Debug)]
6407 struct CapturedEvent {
6408 target: String,
6409 level: tracing::Level,
6410 fields: BTreeMap<String, String>,
6411 }
6412
6413 impl EventCapture {
6414 fn events(&self) -> Vec<CapturedEvent> {
6415 self.events.lock().unwrap().clone()
6416 }
6417 }
6418
6419 impl<S> Layer<S> for EventCapture
6420 where
6421 S: Subscriber,
6422 {
6423 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
6424 let mut visitor = EventFieldVisitor::default();
6425 event.record(&mut visitor);
6426 self.events.lock().unwrap().push(CapturedEvent {
6427 target: event.metadata().target().to_string(),
6428 level: *event.metadata().level(),
6429 fields: visitor.fields,
6430 });
6431 }
6432 }
6433
6434 #[derive(Default)]
6435 struct EventFieldVisitor {
6436 fields: BTreeMap<String, String>,
6437 }
6438
6439 impl Visit for EventFieldVisitor {
6440 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
6441 self.fields
6442 .insert(field.name().to_string(), format!("{value:?}"));
6443 }
6444 }
6445
6446 fn health_response(corr: u64, status: HealthStatus) -> Frame {
6447 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
6448 status,
6449 detail: Some("warming".to_string()),
6450 metrics: Some(json!({"queue_depth": 3})),
6451 })
6452 .unwrap();
6453 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6454 }
6455
6456 fn route_bind_ack(corr: u64) -> Frame {
6457 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
6458 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
6459 }
6460
6461 fn unique_project_root(label: &str) -> TestTempDir {
6462 TestTempDir::new(label)
6463 }
6464
6465 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
6466 match parse_route_poll(frame) {
6467 ClientControlResponse::RoutePoll {
6468 status: None,
6469 live: Some(live),
6470 ..
6471 } => assert_eq!(live, expected_live),
6472 other => panic!("unexpected route.poll response: {other:?}"),
6473 }
6474 }
6475
6476 fn bind_liveness_route(
6477 registry: &Registry,
6478 forwarding: &ForwardingTable,
6479 module_id: &str,
6480 ) -> (RouteCtx, u16, u32) {
6481 let module_connection = ConnectionId::new(101);
6482 let client_connection = ConnectionId::new(202);
6483 let registration = registry
6484 .register_with_control_ops(
6485 manifest(module_id, PROTOCOL_VERSION),
6486 PROTOCOL_VERSION,
6487 module_connection,
6488 module_baseline_control_ops(),
6489 )
6490 .unwrap();
6491 let (module_tx, _module_rx) = mpsc::channel(8);
6492 let endpoint = forwarding
6493 .register_module_connection(
6494 module_connection,
6495 module_id.to_string(),
6496 PROTOCOL_VERSION,
6497 manifest_concurrency(®istration.manifest),
6498 FrameSink::new(module_tx),
6499 )
6500 .unwrap();
6501 let (client_ctx, _client_rx) = route_ctx(client_connection);
6502 let pending = forwarding
6503 .begin_route_bind_relay_for_test(
6504 client_connection,
6505 client_ctx.egress.clone(),
6506 1,
6507 module_id,
6508 )
6509 .unwrap();
6510 assert_eq!(pending.endpoint, endpoint);
6511 let route_channel = pending.client_channel;
6512 let route_epoch = pending.client_epoch;
6513 forwarding
6514 .complete_pending_relay(
6515 module_connection,
6516 pending.corr,
6517 RouteBindRelayOutcome::Accepted,
6518 )
6519 .unwrap();
6520 (client_ctx, route_channel, route_epoch)
6521 }
6522
6523 struct FakeProcessLiveness {
6524 live: Option<bool>,
6525 }
6526
6527 impl ModuleProcessLiveness for FakeProcessLiveness {
6528 fn process_live(&self, _module_id: &str) -> Option<bool> {
6529 self.live
6530 }
6531 }
6532
6533 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6534 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
6535 {
6536 let registry = Arc::new(Registry::default());
6537 let supervisor_handle = SupervisorHandle::new();
6538 let supervisor = Supervisor::new(
6539 Arc::clone(®istry),
6540 RestartPolicy::new(1, Duration::from_millis(10)),
6541 )
6542 .with_handle(supervisor_handle.clone());
6543 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
6544 let module = supervisor
6545 .spawn(ModuleSpec {
6546 module_id: "stderr-tail-wire".to_string(),
6547 program: fake_aft_stub_path(),
6548 args: Vec::new(),
6549 env: vec![
6550 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
6551 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
6552 ],
6553 reserved: false,
6554 reserved_prefixes: Vec::new(),
6555 protocol: ModuleProtocol::Subc,
6556 overlap: Default::default(),
6557 })
6558 .unwrap();
6559
6560 let deadline = Instant::now() + Duration::from_secs(5);
6561 loop {
6562 let tail = module.stderr_tail(None, None);
6563 if tail
6564 .entries
6565 .iter()
6566 .any(|entry| matches!(entry, TailEntry::ProcessStart))
6567 && tail.entries.iter().any(|entry| {
6568 matches!(
6569 entry,
6570 TailEntry::Line {
6571 truncated: true,
6572 ..
6573 }
6574 )
6575 })
6576 {
6577 break;
6578 }
6579 assert!(
6580 Instant::now() < deadline,
6581 "module did not produce a truncated line and restart boundary: {tail:?}"
6582 );
6583 sleep(Duration::from_millis(10)).await;
6584 }
6585
6586 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6587 let request = ClientControlRequest::SupervisorStderrTail {
6588 module_id: "stderr-tail-wire".to_string(),
6589 max_lines: None,
6590 max_bytes: None,
6591 };
6592 let frame = Frame::build(
6593 FrameType::Request,
6594 control_flags(),
6595 0,
6596 0,
6597 1,
6598 serde_json::to_vec(&request).unwrap(),
6599 )
6600 .unwrap();
6601 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6602 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6603 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
6604 serde_json::from_slice(&responses[0].body).unwrap()
6605 else {
6606 panic!("expected supervisor.stderr_tail response");
6607 };
6608
6609 assert!(
6610 tail.entries
6611 .iter()
6612 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
6613 "the control response lost the restart boundary"
6614 );
6615 let Some(StderrTailEntry::Line {
6616 text,
6617 truncated,
6618 at_ms,
6619 }) = tail.entries.iter().find(|entry| {
6620 matches!(
6621 entry,
6622 StderrTailEntry::Line {
6623 truncated: true,
6624 ..
6625 }
6626 )
6627 })
6628 else {
6629 panic!("the control response lost the truncated line");
6630 };
6631 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
6632 assert!(*truncated);
6633 assert!(
6634 at_ms.is_some(),
6635 "the control response lost the line's capture time"
6636 );
6637 }
6638
6639 #[tokio::test(flavor = "current_thread")]
6644 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
6645 let dir = TestTempDir::new("terminals-off-worker");
6646 let journal_path = dir.join("terminals.jsonl");
6647 let registry = Arc::new(Registry::default());
6648 let supervisor_handle = SupervisorHandle::new();
6649 let supervisor =
6650 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6651 .with_handle(supervisor_handle.clone())
6652 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
6653 let module = supervisor
6654 .spawn(ModuleSpec {
6655 module_id: "terminal-off-worker".to_string(),
6656 program: fake_aft_stub_path(),
6657 args: Vec::new(),
6658 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6659 reserved: false,
6660 reserved_prefixes: Vec::new(),
6661 protocol: ModuleProtocol::Subc,
6662 overlap: Default::default(),
6663 })
6664 .unwrap();
6665 let deadline = Instant::now() + Duration::from_secs(5);
6666 while module.terminal_history().entries.len() != 2 {
6667 assert!(Instant::now() < deadline, "module did not record two exits");
6668 sleep(Duration::from_millis(10)).await;
6669 }
6670
6671 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
6672 let handler =
6673 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
6674 let frame = Frame::build(
6675 FrameType::Request,
6676 control_flags(),
6677 0,
6678 0,
6679 1,
6680 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
6681 module_id: "terminal-off-worker".to_string(),
6682 })
6683 .unwrap(),
6684 )
6685 .unwrap();
6686 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6687 let spawned_at = std::time::Instant::now();
6688 let read = tokio::spawn({
6689 let handler = Arc::clone(&handler);
6690 async move { handler.handle_control_frame(&ctx, frame).await }
6691 });
6692 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
6695 .await
6696 .unwrap()
6697 .expect("the history read reached its pause");
6698 let elapsed = spawned_at.elapsed();
6699 assert!(
6700 elapsed < Duration::from_secs(2) && !read.is_finished(),
6701 "this task could not run while the history read was paused \
6702 (resumed after {elapsed:?}, read finished: {})",
6703 read.is_finished()
6704 );
6705
6706 drop(release);
6707 let responses = read.await.unwrap().unwrap();
6708 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6709 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
6710 panic!("expected supervisor.terminals response");
6711 };
6712 assert_eq!(terminals.entries.len(), 2);
6713 assert_eq!(terminals.journal_skipped_lines, 0);
6714 assert_eq!(terminals.journal_read_errors, 0);
6715 }
6716
6717 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6718 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
6719 let registry = Arc::new(Registry::default());
6720 let supervisor_handle = SupervisorHandle::new();
6721 let supervisor =
6722 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
6723 .with_handle(supervisor_handle.clone());
6724 let module = supervisor
6725 .spawn(ModuleSpec {
6726 module_id: "terminal-golden".to_string(),
6727 program: fake_aft_stub_path(),
6728 args: Vec::new(),
6729 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
6730 reserved: false,
6731 reserved_prefixes: Vec::new(),
6732 protocol: ModuleProtocol::Subc,
6733 overlap: Default::default(),
6734 })
6735 .unwrap();
6736
6737 let deadline = Instant::now() + Duration::from_secs(5);
6738 while module.terminal_history().entries.len() != 2 {
6739 assert!(
6740 Instant::now() < deadline,
6741 "module did not retain two terminal exits: {:?}",
6742 module.terminal_history()
6743 );
6744 sleep(Duration::from_millis(10)).await;
6745 }
6746
6747 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
6748 let request = ClientControlRequest::SupervisorTerminals {
6749 module_id: "terminal-golden".to_string(),
6750 };
6751 let frame = Frame::build(
6752 FrameType::Request,
6753 control_flags(),
6754 0,
6755 0,
6756 1,
6757 serde_json::to_vec(&request).unwrap(),
6758 )
6759 .unwrap();
6760 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
6761 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
6762 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
6763 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
6764 panic!("expected supervisor.terminals response");
6765 };
6766 assert_eq!(terminals.entries.len(), 2);
6767 assert_eq!(terminals.dropped, 0);
6768
6769 let mut rendered = serde_json::to_value(response).unwrap();
6770 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
6773 for (index, entry) in rendered["entries"]
6774 .as_array_mut()
6775 .expect("terminal response entries array")
6776 .iter_mut()
6777 .enumerate()
6778 {
6779 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
6780 }
6781
6782 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
6783 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
6784 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
6785 if std::env::var_os("UPDATE_GOLDEN").is_some() {
6786 std::fs::write(&golden_path, &serialized).unwrap();
6787 }
6788 let expected: Value =
6789 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
6790 assert_eq!(rendered, expected);
6791 }
6792
6793 #[test]
6794 fn hello_registers_manifest_and_returns_ack() {
6795 let registry = Arc::new(Registry::default());
6796 let handler = ControlHandler::new(Arc::clone(®istry));
6797 let conn = ConnectionId::new(1);
6798
6799 let responses = handler
6800 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
6801 .unwrap();
6802
6803 assert_eq!(responses.len(), 1);
6804 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
6805 assert_eq!(responses[0].header.channel, 0);
6806 assert_eq!(responses[0].header.corr, 7);
6807 let ack = parse_ack(&responses[0]);
6808 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
6809 assert!(ack
6810 .subc_capabilities
6811 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
6812 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
6813 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
6814 assert!(ack
6815 .subc_ops
6816 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
6817 assert!(ack
6818 .subc_ops
6819 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
6820
6821 let registration = registry.get_module("aft").unwrap().unwrap();
6822 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
6823 assert_eq!(registration.state, ChannelState::Active);
6824 assert_eq!(registration.connection_id, conn);
6825 assert_eq!(registration.control_ops, module_baseline_control_ops());
6826 }
6827
6828 #[test]
6829 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
6830 let invalid_identifiers = [
6831 ("case_change", "credentials-Provider/v1"),
6832 ("leading_zero", "credentials-provider/v01"),
6833 ("trailing_hyphen", "credentials-provider-/v1"),
6834 ("consecutive_hyphens", "credentials--provider/v1"),
6835 ("uppercase", "Credentials-provider/v1"),
6836 ("missing_v", "credentials-provider/1"),
6837 ("whitespace", "credentials provider/v1"),
6838 ("zero_version", "credentials-provider/v0"),
6839 ("out_of_range_version", "credentials-provider/v4294967296"),
6840 (
6841 "overlength_name",
6842 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
6843 ),
6844 ];
6845 let mut cases = invalid_identifiers
6846 .into_iter()
6847 .map(|(name, identifier)| {
6848 (
6849 format!("identifier_{name}"),
6850 "capabilities.provides[0]".to_string(),
6851 identifier.to_string(),
6852 json!({ "provides": [identifier] }),
6853 None,
6854 )
6855 })
6856 .collect::<Vec<_>>();
6857 cases.extend([
6858 (
6859 "unknown_need".to_string(),
6860 "capabilities.requires[0].need".to_string(),
6861 "deferred".to_string(),
6862 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
6863 None,
6864 ),
6865 (
6866 "duplicate_provides".to_string(),
6867 "capabilities.provides[1]".to_string(),
6868 "credentials-provider/v1".to_string(),
6869 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
6870 None,
6871 ),
6872 (
6873 "duplicate_must_never_reach".to_string(),
6874 "capabilities.must_never_reach[1]".to_string(),
6875 "credentials-provider/v1".to_string(),
6876 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
6877 None,
6878 ),
6879 (
6880 "duplicate_requires_same_need".to_string(),
6881 "capabilities.requires[1]".to_string(),
6882 "credentials-provider/v1".to_string(),
6883 json!({ "requires": [
6884 { "capability": "credentials-provider/v1", "need": "required" },
6885 { "capability": "credentials-provider/v1", "need": "required" }
6886 ] }),
6887 None,
6888 ),
6889 (
6890 "duplicate_requires_conflicting_need".to_string(),
6891 "capabilities.requires[1]".to_string(),
6892 "credentials-provider/v1".to_string(),
6893 json!({ "requires": [
6894 { "capability": "credentials-provider/v1", "need": "required" },
6895 { "capability": "credentials-provider/v1", "need": "optional" }
6896 ] }),
6897 None,
6898 ),
6899 (
6900 "capabilities_root_pointer".to_string(),
6901 "runtime_computed[0]".to_string(),
6902 "/capabilities".to_string(),
6903 json!({}),
6904 Some(json!(["/capabilities"])),
6905 ),
6906 (
6907 "capabilities_descendant_pointer".to_string(),
6908 "runtime_computed[0]".to_string(),
6909 "/capabilities/provides".to_string(),
6910 json!({}),
6911 Some(json!(["/capabilities/provides"])),
6912 ),
6913 (
6914 "malformed_pointer_without_leading_slash".to_string(),
6915 "runtime_computed[0]".to_string(),
6916 "capabilities".to_string(),
6917 json!({}),
6918 Some(json!(["capabilities"])),
6919 ),
6920 (
6921 "malformed_pointer_escape".to_string(),
6922 "runtime_computed[0]".to_string(),
6923 "/roles/~2/tools".to_string(),
6924 json!({}),
6925 Some(json!(["/roles/~2/tools"])),
6926 ),
6927 (
6928 "unknown_capabilities_field".to_string(),
6929 "capabilities.future".to_string(),
6930 "<array>".to_string(),
6931 json!({ "future": [] }),
6932 None,
6933 ),
6934 ]);
6935
6936 for (index, (name, field, value, capabilities, runtime_computed)) in
6937 cases.into_iter().enumerate()
6938 {
6939 let registry = Arc::new(Registry::default());
6940 let handler = ControlHandler::new(Arc::clone(®istry));
6941 let response = handler
6942 .handle_control(
6943 ConnectionId::new((index + 1) as u64),
6944 capability_grammar_hello_frame(
6945 capabilities,
6946 runtime_computed,
6947 index as u64 + 1,
6948 ),
6949 )
6950 .expect("invalid HELLO returns a refusal");
6951
6952 assert_eq!(response.len(), 1, "{name} must emit one refusal");
6953 let error = parse_error(&response[0]);
6954 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
6955 let message = error["message"]
6956 .as_str()
6957 .expect("error message is a string");
6958 assert!(
6959 message.contains(&field),
6960 "{name}: field missing from {message}"
6961 );
6962 assert!(
6963 message.contains(&value),
6964 "{name}: value missing from {message}"
6965 );
6966 assert_eq!(
6967 registry
6968 .active_registration_count()
6969 .expect("registry reads"),
6970 0,
6971 "{name}: refused HELLO must not create a catalog entry"
6972 );
6973 }
6974 }
6975
6976 #[test]
6977 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
6978 let registry = Arc::new(Registry::default());
6979 let handler = ControlHandler::new(Arc::clone(®istry));
6980 let capabilities = json!({
6981 "provides": ["credentials-provider/v1"],
6982 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
6983 "must_never_reach": ["federation-transport/v1"]
6984 });
6985 let response = handler
6986 .handle_control(
6987 ConnectionId::new(99),
6988 capability_grammar_hello_frame(
6989 capabilities.clone(),
6990 Some(json!(["/roles/0/tools"])),
6991 99,
6992 ),
6993 )
6994 .expect("valid HELLO registers");
6995 assert_eq!(response[0].header.ty, FrameType::HelloAck);
6996
6997 let request = Frame::build(
6998 FrameType::Request,
6999 control_flags(),
7000 0,
7001 0,
7002 100,
7003 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7004 .expect("catalog request serializes"),
7005 )
7006 .expect("catalog request frame builds");
7007 let response = handler
7008 .handle_catalog_list(request, None)
7009 .expect("catalog list succeeds");
7010 let ClientControlResponse::CatalogList { modules, .. } =
7011 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7012 else {
7013 panic!("catalog request must return catalog.list");
7014 };
7015 assert_eq!(modules.len(), 1);
7016 assert_eq!(
7017 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7018 capabilities
7019 );
7020 }
7021
7022 #[test]
7023 fn catalog_list_mirrors_management_operation_description() {
7024 let registry = Arc::new(Registry::default());
7025 let handler = ControlHandler::new(Arc::clone(®istry));
7026 let description = "List managed records and return their identifiers and metadata.";
7027 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7028 manifest.provides = vec![ProviderRole::ManagementSurface {
7029 operations: vec![ManagementOperation {
7030 name: "records.list".to_string(),
7031 kind: ManagementOperationKind::Query,
7032 description: Some(description.to_string()),
7033 }],
7034 config_schema: json!({"type": "object"}),
7035 observability: vec![ObservabilitySurface {
7036 name: "records.stats".to_string(),
7037 kind: ObservabilityKind::Snapshot,
7038 }],
7039 identity_scope: vec![IdentityScope::Project],
7040 concurrency: Concurrency::ModuleManaged,
7041 }];
7042 registry
7043 .register_with_control_ops(
7044 manifest,
7045 PROTOCOL_VERSION,
7046 ConnectionId::new(99),
7047 Vec::new(),
7048 )
7049 .expect("described management manifest registers");
7050
7051 let request = Frame::build(
7052 FrameType::Request,
7053 control_flags(),
7054 0,
7055 0,
7056 100,
7057 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7058 .expect("catalog request serializes"),
7059 )
7060 .expect("catalog request frame builds");
7061 let response = handler
7062 .handle_catalog_list(request, None)
7063 .expect("catalog list succeeds");
7064 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7065 assert_eq!(
7066 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7067 "catalog.list must preserve the declared operation description verbatim"
7068 );
7069 }
7070
7071 #[test]
7072 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7073 let registry = Arc::new(Registry::default());
7074 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7075 [("vault".to_string(), true), ("squatter".to_string(), true)],
7076 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7077 );
7078 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7079 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7080 provides: vec!["credentials-provider/v1".to_string()],
7081 requires: Vec::new(),
7082 must_never_reach: Vec::new(),
7083 });
7084 let frame = Frame::build(
7085 FrameType::Hello,
7086 control_flags(),
7087 0,
7088 0,
7089 77,
7090 serde_json::to_vec(&ModuleHelloBody {
7091 manifest: squatter,
7092 protocol_ver: PROTOCOL_VERSION,
7093 control_ops: None,
7094 launch_nonce: None,
7095 })
7096 .expect("HELLO serializes"),
7097 )
7098 .expect("HELLO frame builds");
7099 let response = handler
7100 .handle_control(ConnectionId::new(77), frame)
7101 .expect("reserved claim receives a typed refusal");
7102 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7103 assert_eq!(
7104 registry
7105 .active_registration_count()
7106 .expect("registry reads"),
7107 0,
7108 "a reserved capability refusal must not leave a catalog entry"
7109 );
7110 }
7111
7112 #[test]
7113 fn server_describe_surfaces_required_capability_verdict_fields() {
7114 let registry = Arc::new(Registry::default());
7115 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7116 [
7117 ("consumer".to_string(), true),
7118 ("provider".to_string(), false),
7119 ],
7120 BTreeMap::new(),
7121 );
7122 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
7123 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7124 provides: Vec::new(),
7125 requires: vec![subc_protocol::manifest::CapabilityRequirement {
7126 capability: "credentials-provider/v1".to_string(),
7127 need: subc_protocol::manifest::CapabilityNeed::Required,
7128 }],
7129 must_never_reach: Vec::new(),
7130 });
7131 let hello = Frame::build(
7132 FrameType::Hello,
7133 control_flags(),
7134 0,
7135 0,
7136 78,
7137 serde_json::to_vec(&ModuleHelloBody {
7138 manifest: consumer,
7139 protocol_ver: PROTOCOL_VERSION,
7140 control_ops: None,
7141 launch_nonce: None,
7142 })
7143 .expect("HELLO serializes"),
7144 )
7145 .expect("HELLO frame builds");
7146 handler
7147 .handle_control(ConnectionId::new(78), hello)
7148 .expect("consumer registers");
7149 let describe = Frame::build(
7150 FrameType::Request,
7151 control_flags(),
7152 0,
7153 0,
7154 79,
7155 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
7156 .expect("request serializes"),
7157 )
7158 .expect("describe frame builds");
7159 let response = handler
7160 .handle_server_describe(describe)
7161 .expect("server.describe succeeds");
7162 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
7163 let requirement = &rendered["capability_requirements"][0];
7164 assert_eq!(requirement["consumer"], "consumer");
7165 assert_eq!(requirement["verdict"], "never_provided");
7166 assert_eq!(requirement["episode_seq"], 1);
7167 assert_eq!(requirement["config_satisfiable"], false);
7168 assert_eq!(requirement["runtime_available"], false);
7169 assert!(requirement["detail"]
7170 .as_str()
7171 .expect("detail string")
7172 .contains("credentials-provider/v1"));
7173 }
7174
7175 #[test]
7176 fn catalog_list_omits_capabilities_for_legacy_manifest() {
7177 let registry = Arc::new(Registry::default());
7178 let handler = ControlHandler::new(Arc::clone(®istry));
7179 let hello = handler
7180 .handle_control(
7181 ConnectionId::new(101),
7182 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
7183 )
7184 .expect("legacy HELLO registers");
7185 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
7186
7187 let request = Frame::build(
7188 FrameType::Request,
7189 control_flags(),
7190 0,
7191 0,
7192 102,
7193 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7194 .expect("catalog request serializes"),
7195 )
7196 .expect("catalog request frame builds");
7197 let response = handler
7198 .handle_catalog_list(request, None)
7199 .expect("catalog list succeeds");
7200 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7201 assert!(
7202 body["modules"][0].get("capabilities").is_none(),
7203 "legacy manifest must retain an absent capabilities field on catalog.list"
7204 );
7205 }
7206
7207 #[test]
7208 fn hello_ack_omits_storage_when_no_storage_config() {
7209 let registry = Arc::new(Registry::default());
7210 let handler = ControlHandler::new(Arc::clone(®istry));
7211 let responses = handler
7212 .handle_control(
7213 ConnectionId::new(1),
7214 hello_frame("aft", PROTOCOL_VERSION, 7),
7215 )
7216 .unwrap();
7217 let ack = parse_ack(&responses[0]);
7218 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
7219 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
7220 }
7221
7222 #[tokio::test]
7223 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
7224 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
7225 let registry = Arc::new(Registry::default());
7226 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
7227 let responses = handler
7228 .handle_control(
7229 ConnectionId::new(1),
7230 hello_frame("aft", PROTOCOL_VERSION, 7),
7231 )
7232 .unwrap();
7233 let ack = parse_ack(&responses[0]);
7234 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
7235
7236 let described = handler
7237 .handle_control_frame(
7238 &route_ctx(ConnectionId::new(2)).0,
7239 Frame::build(
7240 FrameType::Request,
7241 control_flags(),
7242 0,
7243 0,
7244 9,
7245 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
7246 )
7247 .unwrap(),
7248 )
7249 .await
7250 .unwrap();
7251 let ClientControlResponse::ServerDescribe { machine_id, .. } =
7252 serde_json::from_slice(&described[0].body).unwrap()
7253 else {
7254 panic!("server.describe answered with another shape");
7255 };
7256 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
7257 }
7258
7259 #[test]
7260 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
7261 let registry = Arc::new(Registry::default());
7264 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
7265 crate::daemon_config::StorageConfig::Sqlite {
7266 data_home: std::path::PathBuf::from("/data"),
7267 },
7268 ));
7269
7270 let responses = handler
7271 .handle_control(
7272 ConnectionId::new(1),
7273 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
7274 )
7275 .unwrap();
7276 let ack = parse_ack(&responses[0]);
7277 assert_eq!(
7278 ack.storage,
7279 Some(serde_json::json!({
7280 "module_id": "alfonso-routing",
7281 "storage_namespace": "default",
7282 "isolation": { "kind": "module" },
7283 "backend": {
7284 "backend": "sqlite",
7285 "path": "/data/cortexkit/alfonso-routing/store.db"
7286 }
7287 })),
7288 "the delivered descriptor is the module's own sqlite store path"
7289 );
7290 }
7291
7292 #[test]
7293 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
7294 let registry = Arc::new(Registry::default());
7295 let handler = ControlHandler::new(Arc::clone(®istry));
7296 let conn = ConnectionId::new(1);
7297 let responses = handler
7298 .handle_control(
7299 conn,
7300 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7301 )
7302 .unwrap();
7303 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7304 let registration = registry.get_module("aft").unwrap().unwrap();
7305 assert_eq!(registration.control_ops, module_baseline_control_ops());
7306
7307 let frame =
7308 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
7309 assert!(handler
7310 .guard_module_control_op(&frame, "aft", "route.bind")
7311 .unwrap()
7312 .is_none());
7313 let error = handler
7314 .guard_module_control_op(&frame, "aft", "test.synthetic")
7315 .unwrap()
7316 .expect("synthetic ungranted op should be rejected");
7317 assert_eq!(error.header.ty, FrameType::Error);
7318 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
7319 }
7320
7321 #[test]
7322 fn hello_control_ops_some_adds_optional_grants() {
7323 let registry = Arc::new(Registry::default());
7324 let handler = ControlHandler::new(Arc::clone(®istry));
7325 handler
7326 .handle_control(
7327 ConnectionId::new(1),
7328 hello_frame_with_control_ops(
7329 "aft",
7330 PROTOCOL_VERSION,
7331 7,
7332 Some(vec![
7333 "future.synthetic".to_string(),
7334 "route.bind".to_string(),
7335 ]),
7336 ),
7337 )
7338 .unwrap();
7339 let registration = registry.get_module("aft").unwrap().unwrap();
7340 assert_eq!(
7341 registration.control_ops,
7342 vec![
7343 "route.bind".to_string(),
7344 "route.status".to_string(),
7345 "future.synthetic".to_string(),
7346 ]
7347 );
7348 let frame =
7349 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
7350 assert!(handler
7351 .guard_module_control_op(&frame, "aft", "future.synthetic")
7352 .unwrap()
7353 .is_none());
7354 }
7355
7356 #[tokio::test]
7357 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
7358 let registry = Arc::new(Registry::default());
7359 let forwarding = Arc::new(ForwardingTable::default());
7360 let handler =
7361 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7362 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
7363 hello_via_sink(
7364 &handler,
7365 &module_ctx,
7366 &mut module_rx,
7367 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
7368 )
7369 .await;
7370
7371 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
7372 let responses = handler
7373 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
7374 .await
7375 .unwrap();
7376 assert_eq!(responses.len(), 1);
7377 assert_eq!(responses[0].header.ty, FrameType::Error);
7378 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
7379 assert!(module_rx.try_recv().is_err());
7380 }
7381
7382 #[tokio::test]
7383 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
7384 let registry = Arc::new(Registry::default());
7385 let forwarding = Arc::new(ForwardingTable::default());
7386 let handler =
7387 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7388 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
7389 hello_via_sink(
7390 &handler,
7391 &module_ctx,
7392 &mut module_rx,
7393 hello_frame_with_control_ops(
7394 "aft",
7395 PROTOCOL_VERSION,
7396 7,
7397 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
7398 ),
7399 )
7400 .await;
7401
7402 let project_root = unique_project_root("demux");
7403 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
7404 let route_handler = handler.clone();
7405 let route_task = tokio::spawn(async move {
7406 route_handler
7407 .handle_control_frame(
7408 &route_client_ctx,
7409 route_open_frame(100, "aft", project_root),
7410 )
7411 .await
7412 .unwrap()
7413 });
7414 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7415 .await
7416 .unwrap()
7417 .unwrap();
7418 assert!(matches!(
7419 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
7420 ModuleControlRequest::RouteBind { .. }
7421 ));
7422
7423 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
7424 let health_handler = handler.clone();
7425 let health_task = tokio::spawn(async move {
7426 health_handler
7427 .handle_control_frame(
7428 &health_client_ctx,
7429 supervisor_health_probe_frame(101, "aft"),
7430 )
7431 .await
7432 .unwrap()
7433 });
7434 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7435 .await
7436 .unwrap()
7437 .unwrap();
7438 assert_eq!(
7439 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
7440 ModuleControlRequest::HealthCheck {}
7441 );
7442
7443 handler
7444 .handle_control_frame(
7445 &module_ctx,
7446 health_response(health_frame.header.corr, HealthStatus::Degraded),
7447 )
7448 .await
7449 .unwrap();
7450 let health_response = health_task.await.unwrap();
7451 assert_eq!(health_response.len(), 1);
7452 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
7453 ClientControlResponse::SupervisorHealthProbe {
7454 module_id,
7455 status,
7456 detail,
7457 metrics,
7458 } => {
7459 assert_eq!(module_id, "aft");
7460 assert_eq!(status, HealthStatus::Degraded);
7461 assert_eq!(detail.as_deref(), Some("warming"));
7462 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
7463 }
7464 other => panic!("unexpected health response: {other:?}"),
7465 }
7466
7467 handler
7468 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
7469 .await
7470 .unwrap();
7471 let route_response = route_task.await.unwrap();
7472 assert!(route_response.is_empty());
7473 let published = route_client_rx.recv().await.unwrap();
7474 assert!(matches!(
7475 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
7476 ClientControlResponse::RouteOpen { .. }
7477 ));
7478 }
7479
7480 async fn relay_route_open(
7485 handler: &ControlHandler,
7486 client_connection: ConnectionId,
7487 client_egress: &FrameSink,
7488 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7489 corr: u64,
7490 module_id: &str,
7491 project_root_label: &str,
7492 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
7493 let ctx = RouteCtx {
7494 connection_id: client_connection,
7495 egress: client_egress.clone(),
7496 };
7497 let handler = handler.clone();
7498 let project_root = unique_project_root(project_root_label);
7499 let module_id = module_id.to_string();
7500 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
7501 let task = tokio::spawn(async move {
7502 let _guard = tracing::dispatcher::set_default(&dispatch);
7503 handler
7504 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
7505 .await
7506 .unwrap()
7507 });
7508 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
7509 .await
7510 .expect("module receives the relayed route.bind")
7511 .expect("module egress is open");
7512 (task, bind.frame)
7513 }
7514
7515 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
7516 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
7517 ModuleControlRequest::RouteBind {
7518 route_channel,
7519 epoch,
7520 ..
7521 } => (route_channel, epoch),
7522 other => panic!("expected a route.bind request, got {other:?}"),
7523 }
7524 }
7525
7526 fn published_route(frame: &Frame) -> (u16, u32) {
7527 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
7528 ClientControlResponse::RouteOpen {
7529 route_channel,
7530 route_epoch,
7531 } => (route_channel, route_epoch),
7532 other => panic!("expected a route.open response, got {other:?}"),
7533 }
7534 }
7535
7536 #[tokio::test]
7556 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
7557 let registry = Arc::new(Registry::default());
7558 let forwarding = Arc::new(ForwardingTable::default());
7559 let handler =
7560 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
7561
7562 let module_connection = ConnectionId::new(30);
7563 let (module_ctx, mut module_rx) = route_ctx(module_connection);
7564 hello_via_sink(
7565 &handler,
7566 &module_ctx,
7567 &mut module_rx,
7568 hello_frame("aft", PROTOCOL_VERSION, 7),
7569 )
7570 .await;
7571
7572 let dying_client = ConnectionId::new(31);
7573 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
7574
7575 let (first_task, first_bind) = relay_route_open(
7578 &handler,
7579 dying_client,
7580 &dying_ctx.egress,
7581 &mut module_rx,
7582 100,
7583 "aft",
7584 "closing-first",
7585 )
7586 .await;
7587 handler
7588 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
7589 .await
7590 .unwrap();
7591 assert!(first_task.await.unwrap().is_empty());
7592 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
7593
7594 let (second_task, second_bind) = relay_route_open(
7596 &handler,
7597 dying_client,
7598 &dying_ctx.egress,
7599 &mut module_rx,
7600 101,
7601 "aft",
7602 "closing-second",
7603 )
7604 .await;
7605 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
7606
7607 assert!(forwarding
7610 .escalate_client_delivery_failure(
7611 dying_client,
7612 first_channel,
7613 first_epoch,
7614 CloseReason::new(
7615 "module_to_client_delivery_failed",
7616 "client egress refused a module frame",
7617 ),
7618 crate::forwarding::UndeliveredFrame {
7619 module_id: None,
7620 sink: &dying_ctx.egress,
7621 },
7622 )
7623 .unwrap());
7624 assert!(!dying_ctx.egress.is_closed());
7625
7626 let ack = handler
7628 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
7629 .await;
7630 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
7631 if module_loop_error.is_some() {
7632 handler.cleanup_connection(module_connection).unwrap();
7636 }
7637 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
7641 .await
7642 .ok()
7643 .flatten();
7644
7645 assert!(
7647 registry
7648 .get_module_by_connection(module_connection)
7649 .unwrap()
7650 .is_some(),
7651 "one client's closing connection ended the shared module connection: \
7652 {module_loop_error:?}"
7653 );
7654 let cotenant = ConnectionId::new(32);
7656 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
7657 let (cotenant_task, cotenant_bind) = relay_route_open(
7658 &handler,
7659 cotenant,
7660 &cotenant_ctx.egress,
7661 &mut module_rx,
7662 102,
7663 "aft",
7664 "closing-cotenant",
7665 )
7666 .await;
7667 handler
7668 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
7669 .await
7670 .unwrap();
7671 assert!(cotenant_task.await.unwrap().is_empty());
7672 let (cotenant_channel, cotenant_epoch) =
7673 published_route(&cotenant_rx.recv().await.unwrap());
7674 assert!(matches!(
7675 forwarding
7676 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
7677 .unwrap(),
7678 DataRoute::Client(DataRouteState::Bound(_))
7679 ));
7680
7681 let goodbye = post_ack_module_frame
7684 .expect("module receives a GOODBYE for the abandoned route channel");
7685 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
7686 assert_eq!(goodbye.header.channel, abandoned_channel);
7687 assert_eq!(goodbye.header.epoch, abandoned_epoch);
7688
7689 assert!(dying_rx.try_recv().is_err());
7693 let second_response = second_task.await.unwrap();
7694 assert_eq!(second_response.len(), 1);
7695 assert_eq!(
7696 parse_error(&second_response[0])["code"],
7697 "target_unavailable"
7698 );
7699 }
7700
7701 #[test]
7708 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
7709 let handler = ControlHandler::default();
7710 let module_connection = ConnectionId::new(30);
7711 let client_connection = ConnectionId::new(31);
7712
7713 handler
7714 .refuse_to_end_module_connection_for_a_client(
7715 module_connection,
7716 77,
7717 ForwardingError::ConnectionClosing {
7718 connection_id: client_connection,
7719 },
7720 )
7721 .expect("a closing client must never end the module connection");
7722
7723 assert!(matches!(
7724 handler.refuse_to_end_module_connection_for_a_client(
7725 module_connection,
7726 78,
7727 ForwardingError::ConnectionClosing {
7728 connection_id: module_connection,
7729 },
7730 ),
7731 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
7732 connection_id
7733 })) if connection_id == module_connection
7734 ));
7735 assert!(matches!(
7736 handler.refuse_to_end_module_connection_for_a_client(
7737 module_connection,
7738 79,
7739 ForwardingError::Poisoned,
7740 ),
7741 Err(RouterError::Forwarding(ForwardingError::Poisoned))
7742 ));
7743 assert!(matches!(
7744 handler.refuse_to_end_module_connection_for_a_client(
7745 module_connection,
7746 80,
7747 ForwardingError::StaleModuleEndpoint,
7748 ),
7749 Err(RouterError::Forwarding(
7750 ForwardingError::StaleModuleEndpoint
7751 ))
7752 ));
7753 }
7754
7755 #[tokio::test]
7768 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
7769 let registry = Arc::new(Registry::default());
7770 let forwarding = Arc::new(ForwardingTable::default());
7771 let supervisor = SupervisorHandle::new();
7772 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7773 let handler =
7774 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7775 .with_supervisor(supervisor);
7776
7777 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
7778 hello_via_sink(
7779 &handler,
7780 &target_ctx,
7781 &mut target_rx,
7782 hello_frame("target", PROTOCOL_VERSION, 1),
7783 )
7784 .await;
7785
7786 let wrong_nonce = handler
7790 .handle_control_frame(
7791 &route_ctx(ConnectionId::new(91)).0,
7792 route_open_frame_with_admission_facts(
7793 20,
7794 "target",
7795 unique_project_root("admission-facts"),
7796 Some(subc_control::ConsumerIdentity {
7797 module_id: "fed".to_string(),
7798 launch_nonce: "not-the-real-nonce".to_string(),
7799 }),
7800 None,
7801 ),
7802 )
7803 .await
7804 .unwrap();
7805 assert_eq!(
7806 parse_error(&wrong_nonce[0])["code"],
7807 "bad_consumer_identity",
7808 "a mismatched launch nonce must be refused, not stamped Reserved"
7809 );
7810
7811 let never_spawned = handler
7815 .handle_control_frame(
7816 &route_ctx(ConnectionId::new(92)).0,
7817 route_open_frame_with_admission_facts(
7818 21,
7819 "target",
7820 unique_project_root("admission-facts"),
7821 Some(subc_control::ConsumerIdentity {
7822 module_id: "never-spawned".to_string(),
7823 launch_nonce: "any-nonce".to_string(),
7824 }),
7825 None,
7826 ),
7827 )
7828 .await
7829 .unwrap();
7830 assert_eq!(
7831 parse_error(&never_spawned[0])["code"],
7832 "bad_consumer_identity",
7833 "an unspawned module_id must be refused rather than accepted for lack of a record"
7834 );
7835 }
7836
7837 #[tokio::test]
7860 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
7861 let registry = Arc::new(Registry::default());
7862 let forwarding = Arc::new(ForwardingTable::default());
7863 let supervisor = SupervisorHandle::new();
7864 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7865 let handler =
7866 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7867 .with_supervisor(supervisor);
7868
7869 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
7870 hello_via_sink(
7871 &handler,
7872 &target_ctx,
7873 &mut target_rx,
7874 hello_frame("target", PROTOCOL_VERSION, 1),
7875 )
7876 .await;
7877
7878 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
7879 let route_handler = handler.clone();
7880 let route_task = tokio::spawn(async move {
7881 route_handler
7882 .handle_control_frame(
7883 &client_ctx,
7884 route_open_frame_with_admission_facts(
7885 30,
7886 "target",
7887 unique_project_root("admission-facts"),
7888 Some(subc_control::ConsumerIdentity {
7889 module_id: "fed".to_string(),
7890 launch_nonce: "fed-nonce".to_string(),
7891 }),
7892 None,
7893 ),
7894 )
7895 .await
7896 .unwrap()
7897 });
7898
7899 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7906 .await
7907 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
7908 .expect("module control channel closed before route.bind");
7909 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
7910 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
7911 panic!("expected route.bind")
7912 };
7913 assert_eq!(
7914 principal,
7915 Some(Principal::Reserved {
7916 module_id: "fed".to_string()
7917 }),
7918 "a correctly attested consumer must be stamped Reserved for its own id"
7919 );
7920
7921 handler
7922 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
7923 .await
7924 .unwrap();
7925 assert!(route_task.await.unwrap().is_empty());
7926 assert!(
7927 matches!(
7928 serde_json::from_slice::<ClientControlResponse>(
7929 &client_rx.recv().await.unwrap().body
7930 )
7931 .unwrap(),
7932 ClientControlResponse::RouteOpen { .. }
7933 ),
7934 "the route must actually open, not merely avoid an error"
7935 );
7936 }
7937
7938 #[tokio::test(start_paused = true)]
7939 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
7940 let registry = Arc::new(Registry::default());
7941 let forwarding = Arc::new(ForwardingTable::default());
7942 let supervisor = SupervisorHandle::new();
7943 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
7944 let handler =
7945 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
7946 .with_supervisor(supervisor);
7947
7948 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
7949 hello_via_sink(
7950 &handler,
7951 &target_ctx,
7952 &mut target_rx,
7953 hello_frame("target", PROTOCOL_VERSION, 1),
7954 )
7955 .await;
7956
7957 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
7958 let direct_handler = handler.clone();
7959 let direct_open = tokio::spawn(async move {
7960 direct_handler
7961 .handle_control_frame(
7962 &direct_ctx,
7963 route_open_frame(2, "target", unique_project_root("route-census-direct")),
7964 )
7965 .await
7966 .unwrap()
7967 });
7968 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
7969 .await
7970 .expect("no direct route.bind within 5s")
7971 .expect("target control channel closed before direct route.bind");
7972 handler
7973 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
7974 .await
7975 .unwrap();
7976 assert!(direct_open.await.unwrap().is_empty());
7977 let _ = direct_rx.recv().await.unwrap();
7978
7979 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
7980 let reserved_handler = handler.clone();
7981 let reserved_open = tokio::spawn(async move {
7982 reserved_handler
7983 .handle_control_frame(
7984 &reserved_ctx,
7985 route_open_frame_with_admission_facts(
7986 3,
7987 "target",
7988 unique_project_root("admission-facts"),
7989 Some(ConsumerIdentity {
7990 module_id: "fed".to_string(),
7991 launch_nonce: "fed-nonce".to_string(),
7992 }),
7993 None,
7994 ),
7995 )
7996 .await
7997 .unwrap()
7998 });
7999 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8000 .await
8001 .expect("no reserved route.bind within 5s")
8002 .expect("target control channel closed before reserved route.bind");
8003 handler
8004 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
8005 .await
8006 .unwrap();
8007 assert!(reserved_open.await.unwrap().is_empty());
8008 let _ = reserved_rx.recv().await.unwrap();
8009
8010 forwarding
8011 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
8012 .unwrap();
8013 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
8014 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
8015 module_id: Some("target".to_string()),
8016 })
8017 .unwrap();
8018 let census_frame =
8019 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
8020 let response = handler
8021 .handle_control_frame(&census_ctx, census_frame)
8022 .await
8023 .unwrap()
8024 .pop()
8025 .unwrap();
8026 let actual: Value = serde_json::from_slice(&response.body).unwrap();
8027 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
8028 assert!(matches!(
8029 decoded,
8030 ClientControlResponse::SupervisorRoutes { .. }
8031 ));
8032 let routes = actual["modules"][0]["routes"].as_array().unwrap();
8033 assert_eq!(routes.len(), 2);
8034 assert!(routes.iter().all(|route| route["draining"] == true));
8035 assert!(
8038 routes.iter().all(|route| route["drain_reason"] == "reload"),
8039 "draining routes must name the drain's reason: {routes:?}"
8040 );
8041 assert!(routes.iter().any(|route| {
8042 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
8043 }));
8044 assert!(routes.iter().any(|route| {
8045 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
8046 }));
8047
8048 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8049 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
8050 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8051 std::fs::write(
8052 &golden_path,
8053 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
8054 )
8055 .unwrap();
8056 }
8057 let expected: Value =
8058 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
8059 assert_eq!(actual, expected);
8060 }
8061
8062 async fn query_live_roots(
8063 handler: &ControlHandler,
8064 module_ctx: &RouteCtx,
8065 ) -> ModuleControlResponseToModule {
8066 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
8067 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
8068 let response = handler
8069 .handle_control_frame(module_ctx, frame)
8070 .await
8071 .unwrap()
8072 .pop()
8073 .unwrap();
8074 serde_json::from_slice(&response.body).unwrap()
8075 }
8076
8077 #[tokio::test(start_paused = true)]
8078 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
8079 let registry = Arc::new(Registry::default());
8080 let forwarding = Arc::new(ForwardingTable::default());
8081 let handler = ControlHandler::with_forwarding(registry, forwarding);
8082 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
8083 hello_via_sink(
8084 &handler,
8085 &target_ctx,
8086 &mut target_rx,
8087 hello_frame("target", PROTOCOL_VERSION, 1),
8088 )
8089 .await;
8090 let root = unique_project_root("live-roots-known");
8091 let path = ProjectRootId::from_path_allowing_missing(root.path())
8092 .unwrap()
8093 .as_path()
8094 .to_path_buf();
8095 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
8096 let open_handler = handler.clone();
8097 let opened = tokio::spawn(async move {
8098 open_handler
8099 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
8100 .await
8101 .unwrap()
8102 });
8103 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8104 .await
8105 .unwrap()
8106 .unwrap();
8107 handler
8108 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
8109 .await
8110 .unwrap();
8111 assert!(opened.await.unwrap().is_empty());
8112 let _ = client_rx.recv().await.unwrap();
8113
8114 let root = unique_project_root("live-roots-pending");
8115 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
8116 .unwrap()
8117 .as_path()
8118 .to_path_buf();
8119 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
8120 let open_handler = handler.clone();
8121 let pending = tokio::spawn(async move {
8122 open_handler
8123 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
8124 .await
8125 .unwrap()
8126 });
8127 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8128 .await
8129 .unwrap()
8130 .unwrap();
8131 let actual = query_live_roots(&handler, &target_ctx).await;
8132 let ModuleControlResponseToModule::LiveRoots {
8133 roots,
8134 unknown_root_bindings,
8135 total_bindings,
8136 } = actual
8137 else {
8138 panic!("expected live roots")
8139 };
8140 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
8141 assert_eq!(unknown_root_bindings, 0);
8142 assert_eq!(
8143 roots.len(),
8144 2,
8145 "root-known arm must retain each canonical root"
8146 );
8147 assert_eq!(
8148 total_bindings,
8149 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8150 );
8151 let counts = roots
8152 .iter()
8153 .map(|root| (root.project_root.clone(), root.bound, root.pending))
8154 .collect::<Vec<_>>();
8155 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
8156 expected.sort_by(|a, b| a.0.cmp(&b.0));
8157 assert_eq!(
8158 counts, expected,
8159 "roots must sort by path and count pending separately"
8160 );
8161 handler
8162 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
8163 .await
8164 .unwrap();
8165 assert!(pending.await.unwrap().is_empty());
8166 }
8167
8168 #[tokio::test(start_paused = true)]
8169 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
8170 let forwarding = Arc::new(ForwardingTable::default());
8171 let handler =
8172 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8173 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
8174 hello_via_sink(
8175 &handler,
8176 &target_ctx,
8177 &mut target_rx,
8178 hello_frame("target", PROTOCOL_VERSION, 1),
8179 )
8180 .await;
8181 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
8182 let pending = forwarding
8183 .begin_route_bind_relay_for_test(
8184 client_ctx.connection_id,
8185 client_ctx.egress.clone(),
8186 2,
8187 "target",
8188 )
8189 .unwrap();
8190 forwarding
8191 .complete_pending_relay(
8192 target_ctx.connection_id,
8193 pending.corr,
8194 RouteBindRelayOutcome::Accepted,
8195 )
8196 .unwrap();
8197 let actual = query_live_roots(&handler, &target_ctx).await;
8198 let ModuleControlResponseToModule::LiveRoots {
8199 roots,
8200 unknown_root_bindings,
8201 total_bindings,
8202 } = actual
8203 else {
8204 panic!("expected live roots")
8205 };
8206 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
8207 assert_eq!(
8208 unknown_root_bindings, 1,
8209 "unknown-root arm must not read as no bindings"
8210 );
8211 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
8212 assert_eq!(
8213 total_bindings,
8214 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8215 );
8216 }
8217
8218 #[tokio::test(start_paused = true)]
8226 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
8227 let forwarding = Arc::new(ForwardingTable::default());
8228 let handler =
8229 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8230 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
8231 let replies = handler
8232 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
8233 .await
8234 .unwrap();
8235 let queued_by_hello = module_rx.len();
8236
8237 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
8238 let open_handler = handler.clone();
8239 let open = tokio::spawn(async move {
8240 open_handler
8241 .handle_control_frame(
8242 &client_ctx,
8243 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
8244 )
8245 .await
8246 .unwrap()
8247 });
8248 let mut spins = 0;
8250 while module_rx.len() == queued_by_hello {
8251 spins += 1;
8252 assert!(spins < 10_000, "route.open never queued a route.bind");
8253 tokio::task::yield_now().await;
8254 }
8255
8256 for reply in replies {
8258 module_ctx.egress.send(reply).await.unwrap();
8259 }
8260
8261 let first = module_rx.recv().await.unwrap().frame;
8262 assert_eq!(
8263 first.header.ty,
8264 FrameType::HelloAck,
8265 "the first frame a registering module reads must be its HELLO_ACK"
8266 );
8267 assert_eq!(first.header.corr, 7);
8268 let second = module_rx.recv().await.unwrap().frame;
8269 assert_eq!(second.header.ty, FrameType::Request);
8270 assert!(
8271 matches!(
8272 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
8273 ModuleControlRequest::RouteBind { .. }
8274 ),
8275 "the route.bind follows the ack"
8276 );
8277 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
8278
8279 handler
8280 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
8281 .await
8282 .unwrap();
8283 assert!(open.await.unwrap().is_empty());
8284 let _ = client_rx.recv().await.unwrap();
8285 }
8286
8287 #[tokio::test(start_paused = true)]
8288 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
8289 let handler = ControlHandler::with_forwarding(
8290 Arc::new(Registry::default()),
8291 Arc::new(ForwardingTable::default()),
8292 );
8293 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
8294 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
8295 hello_via_sink(
8296 &handler,
8297 &first_ctx,
8298 &mut first_rx,
8299 hello_frame("first", PROTOCOL_VERSION, 1),
8300 )
8301 .await;
8302 hello_via_sink(
8303 &handler,
8304 &second_ctx,
8305 &mut second_rx,
8306 hello_frame("second", PROTOCOL_VERSION, 2),
8307 )
8308 .await;
8309 let root = unique_project_root("second-only");
8310 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
8311 let cloned = handler.clone();
8312 let open = tokio::spawn(async move {
8313 cloned
8314 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
8315 .await
8316 .unwrap()
8317 });
8318 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
8319 .await
8320 .unwrap()
8321 .unwrap();
8322 let first = query_live_roots(&handler, &first_ctx).await;
8323 let second = query_live_roots(&handler, &second_ctx).await;
8324 assert!(
8325 matches!(
8326 first,
8327 ModuleControlResponseToModule::LiveRoots {
8328 total_bindings: 0,
8329 ..
8330 }
8331 ),
8332 "cross-module scope must not expose another module's roots"
8333 );
8334 assert!(
8335 matches!(
8336 second,
8337 ModuleControlResponseToModule::LiveRoots {
8338 total_bindings: 1,
8339 ..
8340 }
8341 ),
8342 "second module must see its pending route"
8343 );
8344 handler
8345 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
8346 .await
8347 .unwrap();
8348 assert!(open.await.unwrap().is_empty());
8349 }
8350
8351 #[tokio::test(start_paused = true)]
8352 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
8353 let handler = ControlHandler::with_forwarding(
8354 Arc::new(Registry::default()),
8355 Arc::new(ForwardingTable::default()),
8356 );
8357 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
8358 hello_via_sink(
8359 &handler,
8360 &target_ctx,
8361 &mut target_rx,
8362 hello_frame("target", PROTOCOL_VERSION, 1),
8363 )
8364 .await;
8365 let actual = query_live_roots(&handler, &target_ctx).await;
8366 let ModuleControlResponseToModule::LiveRoots {
8367 roots,
8368 unknown_root_bindings,
8369 total_bindings,
8370 } = actual
8371 else {
8372 panic!("expected live roots")
8373 };
8374 assert!(roots.is_empty());
8375 assert_eq!(unknown_root_bindings, 0);
8376 assert_eq!(total_bindings, 0);
8377 assert_eq!(
8378 total_bindings,
8379 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8380 );
8381 }
8382
8383 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
8390 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8391 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
8392 let text = std::fs::read_to_string(&path)
8393 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
8394 let vectors: Vec<(String, Value)> = text
8395 .lines()
8396 .filter(|line| !line.trim().is_empty())
8397 .map(|line| {
8398 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
8399 let id = entry["corpus_id"]
8400 .as_str()
8401 .expect("every vector carries a corpus_id")
8402 .to_string();
8403 (id, entry["package"].clone())
8404 })
8405 .collect();
8406 assert_eq!(
8410 vectors.len(),
8411 3,
8412 "vendored fed corpus changed size; re-sync from subc-federation"
8413 );
8414
8415 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
8430 let richest = vectors
8431 .iter()
8432 .filter_map(|(_, package)| package.as_object())
8433 .map(|object| {
8434 object
8435 .keys()
8436 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
8437 .count()
8438 })
8439 .max()
8440 .unwrap_or(0);
8441 assert!(
8442 richest >= 2,
8443 "vendored corpus no longer carries a package with unmodelled fields, \
8444 so the relay test can no longer distinguish a verbatim relay from a lossy one"
8445 );
8446
8447 vectors
8448 }
8449
8450 #[tokio::test]
8458 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
8459 for (corpus_id, package) in fed_admission_facts_vectors() {
8460 let registry = Arc::new(Registry::default());
8461 let forwarding = Arc::new(ForwardingTable::default());
8462 let supervisor = SupervisorHandle::new();
8463 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8464 let handler =
8465 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8466 .with_supervisor(supervisor)
8467 .with_admission_facts_config(
8468 Some("fed".to_string()),
8469 Some(vec!["target".to_string()]),
8470 );
8471
8472 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8473 hello_via_sink(
8474 &handler,
8475 &target_ctx,
8476 &mut target_rx,
8477 hello_frame("target", PROTOCOL_VERSION, 1),
8478 )
8479 .await;
8480
8481 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
8482 let route_handler = handler.clone();
8483 let expected = package.clone();
8484 let route_task = tokio::spawn(async move {
8485 route_handler
8486 .handle_control_frame(
8487 &client_ctx,
8488 route_open_frame_with_admission_facts(
8489 20,
8490 "target",
8491 unique_project_root("admission-facts"),
8492 Some(subc_control::ConsumerIdentity {
8493 module_id: "fed".to_string(),
8494 launch_nonce: "fed-nonce".to_string(),
8495 }),
8496 Some(package),
8497 ),
8498 )
8499 .await
8500 .unwrap()
8501 });
8502
8503 let bind_frame = target_rx.recv().await.unwrap();
8504 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8505 let ModuleControlRequest::RouteBind {
8506 admission_facts, ..
8507 } = bind
8508 else {
8509 panic!("{corpus_id}: expected route.bind")
8510 };
8511 assert_eq!(
8512 admission_facts,
8513 Some(expected),
8514 "{corpus_id}: relay must not add, drop or reshape any field"
8515 );
8516
8517 handler
8518 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8519 .await
8520 .unwrap();
8521 route_task.await.unwrap();
8522 }
8523 }
8524
8525 #[tokio::test]
8526 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
8527 let registry = Arc::new(Registry::default());
8528 let forwarding = Arc::new(ForwardingTable::default());
8529 let supervisor = SupervisorHandle::new();
8530 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8531 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
8532 let handler =
8533 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8534 .with_supervisor(supervisor)
8535 .with_admission_facts_config(
8536 Some("fed".to_string()),
8537 Some(vec!["target".to_string()]),
8538 );
8539
8540 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
8541 hello_via_sink(
8542 &handler,
8543 &target_ctx,
8544 &mut target_rx,
8545 hello_frame("target", PROTOCOL_VERSION, 1),
8546 )
8547 .await;
8548 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
8549 hello_via_sink(
8550 &handler,
8551 &other_ctx,
8552 &mut other_rx,
8553 hello_frame("other", PROTOCOL_VERSION, 2),
8554 )
8555 .await;
8556
8557 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
8558 let expected_facts = facts.clone();
8559 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
8560 let route_handler = handler.clone();
8561 let route_task = tokio::spawn(async move {
8562 route_handler
8563 .handle_control_frame(
8564 &client_ctx,
8565 route_open_frame_with_admission_facts(
8566 10,
8567 "target",
8568 unique_project_root("admission-facts"),
8569 Some(subc_control::ConsumerIdentity {
8570 module_id: "fed".to_string(),
8571 launch_nonce: "fed-nonce".to_string(),
8572 }),
8573 Some(facts.clone()),
8574 ),
8575 )
8576 .await
8577 .unwrap()
8578 });
8579 let bind_frame = target_rx.recv().await.unwrap();
8580 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8581 let ModuleControlRequest::RouteBind {
8582 admission_facts, ..
8583 } = bind
8584 else {
8585 panic!("expected route.bind")
8586 };
8587 assert_eq!(admission_facts, Some(expected_facts));
8588 handler
8589 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8590 .await
8591 .unwrap();
8592 assert!(route_task.await.unwrap().is_empty());
8593 assert!(matches!(
8594 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
8595 .unwrap(),
8596 ClientControlResponse::RouteOpen { .. }
8597 ));
8598
8599 let direct = handler
8600 .handle_control_frame(
8601 &route_ctx(ConnectionId::new(73)).0,
8602 route_open_frame_with_admission_facts(
8603 11,
8604 "target",
8605 unique_project_root("admission-facts"),
8606 None,
8607 Some(json!({"x": 1})),
8608 ),
8609 )
8610 .await
8611 .unwrap();
8612 assert_eq!(
8613 parse_error(&direct[0])["code"],
8614 "admission_facts_not_permitted"
8615 );
8616
8617 let different_reserved = handler
8618 .handle_control_frame(
8619 &route_ctx(ConnectionId::new(77)).0,
8620 route_open_frame_with_admission_facts(
8621 15,
8622 "target",
8623 unique_project_root("admission-facts"),
8624 Some(subc_control::ConsumerIdentity {
8625 module_id: "other".to_string(),
8626 launch_nonce: "other-nonce".to_string(),
8627 }),
8628 Some(json!({"x": 1})),
8629 ),
8630 )
8631 .await
8632 .unwrap();
8633 assert_eq!(
8634 parse_error(&different_reserved[0])["code"],
8635 "admission_facts_not_permitted"
8636 );
8637
8638 let other_target = handler
8639 .handle_control_frame(
8640 &route_ctx(ConnectionId::new(74)).0,
8641 route_open_frame_with_admission_facts(
8642 12,
8643 "other",
8644 unique_project_root("admission-facts"),
8645 Some(subc_control::ConsumerIdentity {
8646 module_id: "fed".to_string(),
8647 launch_nonce: "fed-nonce".to_string(),
8648 }),
8649 Some(json!({"x": 1})),
8650 ),
8651 )
8652 .await
8653 .unwrap();
8654 assert_eq!(
8655 parse_error(&other_target[0])["code"],
8656 "admission_facts_target_not_allowed"
8657 );
8658
8659 let nonexistent = handler
8660 .handle_control_frame(
8661 &route_ctx(ConnectionId::new(75)).0,
8662 route_open_frame_with_admission_facts(
8663 13,
8664 "missing",
8665 unique_project_root("admission-facts"),
8666 None,
8667 Some(json!({"x": 1})),
8668 ),
8669 )
8670 .await
8671 .unwrap();
8672 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
8673
8674 let described = handler
8675 .handle_control_frame(
8676 &route_ctx(ConnectionId::new(76)).0,
8677 Frame::build(
8678 FrameType::Request,
8679 control_flags(),
8680 0,
8681 0,
8682 14,
8683 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8684 )
8685 .unwrap(),
8686 )
8687 .await
8688 .unwrap();
8689 let ClientControlResponse::ServerDescribe { capabilities, .. } =
8690 serde_json::from_slice(&described[0].body).unwrap()
8691 else {
8692 panic!("expected server.describe response")
8693 };
8694 assert!(capabilities
8695 .iter()
8696 .any(|cap| cap == "admission_facts_relay_v1"));
8697 }
8698
8699 #[tokio::test]
8700 async fn admission_facts_without_configured_carrier_are_rejected() {
8701 let registry = Arc::new(Registry::default());
8702 let forwarding = Arc::new(ForwardingTable::default());
8703 let handler = ControlHandler::with_forwarding(registry, forwarding);
8704 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
8705 hello_via_sink(
8706 &handler,
8707 &target_ctx,
8708 &mut target_rx,
8709 hello_frame("target", PROTOCOL_VERSION, 1),
8710 )
8711 .await;
8712
8713 let responses = handler
8714 .handle_control_frame(
8715 &route_ctx(ConnectionId::new(79)).0,
8716 route_open_frame_with_admission_facts(
8717 16,
8718 "target",
8719 unique_project_root("admission-facts"),
8720 None,
8721 Some(json!({"x": 1})),
8722 ),
8723 )
8724 .await
8725 .unwrap();
8726 assert_eq!(
8727 parse_error(&responses[0])["code"],
8728 "admission_facts_not_permitted"
8729 );
8730 }
8731
8732 #[tokio::test]
8733 async fn route_open_relays_consumer_capabilities_verbatim() {
8734 let registry = Arc::new(Registry::default());
8735 let forwarding = Arc::new(ForwardingTable::default());
8736 let handler =
8737 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8738 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
8739 hello_via_sink(
8740 &handler,
8741 &module_ctx,
8742 &mut module_rx,
8743 hello_frame("aft", PROTOCOL_VERSION, 7),
8744 )
8745 .await;
8746
8747 let expected = vec!["elicitation".to_string(), "roots".to_string()];
8748 let expected_for_request = expected.clone();
8749 let project_root = unique_project_root("consumer-capabilities-present");
8750 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
8751 let route_handler = handler.clone();
8752 let route_task = tokio::spawn(async move {
8753 route_handler
8754 .handle_control_frame(
8755 &client_ctx,
8756 route_open_frame_with_consumer_capabilities(
8757 401,
8758 "aft",
8759 project_root,
8760 Some(expected_for_request),
8761 ),
8762 )
8763 .await
8764 .unwrap()
8765 });
8766 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8767 .await
8768 .unwrap()
8769 .unwrap();
8770 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8771 let ModuleControlRequest::RouteBind {
8772 consumer_capabilities,
8773 ..
8774 } = bind
8775 else {
8776 panic!("expected route.bind request, got {bind:?}");
8777 };
8778 assert_eq!(consumer_capabilities, Some(expected.clone()));
8779
8780 handler
8781 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8782 .await
8783 .unwrap();
8784 let route_response = route_task.await.unwrap();
8785 assert!(route_response.is_empty());
8786 let published = client_rx.recv().await.unwrap();
8787 assert!(matches!(
8788 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8789 ClientControlResponse::RouteOpen { .. }
8790 ));
8791 }
8792
8793 #[tokio::test]
8794 async fn route_open_without_consumer_capabilities_relays_none() {
8795 let registry = Arc::new(Registry::default());
8796 let forwarding = Arc::new(ForwardingTable::default());
8797 let handler =
8798 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8799 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
8800 hello_via_sink(
8801 &handler,
8802 &module_ctx,
8803 &mut module_rx,
8804 hello_frame("aft", PROTOCOL_VERSION, 7),
8805 )
8806 .await;
8807
8808 let project_root = unique_project_root("consumer-capabilities-absent");
8809 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
8810 let route_handler = handler.clone();
8811 let route_task = tokio::spawn(async move {
8812 route_handler
8813 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
8814 .await
8815 .unwrap()
8816 });
8817 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8818 .await
8819 .unwrap()
8820 .unwrap();
8821 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8822 let ModuleControlRequest::RouteBind {
8823 consumer_capabilities,
8824 ..
8825 } = bind
8826 else {
8827 panic!("expected route.bind request, got {bind:?}");
8828 };
8829 assert_eq!(consumer_capabilities, None);
8830
8831 handler
8832 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8833 .await
8834 .unwrap();
8835 let route_response = route_task.await.unwrap();
8836 assert!(route_response.is_empty());
8837 let published = client_rx.recv().await.unwrap();
8838 assert!(matches!(
8839 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8840 ClientControlResponse::RouteOpen { .. }
8841 ));
8842 }
8843
8844 #[tokio::test]
8845 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
8846 let registry = Arc::new(Registry::default());
8847 let forwarding = Arc::new(ForwardingTable::default());
8848 let handler =
8849 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8850 .with_health_probe_timeout(Duration::from_secs(5));
8851 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
8852 hello_via_sink(
8853 &handler,
8854 &module_ctx,
8855 &mut module_rx,
8856 non_routable_hello_frame_with_control_ops(
8857 "mcp",
8858 300,
8859 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8860 ),
8861 )
8862 .await;
8863 assert!(registry
8864 .get_module("mcp")
8865 .unwrap()
8866 .unwrap()
8867 .manifest
8868 .provides
8869 .is_empty());
8870
8871 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
8872 let route_response = handler
8873 .handle_control_frame(
8874 &route_client_ctx,
8875 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
8876 )
8877 .await
8878 .unwrap();
8879 assert_eq!(route_response[0].header.ty, FrameType::Error);
8880 assert_eq!(
8881 parse_error(&route_response[0])["code"],
8882 "target_unavailable"
8883 );
8884 assert!(parse_error(&route_response[0])["message"]
8885 .as_str()
8886 .unwrap()
8887 .contains("does not provide the requested target"));
8888 assert!(module_rx.try_recv().is_err());
8889
8890 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
8891 let health_handler = handler.clone();
8892 let health_task = tokio::spawn(async move {
8893 health_handler
8894 .handle_control_frame(
8895 &health_client_ctx,
8896 supervisor_health_probe_frame(302, "mcp"),
8897 )
8898 .await
8899 .unwrap()
8900 });
8901 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8902 .await
8903 .unwrap()
8904 .unwrap();
8905 assert_eq!(
8906 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8907 ModuleControlRequest::HealthCheck {}
8908 );
8909 handler
8910 .handle_control_frame(
8911 &module_ctx,
8912 health_response(health_frame.header.corr, HealthStatus::Ok),
8913 )
8914 .await
8915 .unwrap();
8916 let health_response = health_task.await.unwrap();
8917 assert_eq!(health_response[0].header.ty, FrameType::Response);
8918 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8919 ClientControlResponse::SupervisorHealthProbe {
8920 module_id, status, ..
8921 } => {
8922 assert_eq!(module_id, "mcp");
8923 assert_eq!(status, HealthStatus::Ok);
8924 }
8925 other => panic!("unexpected health response: {other:?}"),
8926 }
8927
8928 forwarding
8933 .cleanup_connection(module_ctx.connection_id)
8934 .unwrap();
8935 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
8936 let cleanup_response = tokio::time::timeout(
8937 Duration::from_millis(200),
8938 handler.handle_control_frame(
8939 &cleanup_probe_ctx,
8940 supervisor_health_probe_frame(303, "mcp"),
8941 ),
8942 )
8943 .await
8944 .expect("probe should fail immediately when the control lane is gone")
8945 .unwrap();
8946 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
8947 assert_eq!(
8948 parse_error(&cleanup_response[0])["code"],
8949 "target_unavailable"
8950 );
8951 assert!(parse_error(&cleanup_response[0])["message"]
8952 .as_str()
8953 .unwrap()
8954 .contains("no module connection"));
8955
8956 handler
8957 .cleanup_connection(module_ctx.connection_id)
8958 .unwrap();
8959 }
8960
8961 #[tokio::test]
8962 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
8963 let registry = Arc::new(Registry::default());
8964 let supervisor_handle = SupervisorHandle::new();
8965 let supervisor =
8966 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
8967 .with_handle(supervisor_handle.clone())
8968 .with_connection_file_path(
8969 std::env::temp_dir()
8970 .join(format!("subc-route-open-warming-{}", std::process::id())),
8971 );
8972 let module = supervisor
8973 .supervise_configured(
8974 ModuleSpec {
8975 module_id: "warming".to_string(),
8976 program: fake_aft_stub_path(),
8977 args: Vec::new(),
8978 env: Vec::new(),
8979 reserved: false,
8980 reserved_prefixes: Vec::new(),
8981 protocol: ModuleProtocol::Subc,
8982 overlap: Default::default(),
8983 },
8984 true,
8985 )
8986 .unwrap();
8987 assert_eq!(module.state().unwrap(), ModuleState::Running);
8988
8989 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8990 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
8991 let response = handler
8992 .handle_control_frame(
8993 &ctx,
8994 route_open_frame(304, "warming", unique_project_root("warming")),
8995 )
8996 .await
8997 .unwrap();
8998 module.stop().await.unwrap();
8999
9000 assert_eq!(response[0].header.ty, FrameType::Error);
9001 let error = parse_error(&response[0]);
9002 assert_eq!(error["code"], "module_warming");
9003 assert!(error["message"]
9004 .as_str()
9005 .unwrap()
9006 .contains("state=running, enabled=true, live=false"));
9007 }
9008
9009 #[test]
9010 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
9011 let handler = ControlHandler::new(Arc::new(Registry::default()));
9012 let capture = EventCapture::default();
9013 let _subscriber =
9014 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9015 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
9016 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
9017 let pending = (0..limit).collect::<Vec<_>>();
9018 let response = handler
9019 .route_open_capacity_refusal(
9020 &ctx,
9021 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
9022 "busy",
9023 pending.len(),
9024 limit,
9025 )
9026 .unwrap();
9027 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9028 let event = capture
9029 .events()
9030 .into_iter()
9031 .find(|event| {
9032 event.target == "control"
9033 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
9034 })
9035 .expect("connection admission refusal event");
9036 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9037 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9038 }
9039
9040 #[test]
9041 fn route_open_target_cap_logs_admission_reason_and_capacity() {
9042 let handler = ControlHandler::new(Arc::new(Registry::default()));
9043 let capture = EventCapture::default();
9044 let _subscriber =
9045 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9046 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
9047 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
9048 let guards = (0..limit)
9049 .map(|_| {
9050 handler
9051 .route_bind_concurrency
9052 .try_admit("busy", limit)
9053 .unwrap()
9054 })
9055 .collect::<Vec<_>>();
9056 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
9057 Err(in_flight) => in_flight,
9058 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
9059 };
9060 let response = handler
9061 .route_open_target_capacity_refusal(
9062 &ctx,
9063 &route_open_frame(397, "busy", unique_project_root("target-cap")),
9064 "busy",
9065 in_flight,
9066 )
9067 .unwrap();
9068 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9069 let event = capture
9070 .events()
9071 .into_iter()
9072 .find(|event| {
9073 event.target == "control"
9074 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
9075 })
9076 .expect("target admission refusal event");
9077 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9078 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9079 drop(guards);
9080 }
9081
9082 #[tokio::test]
9087 async fn route_open_refusal_names_the_check_that_refused() {
9088 let handler = ControlHandler::new(Arc::new(Registry::default()));
9089 let capture = EventCapture::default();
9090 let _subscriber =
9091 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9092 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9093 let response = handler
9094 .handle_control_frame(
9095 &ctx,
9096 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
9097 )
9098 .await
9099 .unwrap();
9100
9101 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9102 let event = capture
9103 .events()
9104 .into_iter()
9105 .find(|event| {
9106 event.target == "control"
9107 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9108 })
9109 .expect("route.open refusal event");
9110 assert_eq!(
9111 event.fields.get("reason"),
9112 Some(&"\"not_registered\"".to_string())
9113 );
9114 }
9115
9116 #[tokio::test]
9117 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
9118 let registry = Arc::new(Registry::default());
9119 let supervisor_handle = SupervisorHandle::new();
9120 let supervisor =
9121 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9122 .with_handle(supervisor_handle.clone())
9123 .with_connection_file_path(std::env::temp_dir().join(format!(
9124 "subc-route-open-refusal-info-{}",
9125 std::process::id()
9126 )));
9127 let module = supervisor
9128 .supervise_configured(
9129 ModuleSpec {
9130 module_id: "warming".to_string(),
9131 program: fake_aft_stub_path(),
9132 args: Vec::new(),
9133 env: Vec::new(),
9134 reserved: false,
9135 reserved_prefixes: Vec::new(),
9136 protocol: ModuleProtocol::Subc,
9137 overlap: Default::default(),
9138 },
9139 true,
9140 )
9141 .unwrap();
9142 assert_eq!(module.state().unwrap(), ModuleState::Running);
9143
9144 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9145 assert!(handler
9146 .counters()
9147 .snapshot()
9148 .get("route_open_refused_by_code")
9149 .is_none());
9150 let capture = EventCapture::default();
9151 let _subscriber =
9152 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9153 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
9154 let response = handler
9155 .handle_control_frame(
9156 &ctx,
9157 route_open_frame(394, "warming", unique_project_root("refusal-info")),
9158 )
9159 .await
9160 .unwrap();
9161 module.stop().await.unwrap();
9162
9163 assert_eq!(parse_error(&response[0])["code"], "module_warming");
9164 let event = capture
9165 .events()
9166 .into_iter()
9167 .find(|event| {
9168 event.target == "control"
9169 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
9170 })
9171 .expect("route.open refusal event");
9172 assert_eq!(
9173 event.fields.get("module_id"),
9174 Some(&"\"warming\"".to_string())
9175 );
9176 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
9177 assert_eq!(
9178 event.fields.get("reason"),
9179 Some(&"\"supervised_not_registered\"".to_string())
9180 );
9181 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
9182 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
9183 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
9184 assert_eq!(
9185 handler.counters().snapshot()["route_open_refused_by_code"],
9186 json!({ "module_warming": 1 })
9187 );
9188 }
9189
9190 const OUTAGE_START: &str = "route.open refusing module: not serving";
9191 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
9192
9193 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
9194 capture
9195 .events()
9196 .into_iter()
9197 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
9198 .collect()
9199 }
9200
9201 fn supervise_stub(
9202 registry: &Arc<Registry>,
9203 module_id: &str,
9204 enabled: bool,
9205 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
9206 let supervisor_handle = SupervisorHandle::new();
9207 let supervisor =
9208 Supervisor::new(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
9209 .with_handle(supervisor_handle.clone())
9210 .with_connection_file_path(std::env::temp_dir().join(format!(
9211 "subc-route-outage-{module_id}-{}",
9212 std::process::id()
9213 )));
9214 let module = supervisor
9215 .supervise_configured(
9216 ModuleSpec {
9217 module_id: module_id.to_string(),
9218 program: fake_aft_stub_path(),
9219 args: Vec::new(),
9220 env: Vec::new(),
9221 reserved: false,
9222 reserved_prefixes: Vec::new(),
9223 protocol: ModuleProtocol::Subc,
9224 overlap: Default::default(),
9225 },
9226 enabled,
9227 )
9228 .unwrap();
9229 (supervisor_handle, module)
9230 }
9231
9232 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
9233 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
9234 module_id: module_id.to_string(),
9235 drain_timeout_ms: Some(50),
9236 })
9237 .unwrap();
9238 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9239 }
9240
9241 #[test]
9245 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
9246 let registry = Arc::new(Registry::default());
9247 let forwarding = Arc::new(ForwardingTable::default());
9248 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9249 let second = ControlHandler::with_forwarding(registry, forwarding);
9250 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
9251 }
9252
9253 #[tokio::test(flavor = "current_thread")]
9257 async fn route_open_unknown_module_refusals_add_no_outage_state() {
9258 let handler = ControlHandler::new(Arc::new(Registry::default()));
9259 let capture = EventCapture::default();
9260 let _subscriber =
9261 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9262 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
9263 for corr in 0..8 {
9264 let response = handler
9265 .handle_control_frame(
9266 &ctx,
9267 route_open_frame(
9268 380 + corr,
9269 &format!("nobody-{corr}"),
9270 unique_project_root("outage-unknown"),
9271 ),
9272 )
9273 .await
9274 .unwrap();
9275 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9276 }
9277
9278 assert_eq!(handler.route_outages.tracked_module_count(), 0);
9279 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
9280 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
9281 }
9282
9283 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9288 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
9289 let registry = Arc::new(Registry::default());
9290 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
9291 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9292 let capture = EventCapture::default();
9293 let _subscriber =
9294 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9295 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
9296 handler.route_outages.record_accepted("outage-restart");
9299
9300 let response = handler
9301 .handle_control_frame(
9302 &ctx,
9303 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
9304 )
9305 .await
9306 .unwrap();
9307 assert_eq!(response[0].header.ty, FrameType::Error);
9308 let starts = outage_lines(&capture, OUTAGE_START);
9309 assert_eq!(starts.len(), 1, "{starts:?}");
9310 assert_eq!(starts[0].level, tracing::Level::WARN);
9311 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
9312 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
9313 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
9314 handler.route_outages.record_accepted("outage-restart");
9315 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
9316
9317 let restart = handler
9318 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
9319 .await
9320 .unwrap();
9321 assert_eq!(
9322 restart[0].header.ty,
9323 FrameType::Response,
9324 "{:?}",
9325 parse_error(&restart[0])
9326 );
9327 handler
9328 .handle_control_frame(
9329 &ctx,
9330 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
9331 )
9332 .await
9333 .unwrap();
9334 module.stop().await.unwrap();
9335
9336 let starts = outage_lines(&capture, OUTAGE_START);
9337 assert_eq!(starts.len(), 2, "{starts:?}");
9338 assert_eq!(starts[1].level, tracing::Level::INFO);
9339 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
9340 }
9341
9342 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9346 async fn failed_operator_restart_leaves_no_operator_mark() {
9347 let registry = Arc::new(Registry::default());
9348 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
9349 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9350 let capture = EventCapture::default();
9351 let _subscriber =
9352 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9353 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
9354 handler.route_outages.record_accepted("outage-disabled");
9355
9356 let restart = handler
9357 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
9358 .await
9359 .unwrap();
9360 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
9361 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
9362
9363 handler
9364 .handle_control_frame(
9365 &ctx,
9366 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
9367 )
9368 .await
9369 .unwrap();
9370 let starts = outage_lines(&capture, OUTAGE_START);
9371 assert_eq!(starts.len(), 1, "{starts:?}");
9372 assert_eq!(starts[0].level, tracing::Level::WARN);
9373 }
9374
9375 #[tokio::test(flavor = "current_thread")]
9376 async fn route_open_unknown_module_escapes_target_module_id() {
9377 let handler = ControlHandler::new(Arc::new(Registry::default()));
9378 let capture = EventCapture::default();
9379 let _subscriber =
9380 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9381 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
9382 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
9383 let response = handler
9384 .handle_control_frame(
9385 &ctx,
9386 route_open_frame(
9387 395,
9388 hostile_module_id,
9389 unique_project_root("hostile-target-module-id"),
9390 ),
9391 )
9392 .await
9393 .unwrap();
9394
9395 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
9396 let event = capture
9397 .events()
9398 .into_iter()
9399 .find(|event| {
9400 event.target == "control"
9401 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
9402 })
9403 .expect("route.open unknown-module refusal event");
9404 let logged = event.fields.get("module_id").expect("module_id field");
9405 assert!(!logged.bytes().any(|byte| byte < 0x20));
9406 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9407 }
9408
9409 #[tokio::test(flavor = "current_thread")]
9410 async fn route_open_module_rejection_uses_daemon_counter_key() {
9411 let registry = Arc::new(Registry::default());
9412 let forwarding = Arc::new(ForwardingTable::default());
9413 let handler =
9414 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9415 let module_connection = ConnectionId::new(95);
9416 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9417 hello_via_sink(
9418 &handler,
9419 &module_ctx,
9420 &mut module_rx,
9421 hello_frame("aft", PROTOCOL_VERSION, 395),
9422 )
9423 .await;
9424
9425 let client_connection = ConnectionId::new(96);
9426 let (client_ctx, _client_rx) = route_ctx(client_connection);
9427 let capture = EventCapture::default();
9428 let _subscriber =
9429 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9430 let (route_task, bind) = relay_route_open(
9431 &handler,
9432 client_connection,
9433 &client_ctx.egress,
9434 &mut module_rx,
9435 396,
9436 "aft",
9437 "hostile-module-code",
9438 )
9439 .await;
9440 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
9441 let rejection = Frame::build(
9442 FrameType::Error,
9443 control_flags(),
9444 0,
9445 0,
9446 bind.header.corr,
9447 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
9448 )
9449 .unwrap();
9450 handler
9451 .handle_control_frame(&module_ctx, rejection)
9452 .await
9453 .unwrap();
9454
9455 let response = route_task.await.unwrap();
9456 assert_eq!(parse_error(&response[0])["code"], hostile_code);
9457 let counters = handler.counters().snapshot();
9458 assert_eq!(
9459 counters["route_open_refused_by_code"],
9460 json!({ "module_rejected": 1 })
9461 );
9462 assert!(counters["route_open_refused_by_code"]
9463 .get(hostile_code)
9464 .is_none());
9465
9466 let event = capture
9467 .events()
9468 .into_iter()
9469 .find(|event| {
9470 event.target == "control"
9471 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
9472 })
9473 .expect("route.open module-rejection refusal event");
9474 let logged = event.fields.get("module_code").expect("module_code field");
9475 assert!(!logged.bytes().any(|byte| byte < 0x20));
9476 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
9477 }
9478
9479 #[tokio::test]
9480 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
9481 let registry = Arc::new(Registry::default());
9482 let supervisor_handle = SupervisorHandle::new();
9483 let missing_program = std::env::temp_dir().join(format!(
9484 "subc-route-open-missing-program-{}",
9485 std::process::id()
9486 ));
9487 let supervisor =
9488 Supervisor::new(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9489 .with_handle(supervisor_handle.clone());
9490 let module = supervisor
9491 .supervise_configured(
9492 ModuleSpec {
9493 module_id: "failed".to_string(),
9494 program: missing_program,
9495 args: Vec::new(),
9496 env: Vec::new(),
9497 reserved: false,
9498 reserved_prefixes: Vec::new(),
9499 protocol: ModuleProtocol::Subc,
9500 overlap: Default::default(),
9501 },
9502 true,
9503 )
9504 .unwrap();
9505 assert_eq!(module.state().unwrap(), ModuleState::Failed);
9506
9507 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9508 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
9509 let response = handler
9510 .handle_control_frame(
9511 &ctx,
9512 route_open_frame(305, "failed", unique_project_root("failed")),
9513 )
9514 .await
9515 .unwrap();
9516
9517 assert_eq!(response[0].header.ty, FrameType::Error);
9518 let error = parse_error(&response[0]);
9519 assert_eq!(error["code"], "target_unavailable");
9520 assert!(error["message"]
9521 .as_str()
9522 .unwrap()
9523 .contains("state=failed, enabled=true, live=false"));
9524 }
9525
9526 #[tokio::test]
9527 async fn route_open_role_mismatch_remains_target_unavailable() {
9528 let registry = Arc::new(Registry::default());
9529 let handler = ControlHandler::new(Arc::clone(®istry));
9530 handler
9531 .handle_control(
9532 ConnectionId::new(41),
9533 non_routable_hello_frame_with_control_ops("health-only", 306, None),
9534 )
9535 .unwrap();
9536
9537 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
9538 let response = handler
9539 .handle_control_frame(
9540 &ctx,
9541 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
9542 )
9543 .await
9544 .unwrap();
9545
9546 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9547 assert!(parse_error(&response[0])["message"]
9548 .as_str()
9549 .unwrap()
9550 .contains("does not provide the requested target"));
9551 }
9552
9553 #[tokio::test]
9554 async fn route_open_inactive_registration_remains_target_unavailable() {
9555 let registry = Arc::new(Registry::default());
9556 let handler = ControlHandler::new(Arc::clone(®istry));
9557 handler
9558 .handle_control(
9559 ConnectionId::new(43),
9560 hello_frame("inactive", PROTOCOL_VERSION, 308),
9561 )
9562 .unwrap();
9563 assert!(registry
9564 .set_module_state_for_test("inactive", ChannelState::Closed)
9565 .unwrap());
9566
9567 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
9568 let response = handler
9569 .handle_control_frame(
9570 &ctx,
9571 route_open_frame(309, "inactive", unique_project_root("inactive")),
9572 )
9573 .await
9574 .unwrap();
9575
9576 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9577 assert!(parse_error(&response[0])["message"]
9578 .as_str()
9579 .unwrap()
9580 .contains("is not active"));
9581 }
9582
9583 #[tokio::test]
9584 async fn late_health_reply_is_recorded_through_the_module_response_path() {
9585 let registry = Arc::new(Registry::default());
9586 let forwarding = Arc::new(ForwardingTable::default());
9587 let supervisor_handle = SupervisorHandle::new();
9588 let supervisor = Supervisor::new(Arc::clone(®istry), crate::RestartPolicy::default())
9589 .with_forwarding(Arc::clone(&forwarding))
9590 .with_handle(supervisor_handle.clone());
9591 let module = supervisor
9592 .supervise_configured(
9593 crate::ModuleSpec {
9594 module_id: "late-health-response".to_string(),
9595 program: PathBuf::from("disabled-module"),
9596 args: Vec::new(),
9597 env: Vec::new(),
9598 reserved: false,
9599 reserved_prefixes: Vec::new(),
9600 protocol: ModuleProtocol::Subc,
9601 overlap: Default::default(),
9602 },
9603 false,
9604 )
9605 .unwrap();
9606 let handler =
9607 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9608 .with_supervisor(supervisor_handle);
9609 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
9610 handler
9611 .handle_control_frame(
9612 &module_ctx,
9613 hello_frame_with_control_ops(
9614 "late-health-response",
9615 PROTOCOL_VERSION,
9616 7,
9617 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9618 ),
9619 )
9620 .await
9621 .unwrap();
9622 let probe_started_at = Instant::now() - Duration::from_millis(80);
9623 let pending = forwarding
9624 .begin_health_probe_rpc_for(
9625 "late-health-response",
9626 MODULE_CONTROL_OP_HEALTH_CHECK,
9627 probe_started_at,
9628 Instant::now() - Duration::from_millis(1),
9629 )
9630 .unwrap();
9631 assert!(forwarding
9632 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
9633 .unwrap());
9634
9635 let responses = handler
9636 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
9637 .await
9638 .unwrap();
9639
9640 assert!(responses.is_empty());
9641 let health = module.status().unwrap().health;
9642 assert_eq!(health.late_answer_count, 1);
9643 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
9644 }
9645
9646 #[tokio::test]
9647 async fn health_probe_timeout_and_module_death_are_typed() {
9648 let registry = Arc::new(Registry::default());
9649 let forwarding = Arc::new(ForwardingTable::default());
9650 let handler =
9651 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9652 .with_health_probe_timeout(Duration::from_millis(50));
9653 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9654 hello_via_sink(
9655 &handler,
9656 &module_ctx,
9657 &mut module_rx,
9658 hello_frame_with_control_ops(
9659 "aft",
9660 PROTOCOL_VERSION,
9661 7,
9662 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9663 ),
9664 )
9665 .await;
9666
9667 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9668 let responses = handler
9669 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
9670 .await
9671 .unwrap();
9672 assert_eq!(responses[0].header.ty, FrameType::Error);
9673 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
9674 let _ = module_rx.try_recv();
9675
9676 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
9677 let health_handler = handler.clone();
9678 let death_task = tokio::spawn(async move {
9679 health_handler
9680 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
9681 .await
9682 .unwrap()
9683 });
9684 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9685 .await
9686 .unwrap()
9687 .unwrap();
9688 handler
9689 .cleanup_connection(module_ctx.connection_id)
9690 .unwrap();
9691 let responses = death_task.await.unwrap();
9692 assert_eq!(responses[0].header.ty, FrameType::Error);
9693 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
9694 }
9695
9696 #[test]
9697 fn hello_requires_exact_protocol_version() {
9698 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
9699 let registry = Arc::new(Registry::default());
9700 let handler = ControlHandler::new(Arc::clone(®istry));
9701 let responses = handler
9702 .handle_control(
9703 ConnectionId::new(connection),
9704 hello_frame("aft", offered, 9),
9705 )
9706 .unwrap();
9707
9708 assert_eq!(responses.len(), 1);
9709 assert_eq!(responses[0].header.ty, FrameType::Error);
9710 let error = parse_error(&responses[0]);
9711 assert_eq!(error["code"], "version_unsupported");
9712 assert!(registry.get_module("aft").unwrap().is_none());
9713 assert_eq!(registry.active_registration_count().unwrap(), 0);
9714 }
9715 }
9716
9717 #[test]
9718 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
9719 let registry = Arc::new(Registry::default());
9720 let forwarding = Arc::new(ForwardingTable::default());
9721 let handler =
9722 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9723 let module_connection = ConnectionId::new(301);
9724 let registration = registry
9725 .register_with_control_ops(
9726 manifest("aft-push", PROTOCOL_VERSION),
9727 PROTOCOL_VERSION,
9728 module_connection,
9729 module_baseline_control_ops(),
9730 )
9731 .unwrap();
9732 let (module_tx, _module_rx) = mpsc::channel(8);
9733 let endpoint = forwarding
9734 .register_module_connection(
9735 module_connection,
9736 "aft-push".to_string(),
9737 PROTOCOL_VERSION,
9738 manifest_concurrency(®istration.manifest),
9739 FrameSink::new(module_tx),
9740 )
9741 .unwrap();
9742
9743 let unknown = Frame::build(
9745 FrameType::Push,
9746 control_flags(),
9747 0,
9748 0,
9749 5,
9750 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
9751 )
9752 .unwrap();
9753 let out = handler.handle_status_update(endpoint, unknown).unwrap();
9754 assert!(
9755 out.is_empty(),
9756 "unknown push op must be ignored, got {out:?}"
9757 );
9758
9759 let malformed = Frame::build(
9761 FrameType::Push,
9762 control_flags(),
9763 0,
9764 0,
9765 6,
9766 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
9767 )
9768 .unwrap();
9769 let out = handler.handle_status_update(endpoint, malformed).unwrap();
9770 assert_eq!(out.len(), 1);
9771 assert_eq!(out[0].header.ty, FrameType::Error);
9772 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
9773 }
9774
9775 #[test]
9776 fn hello_rejected_when_connection_already_owns_client_routes() {
9777 let registry = Arc::new(Registry::default());
9778 let forwarding = Arc::new(ForwardingTable::default());
9779 let handler =
9780 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9781 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
9783 let client_connection = ConnectionId::new(202);
9784
9785 let responses = handler
9788 .handle_control(
9789 client_connection,
9790 hello_frame("aft-second", PROTOCOL_VERSION, 9),
9791 )
9792 .unwrap();
9793 assert_eq!(responses[0].header.ty, FrameType::Error);
9794 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
9795 assert!(registry.get_module("aft-second").unwrap().is_none());
9796 }
9797
9798 #[test]
9799 fn reserved_module_hello_requires_matching_launch_nonce() {
9800 let registry = Arc::new(Registry::default());
9801 let supervisor = SupervisorHandle::new();
9802 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9805 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9806
9807 let no_nonce = handler
9809 .handle_control(
9810 ConnectionId::new(1),
9811 hello_frame("vault", PROTOCOL_VERSION, 1),
9812 )
9813 .unwrap();
9814 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
9815 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
9816 assert!(registry.get_module("vault").unwrap().is_none());
9817
9818 let wrong = handler
9820 .handle_control(
9821 ConnectionId::new(2),
9822 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
9823 )
9824 .unwrap();
9825 assert_eq!(wrong[0].header.ty, FrameType::Error);
9826 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
9827 assert!(registry.get_module("vault").unwrap().is_none());
9828
9829 let ok = handler
9831 .handle_control(
9832 ConnectionId::new(3),
9833 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
9834 )
9835 .unwrap();
9836 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
9837 assert!(registry.get_module("vault").unwrap().is_some());
9838 }
9839
9840 #[test]
9841 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
9842 let registry = Arc::new(Registry::default());
9843 let supervisor = SupervisorHandle::new();
9844 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9845 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9846 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9847
9848 let squat = handler
9849 .handle_control(
9850 ConnectionId::new(1),
9851 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
9852 )
9853 .unwrap();
9854 assert_eq!(squat[0].header.ty, FrameType::Error);
9855 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
9856 assert!(parse_error(&squat[0])["message"]
9857 .as_str()
9858 .unwrap()
9859 .contains("fed:"));
9860
9861 let accepted_peer = handler
9862 .handle_control(
9863 ConnectionId::new(2),
9864 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
9865 )
9866 .unwrap();
9867 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
9868
9869 let accepted_short = handler
9870 .handle_control(
9871 ConnectionId::new(3),
9872 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
9873 )
9874 .unwrap();
9875 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
9876
9877 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
9878 let response = handler
9879 .handle_control(
9880 ConnectionId::new(conn),
9881 hello_frame(module_id, PROTOCOL_VERSION, conn),
9882 )
9883 .unwrap();
9884 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
9885 }
9886 }
9887
9888 #[test]
9889 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
9890 let registry = Arc::new(Registry::default());
9891 let supervisor = SupervisorHandle::new();
9892 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
9893 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
9894 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
9895 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9896
9897 let owner_nonce = handler
9898 .handle_control(
9899 ConnectionId::new(1),
9900 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
9901 )
9902 .unwrap();
9903 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
9904 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
9905 assert!(registry.get_module("fed:special").unwrap().is_none());
9906
9907 let exact_nonce = handler
9908 .handle_control(
9909 ConnectionId::new(2),
9910 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
9911 )
9912 .unwrap();
9913 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
9914 assert!(registry.get_module("fed:special").unwrap().is_some());
9915 }
9916
9917 #[test]
9918 fn non_reserved_module_ignores_launch_nonce() {
9919 let registry = Arc::new(Registry::default());
9920 let handler = ControlHandler::new(Arc::clone(®istry));
9923 let no_nonce = handler
9924 .handle_control(
9925 ConnectionId::new(1),
9926 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
9927 )
9928 .unwrap();
9929 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
9930 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
9931
9932 let echoed_nonce = handler
9933 .handle_control(
9934 ConnectionId::new(2),
9935 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
9936 )
9937 .unwrap();
9938 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
9939 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
9940 }
9941
9942 #[test]
9943 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
9944 let handler = ControlHandler::default();
9945 let conn = ConnectionId::new(1);
9946 let malformed = Frame::build(
9947 FrameType::Hello,
9948 control_flags(),
9949 0,
9950 0,
9951 3,
9952 b"{not json".to_vec(),
9953 )
9954 .unwrap();
9955
9956 let error = handler.handle_control(conn, malformed).unwrap();
9957 assert_eq!(error[0].header.ty, FrameType::Error);
9958 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
9959
9960 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
9961 let pong = handler.handle_control(conn, ping).unwrap();
9962 assert_eq!(pong[0].header.ty, FrameType::Pong);
9963 assert_eq!(pong[0].header.corr, 4);
9964 }
9965
9966 #[test]
9967 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
9968 let registry = Arc::new(Registry::default());
9969 let handler = ControlHandler::new(Arc::clone(®istry));
9970
9971 handler
9972 .handle_control(
9973 ConnectionId::new(1),
9974 hello_frame("aft", PROTOCOL_VERSION, 1),
9975 )
9976 .unwrap();
9977 let duplicate = handler
9978 .handle_control(
9979 ConnectionId::new(2),
9980 hello_frame("aft", PROTOCOL_VERSION, 2),
9981 )
9982 .unwrap();
9983
9984 assert_eq!(duplicate[0].header.ty, FrameType::Error);
9985 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9986 let registration = registry.get_module("aft").unwrap().unwrap();
9987 assert_eq!(registration.connection_id, ConnectionId::new(1));
9988 }
9989
9990 #[test]
9991 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
9992 let registry = Arc::new(Registry::default());
9993 let forwarding = Arc::new(ForwardingTable::default());
9994 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
9995 let handler =
9996 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9997 .with_process_liveness(process_liveness);
9998 let (ctx, route_channel, route_epoch) =
9999 bind_liveness_route(®istry, &forwarding, "aft-dead");
10000 let responses = handler
10001 .handle_route_poll(
10002 &ctx,
10003 route_poll_frame(41, PollKind::Liveness, route_channel),
10004 route_channel,
10005 route_epoch,
10006 PollKind::Liveness,
10007 )
10008 .unwrap();
10009
10010 assert_eq!(responses.len(), 1);
10011 assert_eq!(responses[0].header.ty, FrameType::Response);
10012 assert_route_poll_liveness(&responses[0], false);
10013 }
10014
10015 #[test]
10016 fn liveness_poll_without_process_source_uses_bound_route() {
10017 let registry = Arc::new(Registry::default());
10018 let forwarding = Arc::new(ForwardingTable::default());
10019 let handler =
10020 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10021 let (ctx, route_channel, route_epoch) =
10022 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
10023 let responses = handler
10024 .handle_route_poll(
10025 &ctx,
10026 route_poll_frame(42, PollKind::Liveness, route_channel),
10027 route_channel,
10028 route_epoch,
10029 PollKind::Liveness,
10030 )
10031 .unwrap();
10032
10033 assert_route_poll_liveness(&responses[0], true);
10034 }
10035
10036 #[test]
10037 fn liveness_poll_untracked_process_source_uses_bound_route() {
10038 let registry = Arc::new(Registry::default());
10039 let forwarding = Arc::new(ForwardingTable::default());
10040 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
10041 let handler =
10042 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10043 .with_process_liveness(process_liveness);
10044 let (ctx, route_channel, route_epoch) =
10045 bind_liveness_route(®istry, &forwarding, "aft-untracked");
10046 let responses = handler
10047 .handle_route_poll(
10048 &ctx,
10049 route_poll_frame(43, PollKind::Liveness, route_channel),
10050 route_channel,
10051 route_epoch,
10052 PollKind::Liveness,
10053 )
10054 .unwrap();
10055
10056 assert_route_poll_liveness(&responses[0], true);
10057 }
10058
10059 #[tokio::test]
10060 async fn unknown_op_returns_unknown_control_op() {
10061 let handler = ControlHandler::default();
10062 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
10063 let request = Frame::build(
10064 FrameType::Request,
10065 control_flags(),
10066 0,
10067 0,
10068 55,
10069 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
10070 )
10071 .unwrap();
10072
10073 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10074
10075 assert_eq!(response.len(), 1);
10076 assert_eq!(response[0].header.ty, FrameType::Error);
10077 assert_eq!(response[0].header.corr, 55);
10078 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
10079 }
10080
10081 #[tokio::test]
10082 async fn supervisor_provenance_rejects_unknown_exact_module() {
10083 let handler = ControlHandler::default();
10084 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
10085 let request = Frame::build(
10086 FrameType::Request,
10087 control_flags(),
10088 0,
10089 0,
10090 57,
10091 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
10092 )
10093 .unwrap();
10094
10095 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10096
10097 assert_eq!(response.len(), 1);
10098 assert_eq!(response[0].header.ty, FrameType::Error);
10099 assert_eq!(response[0].header.corr, 57);
10100 let error = parse_error(&response[0]);
10101 assert_eq!(error["code"], "unknown_module");
10102 assert_eq!(error["message"], "module_id 'missing' is not supervised");
10103 }
10104
10105 #[test]
10106 fn provenance_probe_override_keeps_handler_tests_deterministic() {
10107 let expected = subc_control::RunningImageAgreement::Unavailable {
10108 reason: subc_control::RunningImageUnavailableReason::HashFailed,
10109 };
10110 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
10111 assert_eq!(handler.provenance_probe_override, Some(expected));
10112 }
10113
10114 #[test]
10115 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
10116 let verdict = reload_verdict(
10117 std::path::Path::new("/bin/new"),
10118 Some(std::path::Path::new("/bin/old")),
10119 subc_control::RunningImageAgreement::Unavailable {
10120 reason: subc_control::RunningImageUnavailableReason::HashFailed,
10121 },
10122 );
10123 assert!(matches!(
10124 verdict.path,
10125 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
10126 if configured == std::path::Path::new("/bin/new")
10127 && spawned_from == std::path::Path::new("/bin/old")
10128 ));
10129 }
10130
10131 #[test]
10132 fn reload_verdict_detects_replaced_image_at_same_path() {
10133 let image = subc_control::RunningImageAgreement::Mismatch {
10134 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
10135 digest: "old".into(),
10136 },
10137 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
10138 digest: "new".into(),
10139 },
10140 };
10141 let verdict = reload_verdict(
10142 std::path::Path::new("/bin/same"),
10143 Some(std::path::Path::new("/bin/same")),
10144 image.clone(),
10145 );
10146 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10147 assert_eq!(verdict.image, image);
10148 }
10149
10150 #[test]
10151 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
10152 let image = subc_control::RunningImageAgreement::Unavailable {
10153 reason: subc_control::RunningImageUnavailableReason::NotRunning,
10154 };
10155 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
10156 assert_eq!(
10157 verdict.path,
10158 subc_control::ReloadPathAgreement::Unavailable {
10159 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
10160 }
10161 );
10162 assert_eq!(verdict.image, image);
10163
10164 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
10165 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
10166 };
10167 let verdict = reload_verdict(
10168 std::path::Path::new("/bin/same"),
10169 Some(std::path::Path::new("/bin/same")),
10170 unconfirmed.clone(),
10171 );
10172 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10173 assert_eq!(verdict.image, unconfirmed);
10174 }
10175
10176 #[test]
10177 fn reload_verdict_preserves_each_image_unavailability_reason() {
10178 use subc_control::RunningImageUnavailableReason as Reason;
10179
10180 for reason in [
10181 Reason::NotRunning,
10182 Reason::UnsupportedPlatform,
10183 Reason::RunningExecutableUnreadable,
10184 Reason::SpawnedPathUnreadable,
10185 Reason::HashFailed,
10186 Reason::ProcessIdentityUnconfirmed,
10187 Reason::Unknown("future_probe_reason".to_string()),
10188 ] {
10189 let image = subc_control::RunningImageAgreement::Unavailable {
10190 reason: reason.clone(),
10191 };
10192 let verdict = reload_verdict(
10193 std::path::Path::new("/bin/same"),
10194 Some(std::path::Path::new("/bin/same")),
10195 image.clone(),
10196 );
10197 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
10198 assert_eq!(verdict.image, image, "{reason:?}");
10199 }
10200 }
10201
10202 #[tokio::test]
10203 async fn malformed_control_bodies_return_invalid_control_body() {
10204 let handler = ControlHandler::default();
10205 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
10206
10207 for (corr, body) in [
10208 (56, br#"{"route_channel":1}"#.as_slice()),
10209 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
10210 (
10211 58,
10212 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
10213 ),
10214 ] {
10215 let request = Frame::build(
10216 FrameType::Request,
10217 control_flags(),
10218 0,
10219 0,
10220 corr,
10221 body.to_vec(),
10222 )
10223 .unwrap();
10224 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
10225
10226 assert_eq!(response.len(), 1);
10227 assert_eq!(response[0].header.ty, FrameType::Error);
10228 assert_eq!(response[0].header.corr, corr);
10229 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
10230 }
10231 }
10232
10233 #[tokio::test]
10234 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
10235 let registry = Arc::new(Registry::default());
10236 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
10237 let router = Router::with_control_handler(Arc::clone(&control));
10238 let connection = router.begin_connection();
10239 let (ctx, mut rx) = route_ctx(connection.id());
10240
10241 router
10242 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
10243 .await
10244 .unwrap();
10245 let response = rx.recv().await.unwrap();
10246 let ack = parse_ack(&response);
10247 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10248 let channel = 1;
10249
10250 let goodbye =
10251 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
10252 router.route_for_connection(&ctx, goodbye).await.unwrap();
10253 assert!(rx.try_recv().is_err());
10254 assert!(registry.get_module("aft").unwrap().is_none());
10255
10256 router
10257 .route_for_connection(&ctx, channel_request(channel, 13))
10258 .await
10259 .unwrap();
10260 let error_frame = rx.recv().await.unwrap();
10261 assert_eq!(error_frame.header.ty, FrameType::Error);
10262 assert_eq!(error_frame.header.channel, channel);
10263 }
10264
10265 #[tokio::test]
10266 async fn dropping_router_connection_releases_registration() {
10267 let registry = Arc::new(Registry::default());
10268 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
10269 let router = Router::with_control_handler(control);
10270 let connection = router.begin_connection();
10271 let (ctx, mut rx) = route_ctx(connection.id());
10272
10273 router
10274 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
10275 .await
10276 .unwrap();
10277 let response = rx.recv().await.unwrap();
10278 let ack = parse_ack(&response);
10279 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10280 assert!(registry.get_module("aft").unwrap().is_some());
10281
10282 drop(connection);
10283
10284 assert!(registry.get_module("aft").unwrap().is_none());
10285 assert_eq!(registry.active_registration_count().unwrap(), 0);
10286 }
10287
10288 fn capability_manifest(
10289 module_id: &str,
10290 provides: &[&str],
10291 must_never_reach: &[&str],
10292 ) -> ModuleManifest {
10293 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
10294 manifest.capabilities = Some(CapabilityDeclarations {
10295 provides: provides
10296 .iter()
10297 .map(|capability| (*capability).to_string())
10298 .collect(),
10299 requires: Vec::new(),
10300 must_never_reach: must_never_reach
10301 .iter()
10302 .map(|capability| (*capability).to_string())
10303 .collect(),
10304 });
10305 manifest
10306 }
10307
10308 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
10309 Frame::build(
10310 FrameType::Hello,
10311 control_flags(),
10312 0,
10313 0,
10314 corr,
10315 serde_json::to_vec(&ModuleHelloBody {
10316 protocol_ver: manifest.protocol_ver,
10317 manifest,
10318 control_ops: None,
10319 launch_nonce: None,
10320 })
10321 .expect("capability test HELLO serializes"),
10322 )
10323 .expect("capability test HELLO frame builds")
10324 }
10325
10326 fn catalog_update_with_capabilities_frame(
10327 corr: u64,
10328 capabilities: CapabilityDeclarations,
10329 ) -> Frame {
10330 Frame::build(
10331 FrameType::Request,
10332 control_flags(),
10333 0,
10334 0,
10335 corr,
10336 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
10337 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
10338 capabilities: Some(capabilities),
10339 ready: None,
10340 })
10341 .expect("capability catalog.update serializes"),
10342 )
10343 .expect("capability catalog.update frame builds")
10344 }
10345
10346 async fn register_capability_manifest(
10347 handler: &ControlHandler,
10348 ctx: &RouteCtx,
10349 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10350 manifest: ModuleManifest,
10351 corr: u64,
10352 ) {
10353 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
10354 }
10355
10356 async fn open_route_for_capability_test(
10357 handler: &ControlHandler,
10358 target_ctx: &RouteCtx,
10359 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10360 client_connection_id: u64,
10361 corr: u64,
10362 target_module_id: &str,
10363 consumer_identity: Option<ConsumerIdentity>,
10364 ) -> (
10365 mpsc::Receiver<crate::router::OutboundFrame>,
10366 ModuleControlRequest,
10367 ) {
10368 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
10369 let route_handler = handler.clone();
10370 let target_module_id = target_module_id.to_string();
10371 let route_task = tokio::spawn(async move {
10372 route_handler
10373 .handle_control_frame(
10374 &client_ctx,
10375 route_open_frame_with_admission_facts(
10376 corr,
10377 &target_module_id,
10378 unique_project_root("admission-facts"),
10379 consumer_identity,
10380 None,
10381 ),
10382 )
10383 .await
10384 .expect("capability test route.open succeeds")
10385 });
10386 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
10387 .await
10388 .expect("capability test route.open must reach route.bind")
10389 .expect("target control receiver stays open");
10390 let bind_request: ModuleControlRequest =
10391 serde_json::from_slice(&bind.body).expect("route.bind decodes");
10392 handler
10393 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
10394 .await
10395 .expect("capability test route.bind ACK succeeds");
10396 assert!(route_task.await.expect("route.open task joins").is_empty());
10397 let opened = client_rx
10398 .recv()
10399 .await
10400 .expect("successful route.open publishes a response");
10401 assert!(matches!(
10402 serde_json::from_slice::<ClientControlResponse>(&opened.body),
10403 Ok(ClientControlResponse::RouteOpen { .. })
10404 ));
10405 (client_rx, bind_request)
10406 }
10407
10408 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
10409 assert_eq!(frame.header.ty, FrameType::Push);
10410 assert_eq!(frame.header.channel, 0);
10411 assert_eq!(
10412 serde_json::from_slice::<ClientControlPush>(&frame.body)
10413 .expect("route.closed control push decodes"),
10414 ClientControlPush::RouteClosed {
10415 module_id: target_module_id.to_string(),
10416 reason: RouteCloseReason::CapabilityDenied,
10417 drained: false,
10418 abandoned: 0,
10419 excluded_subscriptions: 0,
10420 terminal: Some(false),
10421 }
10422 );
10423 }
10424
10425 #[tokio::test]
10426 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
10427 let registry = Arc::new(Registry::default());
10428 let forwarding = Arc::new(ForwardingTable::default());
10429 let supervisor = SupervisorHandle::new();
10430 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10431 let handler =
10432 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10433 .with_supervisor(supervisor);
10434 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
10435 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
10436 register_capability_manifest(
10437 &handler,
10438 &target_ctx,
10439 &mut target_rx,
10440 capability_manifest("target", &["credentials-provider/v1"], &[]),
10441 1,
10442 )
10443 .await;
10444 register_capability_manifest(
10445 &handler,
10446 &opener_ctx,
10447 &mut opener_rx,
10448 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10449 2,
10450 )
10451 .await;
10452
10453 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
10454 let replies = handler
10455 .handle_control_frame(
10456 &client_ctx,
10457 route_open_frame_with_admission_facts(
10458 3,
10459 "target",
10460 unique_project_root("admission-facts"),
10461 Some(ConsumerIdentity {
10462 module_id: "opener".to_string(),
10463 launch_nonce: "opener-nonce".to_string(),
10464 }),
10465 None,
10466 ),
10467 )
10468 .await
10469 .expect("denied route.open returns a typed frame");
10470 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10471 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10472 assert!(
10473 target_rx.try_recv().is_err(),
10474 "forbidden route.open must not relay route.bind"
10475 );
10476 }
10477
10478 #[tokio::test]
10479 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
10480 let registry = Arc::new(Registry::default());
10481 let forwarding = Arc::new(ForwardingTable::default());
10482 let supervisor = SupervisorHandle::new();
10483 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10484 let handler =
10485 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10486 .with_supervisor(supervisor);
10487 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
10488 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
10489 register_capability_manifest(
10490 &handler,
10491 &target_ctx,
10492 &mut target_rx,
10493 capability_manifest("target", &["credentials-provider/v1"], &[]),
10494 1,
10495 )
10496 .await;
10497 register_capability_manifest(
10498 &handler,
10499 &old_opener_ctx,
10500 &mut old_opener_rx,
10501 capability_manifest("opener", &[], &[]),
10502 2,
10503 )
10504 .await;
10505 let (mut client_rx, _) = open_route_for_capability_test(
10506 &handler,
10507 &target_ctx,
10508 &mut target_rx,
10509 712,
10510 3,
10511 "target",
10512 Some(ConsumerIdentity {
10513 module_id: "opener".to_string(),
10514 launch_nonce: "opener-nonce".to_string(),
10515 }),
10516 )
10517 .await;
10518 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10519
10520 handler
10521 .cleanup_connection(old_opener_ctx.connection_id)
10522 .expect("old opener registration cleans up");
10523 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
10524 register_capability_manifest(
10525 &handler,
10526 &new_opener_ctx,
10527 &mut new_opener_rx,
10528 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10529 4,
10530 )
10531 .await;
10532
10533 assert_capability_denied_push(
10534 client_rx
10535 .try_recv()
10536 .expect("HELLO deny addition must emit route.closed")
10537 .frame,
10538 "target",
10539 );
10540 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10541 assert!(matches!(
10542 target_rx.try_recv(),
10543 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10544 ));
10545 }
10546
10547 #[tokio::test]
10548 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
10549 let registry = Arc::new(Registry::default());
10550 let forwarding = Arc::new(ForwardingTable::default());
10551 let supervisor = SupervisorHandle::new();
10552 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10553 let handler =
10554 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10555 .with_supervisor(supervisor);
10556 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
10557 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
10558 register_capability_manifest(
10559 &handler,
10560 &target_ctx,
10561 &mut target_rx,
10562 capability_manifest("target", &[], &[]),
10563 1,
10564 )
10565 .await;
10566 register_capability_manifest(
10567 &handler,
10568 &opener_ctx,
10569 &mut opener_rx,
10570 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10571 2,
10572 )
10573 .await;
10574 let (mut client_rx, _) = open_route_for_capability_test(
10575 &handler,
10576 &target_ctx,
10577 &mut target_rx,
10578 722,
10579 3,
10580 "target",
10581 Some(ConsumerIdentity {
10582 module_id: "opener".to_string(),
10583 launch_nonce: "opener-nonce".to_string(),
10584 }),
10585 )
10586 .await;
10587 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10588
10589 let replies = handler
10590 .handle_control_frame(
10591 &target_ctx,
10592 catalog_update_with_capabilities_frame(
10593 4,
10594 CapabilityDeclarations {
10595 provides: vec!["credentials-provider/v1".to_string()],
10596 requires: Vec::new(),
10597 must_never_reach: Vec::new(),
10598 },
10599 ),
10600 )
10601 .await
10602 .expect("claim catalog.update succeeds");
10603 assert!(matches!(
10604 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
10605 Ok(ModuleControlResponseToModule::CatalogUpdate {})
10606 ));
10607 assert_capability_denied_push(
10608 client_rx
10609 .try_recv()
10610 .expect("claim addition must emit route.closed")
10611 .frame,
10612 "target",
10613 );
10614 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10615 assert!(matches!(
10616 target_rx.try_recv(),
10617 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
10618 ));
10619 }
10620
10621 #[tokio::test]
10622 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
10623 let registry = Arc::new(Registry::default());
10624 let forwarding = Arc::new(ForwardingTable::default());
10625 let supervisor = SupervisorHandle::new();
10626 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10627 let handler =
10628 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10629 .with_supervisor(supervisor);
10630 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
10631 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
10632 register_capability_manifest(
10633 &handler,
10634 &target_ctx,
10635 &mut target_rx,
10636 capability_manifest("target", &["credentials-provider/v1"], &[]),
10637 1,
10638 )
10639 .await;
10640 register_capability_manifest(
10641 &handler,
10642 &opener_ctx,
10643 &mut opener_rx,
10644 capability_manifest("opener", &[], &[]),
10645 2,
10646 )
10647 .await;
10648 let (mut client_rx, _) = open_route_for_capability_test(
10649 &handler,
10650 &target_ctx,
10651 &mut target_rx,
10652 732,
10653 3,
10654 "target",
10655 Some(ConsumerIdentity {
10656 module_id: "opener".to_string(),
10657 launch_nonce: "opener-nonce".to_string(),
10658 }),
10659 )
10660 .await;
10661 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10662
10663 handler
10664 .handle_control_frame(
10665 &target_ctx,
10666 catalog_update_with_capabilities_frame(
10667 4,
10668 CapabilityDeclarations {
10669 provides: Vec::new(),
10670 requires: Vec::new(),
10671 must_never_reach: Vec::new(),
10672 },
10673 ),
10674 )
10675 .await
10676 .expect("claim removal catalog.update succeeds");
10677 assert_eq!(
10678 forwarding.active_binding_count().unwrap(),
10679 1,
10680 "removing an attested target claim must leave the route census unchanged"
10681 );
10682 assert!(
10683 client_rx.try_recv().is_err(),
10684 "claim removal must not emit route.closed capability_denied"
10685 );
10686 assert!(
10687 target_rx.try_recv().is_err(),
10688 "claim removal must not send the target a route GOODBYE"
10689 );
10690 }
10691
10692 #[tokio::test]
10695 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
10696 let registry = Arc::new(Registry::default());
10697 let forwarding = Arc::new(ForwardingTable::default());
10698 let supervisor = SupervisorHandle::new();
10699 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
10700 let handler =
10701 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10702 .with_supervisor(supervisor);
10703 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
10704 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
10705 register_capability_manifest(
10706 &handler,
10707 &target_ctx,
10708 &mut target_rx,
10709 capability_manifest("target", &["credentials-provider/v1"], &[]),
10710 1,
10711 )
10712 .await;
10713 register_capability_manifest(
10714 &handler,
10715 &opener_ctx,
10716 &mut opener_rx,
10717 capability_manifest("opener", &[], &["credentials-provider/v1"]),
10718 2,
10719 )
10720 .await;
10721
10722 let (_client_rx, bind) = open_route_for_capability_test(
10723 &handler,
10724 &target_ctx,
10725 &mut target_rx,
10726 742,
10727 3,
10728 "target",
10729 None,
10730 )
10731 .await;
10732 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10733 panic!("direct scope-honesty route must bind");
10734 };
10735 assert_eq!(principal, Some(Principal::Direct));
10736 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
10737 }
10738
10739 #[tokio::test]
10742 async fn must_never_reach_self_route_is_capability_forbidden() {
10743 let registry = Arc::new(Registry::default());
10744 let forwarding = Arc::new(ForwardingTable::default());
10745 let supervisor = SupervisorHandle::new();
10746 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
10747 let handler =
10748 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10749 .with_supervisor(supervisor);
10750 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
10751 register_capability_manifest(
10752 &handler,
10753 &self_ctx,
10754 &mut self_rx,
10755 capability_manifest(
10756 "self-provider",
10757 &["credentials-provider/v1"],
10758 &["credentials-provider/v1"],
10759 ),
10760 1,
10761 )
10762 .await;
10763
10764 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
10765 let replies = handler
10766 .handle_control_frame(
10767 &client_ctx,
10768 route_open_frame_with_admission_facts(
10769 2,
10770 "self-provider",
10771 unique_project_root("admission-facts"),
10772 Some(ConsumerIdentity {
10773 module_id: "self-provider".to_string(),
10774 launch_nonce: "self-nonce".to_string(),
10775 }),
10776 None,
10777 ),
10778 )
10779 .await
10780 .expect("self-route refusal returns a typed frame");
10781 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
10782 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
10783 assert!(
10784 self_rx.try_recv().is_err(),
10785 "self denial must not relay route.bind"
10786 );
10787 }
10788
10789 #[test]
10790 fn unsupported_channel_zero_frame_returns_error() {
10791 let handler = ControlHandler::default();
10792 let request = Frame::build(
10793 FrameType::Request,
10794 control_flags(),
10795 0,
10796 0,
10797 21,
10798 b"opaque".to_vec(),
10799 )
10800 .unwrap();
10801
10802 let response = handler
10803 .handle_control(ConnectionId::new(1), request)
10804 .unwrap();
10805
10806 assert_eq!(response[0].header.ty, FrameType::Error);
10807 assert_eq!(
10808 parse_error(&response[0])["code"],
10809 "unsupported_control_frame"
10810 );
10811 }
10812
10813 mod swap {
10818 use super::*;
10819
10820 const INCUMBENT: ConnectionId = ConnectionId::new(30);
10821 const CANDIDATE: ConnectionId = ConnectionId::new(40);
10822
10823 struct Swap {
10824 registry: Arc<Registry>,
10825 forwarding: Arc<ForwardingTable>,
10826 handler: ControlHandler,
10827 incumbent_ctx: RouteCtx,
10828 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10829 candidate_ctx: RouteCtx,
10830 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
10831 }
10832
10833 async fn swap_with_incumbent() -> Swap {
10834 let registry = Arc::new(Registry::default());
10835 let forwarding = Arc::new(ForwardingTable::default());
10836 let handler =
10837 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10838 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
10839 hello_via_sink(
10840 &handler,
10841 &incumbent_ctx,
10842 &mut incumbent_rx,
10843 hello_frame("aft", PROTOCOL_VERSION, 7),
10844 )
10845 .await;
10846 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
10847 Swap {
10848 registry,
10849 forwarding,
10850 handler,
10851 incumbent_ctx,
10852 incumbent_rx,
10853 candidate_ctx,
10854 candidate_rx,
10855 }
10856 }
10857
10858 fn register_candidate(swap: &Swap, ready: Option<bool>) {
10859 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
10860 candidate_manifest.ready = ready;
10861 let registration = swap
10862 .registry
10863 .register_candidate_with_control_ops(
10864 candidate_manifest,
10865 PROTOCOL_VERSION,
10866 CANDIDATE,
10867 module_baseline_control_ops(),
10868 )
10869 .unwrap();
10870 swap.forwarding
10871 .register_candidate_module_connection(
10872 CANDIDATE,
10873 "aft".to_string(),
10874 PROTOCOL_VERSION,
10875 manifest_concurrency(®istration.manifest),
10876 swap.candidate_ctx.egress.clone(),
10877 )
10878 .unwrap();
10879 }
10880
10881 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
10882 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
10883 swap.registry.promote_candidate("aft").unwrap().unwrap();
10884 cutover.incumbent.unwrap()
10885 }
10886
10887 fn keyed_total(counters: &Value, key: &str) -> u64 {
10888 counters[key]
10889 .as_object()
10890 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
10891 .unwrap_or(0)
10892 }
10893
10894 #[tokio::test]
10900 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
10901 let mut swap = swap_with_incumbent().await;
10902 let handler = swap.handler.clone();
10903
10904 let cotenant = ConnectionId::new(31);
10906 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10907 let (cotenant_task, cotenant_bind) = relay_route_open(
10908 &handler,
10909 cotenant,
10910 &cotenant_ctx.egress,
10911 &mut swap.incumbent_rx,
10912 100,
10913 "aft",
10914 "swap-cotenant",
10915 )
10916 .await;
10917 handler
10918 .handle_control_frame(
10919 &swap.incumbent_ctx,
10920 route_bind_ack(cotenant_bind.header.corr),
10921 )
10922 .await
10923 .unwrap();
10924 assert!(cotenant_task.await.unwrap().is_empty());
10925 let (cotenant_channel, cotenant_epoch) =
10926 published_route(&cotenant_rx.recv().await.unwrap());
10927
10928 let caller = ConnectionId::new(32);
10930 let (caller_ctx, mut caller_rx) = route_ctx(caller);
10931 let (caller_task, caller_bind) = relay_route_open(
10932 &handler,
10933 caller,
10934 &caller_ctx.egress,
10935 &mut swap.incumbent_rx,
10936 101,
10937 "aft",
10938 "swap-caller",
10939 )
10940 .await;
10941 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
10942
10943 register_candidate(&swap, None);
10944 cutover(&swap);
10945
10946 let ack = handler
10948 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
10949 .await;
10950 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10951 if module_loop_error.is_some() {
10952 handler.cleanup_connection(INCUMBENT).unwrap();
10955 }
10956
10957 assert!(
10959 cotenant_rx.try_recv().is_err(),
10960 "the co-tenant route on the incumbent was torn down by one late ack: \
10961 {module_loop_error:?}"
10962 );
10963 assert!(matches!(
10964 swap.forwarding
10965 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10966 .unwrap(),
10967 DataRoute::Client(DataRouteState::Bound(_))
10968 ));
10969 assert_eq!(module_loop_error, None);
10970 assert!(swap
10971 .registry
10972 .get_module_by_connection(INCUMBENT)
10973 .unwrap()
10974 .is_some());
10975
10976 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
10978 .await
10979 .expect("the incumbent is told to drop the abandoned binding")
10980 .unwrap()
10981 .frame;
10982 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10983 assert_eq!(goodbye.header.channel, abandoned_channel);
10984 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10985 assert!(swap.incumbent_rx.try_recv().is_err());
10986
10987 let response = caller_task.await.unwrap();
10989 assert_eq!(response.len(), 1);
10990 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
10991 assert!(caller_rx.try_recv().is_err());
10992
10993 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
10997 let counters = handler.counters().snapshot();
10998 assert_eq!(
10999 keyed_total(&counters, "route_open_accepted_by_principal"),
11000 1
11001 );
11002 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
11003 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
11004 }
11005
11006 #[tokio::test]
11010 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
11011 let mut swap = swap_with_incumbent().await;
11012 register_candidate(&swap, None);
11013 let incumbent = cutover(&swap);
11014 swap.forwarding
11015 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
11016 .unwrap()
11017 .expect("the incumbent is still registered");
11018
11019 let client = ConnectionId::new(33);
11020 let (client_ctx, mut client_rx) = route_ctx(client);
11021 let route_handler = swap.handler.clone();
11022 let open_ctx = RouteCtx {
11023 connection_id: client,
11024 egress: client_ctx.egress.clone(),
11025 };
11026 let mut route_task = tokio::spawn(async move {
11027 route_handler
11028 .handle_control_frame(
11029 &open_ctx,
11030 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
11031 )
11032 .await
11033 .unwrap()
11034 });
11035 let bind = tokio::select! {
11036 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
11037 response = &mut route_task => {
11038 let response = response.unwrap();
11039 panic!(
11040 "post-cutover route.open was refused instead of relayed to the candidate: {}",
11041 parse_error(&response[0])["code"]
11042 );
11043 }
11044 };
11045 swap.handler
11046 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
11047 .await
11048 .unwrap();
11049 assert!(route_task.await.unwrap().is_empty());
11050 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
11051 match swap
11052 .forwarding
11053 .lookup_data_route(client, channel, epoch)
11054 .unwrap()
11055 {
11056 DataRoute::Client(DataRouteState::Bound(route)) => {
11057 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
11058 }
11059 other => panic!("expected a bound route on the candidate, got {other:?}"),
11060 }
11061 assert!(swap.incumbent_rx.try_recv().is_err());
11062 }
11063
11064 #[tokio::test]
11069 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
11070 let swap = swap_with_incumbent().await;
11071 register_candidate(&swap, Some(false));
11072 let update = Frame::build(
11073 FrameType::Request,
11074 control_flags(),
11075 0,
11076 0,
11077 55,
11078 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11079 provides: manifest("aft", PROTOCOL_VERSION).provides,
11080 capabilities: None,
11081 ready: Some(true),
11082 })
11083 .unwrap(),
11084 )
11085 .unwrap();
11086
11087 let replies = swap
11088 .handler
11089 .handle_control_frame(&swap.candidate_ctx, update)
11090 .await
11091 .unwrap();
11092
11093 assert_eq!(replies.len(), 1);
11094 assert_eq!(
11095 replies[0].header.ty,
11096 FrameType::Response,
11097 "candidate catalog.update was refused: {:?}",
11098 serde_json::from_slice::<Value>(&replies[0].body).ok()
11099 );
11100 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
11101 assert_eq!(
11102 swap.registry
11103 .get_module("aft")
11104 .unwrap()
11105 .unwrap()
11106 .connection_id,
11107 INCUMBENT
11108 );
11109 }
11110 }
11111
11112 mod swap_admission {
11116 use super::*;
11117
11118 const INCUMBENT_NONCE: &str = "incumbent-nonce";
11119 const CANDIDATE_NONCE: &str = "candidate-nonce";
11120
11121 fn handler_with_incumbent(
11122 module_id: &str,
11123 reserved: bool,
11124 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
11125 let registry = Arc::new(Registry::default());
11126 let supervisor = SupervisorHandle::new();
11127 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
11128 if reserved {
11129 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
11130 }
11131 let handler =
11132 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
11133 let incumbent = handler
11134 .handle_control(
11135 ConnectionId::new(1),
11136 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
11137 )
11138 .unwrap();
11139 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
11140 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
11141 (registry, supervisor, handler)
11142 }
11143
11144 #[test]
11150 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
11151 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
11152
11153 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
11154 let replies = handler
11155 .handle_control(
11156 ConnectionId::new(connection),
11157 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
11158 )
11159 .unwrap();
11160 assert_eq!(replies[0].header.ty, FrameType::Error);
11161 assert_eq!(
11162 parse_error(&replies[0])["code"],
11163 "swap_token_invalid",
11164 "nonce {nonce:?}"
11165 );
11166 }
11167 assert!(registry.get_candidate("aft").unwrap().is_none());
11168 assert_eq!(
11169 registry.get_module("aft").unwrap().unwrap().connection_id,
11170 ConnectionId::new(1)
11171 );
11172
11173 let admitted = handler
11176 .handle_control(
11177 ConnectionId::new(4),
11178 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
11179 )
11180 .unwrap();
11181 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
11182 assert_eq!(
11183 registry
11184 .get_candidate("aft")
11185 .unwrap()
11186 .unwrap()
11187 .connection_id,
11188 ConnectionId::new(4)
11189 );
11190 assert_eq!(
11191 registry.get_module("aft").unwrap().unwrap().connection_id,
11192 ConnectionId::new(1),
11193 "the candidate must not take the active slot"
11194 );
11195 let replayed = handler
11196 .handle_control(
11197 ConnectionId::new(5),
11198 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
11199 )
11200 .unwrap();
11201 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
11202
11203 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
11207 let squatter = handler
11208 .handle_control(
11209 ConnectionId::new(6),
11210 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
11211 )
11212 .unwrap();
11213 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
11214 assert!(
11215 registry.get_module("aft").unwrap().is_none(),
11216 "a squatter took the active slot of an id being swapped"
11217 );
11218 }
11219
11220 #[test]
11225 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
11226 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
11227
11228 let replies = handler
11229 .handle_control(
11230 ConnectionId::new(2),
11231 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
11232 )
11233 .unwrap();
11234
11235 assert_eq!(
11236 replies[0].header.ty,
11237 FrameType::HelloAck,
11238 "reserved candidate refused: {:?}",
11239 serde_json::from_slice::<Value>(&replies[0].body).ok()
11240 );
11241 assert_eq!(
11242 registry
11243 .get_candidate("vault")
11244 .unwrap()
11245 .unwrap()
11246 .connection_id,
11247 ConnectionId::new(2)
11248 );
11249 }
11250
11251 #[test]
11254 fn without_an_open_swap_the_ordinary_gates_decide() {
11255 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
11256 supervisor.close_swap("vault");
11257
11258 let candidate = handler
11259 .handle_control(
11260 ConnectionId::new(2),
11261 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
11262 )
11263 .unwrap();
11264 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
11265 let duplicate = handler
11266 .handle_control(
11267 ConnectionId::new(3),
11268 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
11269 )
11270 .unwrap();
11271 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11272 assert!(registry.get_candidate("vault").unwrap().is_none());
11273 }
11274 }
11275
11276 mod scopes {
11280 use subc_protocol::scope::{
11281 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
11282 ScopeStatus,
11283 };
11284
11285 use super::*;
11286
11287 const OWNER: &str = "prefrontal-core";
11288
11289 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
11290 ScopeRecord {
11291 scope_ref: scope_ref.to_string(),
11292 scope_epoch,
11293 kind: ScopeKind::Head,
11294 parent: None,
11295 child_owners: Vec::new(),
11296 carriers: Vec::new(),
11297 attributes: Default::default(),
11298 }
11299 }
11300
11301 async fn call(
11302 handler: &ControlHandler,
11303 ctx: &RouteCtx,
11304 request: &ModuleControlRequestFromModule,
11305 ) -> Frame {
11306 let body = serde_json::to_vec(request).unwrap();
11307 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
11308 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
11309 assert_eq!(replies.len(), 1, "{replies:?}");
11310 replies.pop().unwrap()
11311 }
11312
11313 async fn sync(
11314 handler: &ControlHandler,
11315 ctx: &RouteCtx,
11316 generation: u64,
11317 scopes: Vec<ScopeRecord>,
11318 ) -> Result<ModuleControlResponseToModule, String> {
11319 let reply = call(
11320 handler,
11321 ctx,
11322 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
11323 )
11324 .await;
11325 match reply.header.ty {
11326 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
11327 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
11328 }
11329 }
11330
11331 async fn describe(
11332 handler: &ControlHandler,
11333 ctx: &RouteCtx,
11334 owner: &str,
11335 scope_ref: &str,
11336 ) -> ModuleControlResponseToModule {
11337 let reply = call(
11338 handler,
11339 ctx,
11340 &ModuleControlRequestFromModule::ScopeDescribe {
11341 owner: Principal::Reserved {
11342 module_id: owner.to_string(),
11343 },
11344 scope_ref: scope_ref.to_string(),
11345 },
11346 )
11347 .await;
11348 assert_eq!(
11349 reply.header.ty,
11350 FrameType::Response,
11351 "{:?}",
11352 parse_error(&reply)
11353 );
11354 serde_json::from_slice(&reply.body).unwrap()
11355 }
11356
11357 async fn module(
11359 handler: &ControlHandler,
11360 connection: u64,
11361 module_id: &str,
11362 nonce: Option<&str>,
11363 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11364 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
11365 hello_via_sink(
11366 handler,
11367 &ctx,
11368 &mut rx,
11369 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
11370 )
11371 .await;
11372 (ctx, rx)
11373 }
11374
11375 #[tokio::test]
11378 async fn a_client_connection_cannot_sync_or_describe() {
11379 let handler = ControlHandler::new(Arc::new(Registry::default()));
11380 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
11381 for request in [
11382 ModuleControlRequestFromModule::ScopeSync {
11383 generation: 1,
11384 scopes: vec![head("s", 1)],
11385 },
11386 ModuleControlRequestFromModule::ScopeDescribe {
11387 owner: Principal::Direct,
11388 scope_ref: "s".to_string(),
11389 },
11390 ] {
11391 let reply = call(&handler, &ctx, &request).await;
11392 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
11393 }
11394 assert!(
11395 !handler
11396 .scopes
11397 .read()
11398 .unwrap()
11399 .describe(
11400 &Principal::Reserved {
11401 module_id: OWNER.to_string()
11402 },
11403 "s"
11404 )
11405 .owner_synced
11406 );
11407 }
11408
11409 #[tokio::test]
11412 async fn a_module_without_a_supervised_launch_cannot_sync() {
11413 let handler = ControlHandler::new(Arc::new(Registry::default()));
11414 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
11415 assert_eq!(
11416 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
11417 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11418 );
11419 }
11420
11421 #[tokio::test]
11422 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
11423 let supervisor = SupervisorHandle::new();
11424 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
11425 let handler = ControlHandler::new(Arc::new(Registry::default()))
11426 .with_supervisor(supervisor.clone());
11427 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
11428 sync(&handler, &incumbent, 1, vec![head("s", 1)])
11429 .await
11430 .expect("the current launch syncs");
11431
11432 supervisor.open_swap(OWNER, "n2".to_string());
11435 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
11436 assert_eq!(
11437 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
11438 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11439 );
11440 sync(&handler, &incumbent, 2, vec![head("s", 1)])
11441 .await
11442 .expect("the serving owner syncs during the swap");
11443
11444 supervisor.close_swap(OWNER);
11447 assert_eq!(
11448 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
11449 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11450 );
11451 sync(&handler, &incumbent, 3, vec![head("s", 1)])
11452 .await
11453 .expect("the serving owner syncs after the rollback");
11454 handler.cleanup_connection(candidate.connection_id).unwrap();
11455
11456 supervisor.open_swap(OWNER, "n3".to_string());
11461 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
11462 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
11463 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
11464 .await
11465 .expect("the promoted launch takes authority");
11466 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
11467 panic!("unexpected reply {reply:?}");
11468 };
11469 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
11470 assert_eq!(
11471 sync(&handler, &incumbent, 4, Vec::new()).await,
11472 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
11473 );
11474 }
11475
11476 #[tokio::test]
11479 async fn closing_the_authority_connection_frees_sync_authority() {
11480 let supervisor = SupervisorHandle::new();
11481 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
11482 let handler = ControlHandler::new(Arc::new(Registry::default()))
11483 .with_supervisor(supervisor.clone());
11484 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
11485 sync(&handler, &first, 10, vec![head("s", 1)])
11486 .await
11487 .unwrap();
11488 handler.cleanup_connection(first.connection_id).unwrap();
11489
11490 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
11491 sync(&handler, &second, 1, vec![head("s", 1)])
11492 .await
11493 .expect("the next connection takes the released authority");
11494 }
11495
11496 #[tokio::test]
11497 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
11498 let registry = Arc::new(Registry::default());
11499 let supervisor_handle = SupervisorHandle::new();
11500 let supervisor = Supervisor::new(Arc::clone(®istry), RestartPolicy::default())
11501 .with_handle(supervisor_handle.clone())
11502 .with_daemon_incarnation("incarnation-7".to_string());
11503 supervisor
11506 .supervise_configured(
11507 ModuleSpec {
11508 module_id: OWNER.to_string(),
11509 program: PathBuf::from("/nonexistent/prefrontal-core"),
11510 args: Vec::new(),
11511 env: Vec::new(),
11512 reserved: false,
11513 reserved_prefixes: Vec::new(),
11514 protocol: ModuleProtocol::Subc,
11515 overlap: Default::default(),
11516 },
11517 false,
11518 )
11519 .unwrap();
11520 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
11521 let handler =
11522 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11523 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
11524
11525 let ModuleControlResponseToModule::ScopeDescribe {
11527 status,
11528 daemon_incarnation,
11529 owner_synced,
11530 owner_configured,
11531 scope,
11532 ..
11533 } = describe(&handler, &reader, OWNER, "s").await
11534 else {
11535 panic!("not a describe reply");
11536 };
11537 assert_eq!(status, ScopeStatus::NotLive);
11538 assert_eq!(daemon_incarnation, "incarnation-7");
11539 assert!(!owner_synced);
11540 assert!(owner_configured);
11541 assert!(scope.is_none());
11542
11543 let ModuleControlResponseToModule::ScopeDescribe {
11546 status,
11547 owner_configured,
11548 ..
11549 } = describe(&handler, &reader, "ghost", "s").await
11550 else {
11551 panic!("not a describe reply");
11552 };
11553 assert_eq!(status, ScopeStatus::NotLive);
11554 assert!(!owner_configured);
11555
11556 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
11558 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
11559 let ModuleControlResponseToModule::ScopeDescribe {
11560 status,
11561 scope_epoch,
11562 owner_synced,
11563 scope,
11564 ..
11565 } = describe(&handler, &reader, OWNER, "s").await
11566 else {
11567 panic!("not a describe reply");
11568 };
11569 assert_eq!(status, ScopeStatus::Live);
11570 assert_eq!(scope_epoch, Some(4));
11571 assert!(owner_synced);
11572 let stamp = scope.expect("a live scope carries its stamp");
11573 assert!(
11574 stamp.owner_authorized,
11575 "prefrontal-core is the default authority"
11576 );
11577 assert_eq!(stamp.kind, ScopeKind::Head);
11578 }
11579
11580 #[tokio::test]
11581 async fn scope_authority_owners_decides_owner_authorized() {
11582 let supervisor = SupervisorHandle::new();
11583 supervisor.set_spawn_nonce("broca", "b1".to_string());
11584 let handler = ControlHandler::new(Arc::new(Registry::default()))
11585 .with_supervisor(supervisor)
11586 .with_scope_authority_owners(vec!["broca".to_string()]);
11587 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
11588 let mut gated = head("s", 1);
11589 gated.attributes.agent_id = Some("agent".to_string());
11590 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
11591 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
11592 describe(&handler, &broca, "broca", "s").await
11593 else {
11594 panic!("not a describe reply");
11595 };
11596 assert!(scope.unwrap().owner_authorized);
11597 }
11598
11599 #[tokio::test]
11603 async fn scope_ops_and_the_scopes_capability_are_advertised() {
11604 let handler = ControlHandler::new(Arc::new(Registry::default()));
11605 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
11606 let ack = hello_via_sink(
11607 &handler,
11608 &ctx,
11609 &mut rx,
11610 hello_frame("m", PROTOCOL_VERSION, 1),
11611 )
11612 .await;
11613 let ack = parse_ack(&ack);
11614 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
11615 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
11616 }
11617 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
11618
11619 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
11620 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11621 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11622 let reply = handler
11623 .handle_control_frame(&client, frame)
11624 .await
11625 .unwrap()
11626 .pop()
11627 .unwrap();
11628 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11629 serde_json::from_slice(&reply.body).unwrap()
11630 else {
11631 panic!("not a server.describe reply");
11632 };
11633 assert!(
11634 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
11635 "{capabilities:?}"
11636 );
11637 }
11638
11639 const PLEXUS: &str = "plexus";
11642 const OTHER: &str = "other";
11643 const AFT: &str = "aft";
11644 const BROCA: &str = "broca";
11645 const MAGIC: &str = "magic-context";
11646
11647 fn nonce(module_id: &str) -> String {
11648 format!("nonce-{module_id}")
11649 }
11650
11651 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
11652 let (tx, rx) = mpsc::channel(64);
11653 (
11654 RouteCtx {
11655 connection_id: ConnectionId::new(connection),
11656 egress: FrameSink::new(tx),
11657 },
11658 rx,
11659 )
11660 }
11661
11662 struct Rig {
11666 handler: ControlHandler,
11667 forwarding: Arc<ForwardingTable>,
11668 owner: RouteCtx,
11669 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11670 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
11671 generation: u64,
11672 next_connection: u64,
11673 _supervisor: Supervisor,
11674 }
11675
11676 async fn rig() -> Rig {
11677 let registry = Arc::new(Registry::default());
11678 let forwarding = Arc::new(ForwardingTable::default());
11679 let supervisor_handle = SupervisorHandle::new();
11680 let supervisor = Supervisor::new(Arc::clone(®istry), RestartPolicy::default())
11681 .with_handle(supervisor_handle.clone());
11682 supervisor
11683 .supervise_configured(
11684 ModuleSpec {
11685 module_id: OWNER.to_string(),
11686 program: PathBuf::from("/nonexistent/prefrontal-core"),
11687 args: Vec::new(),
11688 env: Vec::new(),
11689 reserved: false,
11690 reserved_prefixes: Vec::new(),
11691 protocol: ModuleProtocol::Subc,
11692 overlap: Default::default(),
11693 },
11694 false,
11695 )
11696 .unwrap();
11697 for module_id in [OWNER, AFT, BROCA, MAGIC] {
11698 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
11699 }
11700 let handler =
11701 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11702 .with_supervisor(supervisor_handle);
11703 let (owner, mut owner_rx) = wide_ctx(1);
11704 hello_via_sink(
11705 &handler,
11706 &owner,
11707 &mut owner_rx,
11708 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
11709 )
11710 .await;
11711 let mut modules = BTreeMap::new();
11712 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
11713 let (ctx, mut rx) = wide_ctx(connection);
11714 hello_via_sink(
11715 &handler,
11716 &ctx,
11717 &mut rx,
11718 hello_frame(module_id, PROTOCOL_VERSION, connection),
11719 )
11720 .await;
11721 modules.insert(module_id.to_string(), (ctx, rx));
11722 }
11723 Rig {
11724 handler,
11725 forwarding,
11726 owner,
11727 _owner_rx: owner_rx,
11728 modules,
11729 generation: 0,
11730 next_connection: 100,
11731 _supervisor: supervisor,
11732 }
11733 }
11734
11735 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
11736 ScopeCarrier {
11737 principal: Principal::Reserved {
11738 module_id: module_id.to_string(),
11739 },
11740 targets: targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()),
11741 }
11742 }
11743
11744 fn session(scope_epoch: u64) -> ScopeRecord {
11747 let mut record = head("s", scope_epoch);
11748 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
11749 record.attributes.agent_id = Some("agent-1".to_string());
11750 record.attributes.delegates = true;
11751 record
11752 }
11753
11754 impl Rig {
11755 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
11756 self.generation += 1;
11757 sync(&self.handler, &self.owner, self.generation, scopes)
11758 .await
11759 .expect("the owner's sync is accepted");
11760 }
11761
11762 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
11763 ScopeSelector {
11764 owner: Principal::Reserved {
11765 module_id: OWNER.to_string(),
11766 },
11767 scope_ref: scope_ref.to_string(),
11768 scope_epoch,
11769 }
11770 }
11771
11772 fn open_frame(
11773 &mut self,
11774 opener: Option<&str>,
11775 target: &str,
11776 scope: Option<ScopeSelector>,
11777 ) -> (
11778 RouteCtx,
11779 mpsc::Receiver<crate::router::OutboundFrame>,
11780 Frame,
11781 ) {
11782 self.next_connection += 1;
11783 let (ctx, rx) = wide_ctx(self.next_connection);
11784 let root = unique_project_root("scoped-open");
11785 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
11786 target: RouteTarget::ToolProvider {
11787 module_id: target.to_string(),
11788 },
11789 identity: BindIdentity::new(
11790 root.path().to_path_buf(),
11791 "unit".to_string(),
11792 "session".to_string(),
11793 ),
11794 consumer_identity: opener.map(|module_id| ConsumerIdentity {
11795 module_id: module_id.to_string(),
11796 launch_nonce: nonce(module_id),
11797 }),
11798 consumer_capabilities: None,
11799 admission_facts: None,
11800 scope,
11801 })
11802 .unwrap();
11803 let frame = Frame::build(
11804 FrameType::Request,
11805 control_flags(),
11806 0,
11807 0,
11808 self.next_connection,
11809 body,
11810 )
11811 .unwrap();
11812 (ctx, rx, frame)
11813 }
11814
11815 async fn refused(
11817 &mut self,
11818 opener: Option<&str>,
11819 target: &str,
11820 scope: Option<ScopeSelector>,
11821 ) -> String {
11822 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
11823 let replies = self
11824 .handler
11825 .handle_control_frame(&ctx, frame)
11826 .await
11827 .unwrap();
11828 assert_eq!(replies.len(), 1, "{replies:?}");
11829 assert_eq!(replies[0].header.ty, FrameType::Error);
11830 let (_, module_rx) = self.modules.get_mut(target).unwrap();
11831 assert!(
11832 module_rx.try_recv().is_err(),
11833 "a refused open relays nothing"
11834 );
11835 parse_error(&replies[0])["code"]
11836 .as_str()
11837 .unwrap()
11838 .to_string()
11839 }
11840
11841 async fn relayed(
11843 &mut self,
11844 opener: Option<&str>,
11845 target: &str,
11846 scope: Option<ScopeSelector>,
11847 ) -> Relayed {
11848 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
11849 let handler = self.handler.clone();
11850 let task_ctx = ctx.clone();
11851 let task = tokio::spawn(async move {
11852 handler
11853 .handle_control_frame(&task_ctx, frame)
11854 .await
11855 .unwrap()
11856 });
11857 let (_, module_rx) = self.modules.get_mut(target).unwrap();
11858 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
11859 .await
11860 .expect("the target receives the relayed route.bind")
11861 .unwrap()
11862 .frame;
11863 Relayed {
11864 target: target.to_string(),
11865 client: ctx,
11866 client_rx: rx,
11867 task,
11868 bind,
11869 }
11870 }
11871
11872 async fn ack(&self, relayed: &Relayed) {
11873 let (module, _) = &self.modules[&relayed.target];
11874 self.handler
11875 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
11876 .await
11877 .unwrap();
11878 }
11879
11880 async fn bound(
11882 &mut self,
11883 opener: Option<&str>,
11884 target: &str,
11885 scope: Option<ScopeSelector>,
11886 ) -> Bound {
11887 let relayed = self.relayed(opener, target, scope).await;
11888 self.ack(&relayed).await;
11889 let Relayed {
11890 target,
11891 client,
11892 mut client_rx,
11893 task,
11894 bind,
11895 } = relayed;
11896 assert!(
11897 task.await.unwrap().is_empty(),
11898 "the open is answered by commit"
11899 );
11900 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
11901 Bound {
11902 target,
11903 client,
11904 client_rx,
11905 channel,
11906 epoch,
11907 bind,
11908 }
11909 }
11910
11911 fn live(&self, route: &Bound) -> bool {
11912 matches!(
11913 self.forwarding
11914 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
11915 .unwrap(),
11916 DataRoute::Client(DataRouteState::Bound(_))
11917 )
11918 }
11919 }
11920
11921 struct Relayed {
11922 target: String,
11923 client: RouteCtx,
11924 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11925 task: tokio::task::JoinHandle<Vec<Frame>>,
11926 bind: Frame,
11927 }
11928
11929 struct Bound {
11930 target: String,
11931 client: RouteCtx,
11932 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
11933 channel: u16,
11934 epoch: u32,
11935 bind: Frame,
11936 }
11937
11938 impl Bound {
11939 fn closed_reason(&mut self) -> RouteCloseReason {
11942 let mut reason = None;
11943 let mut goodbye = false;
11944 while let Ok(outbound) = self.client_rx.try_recv() {
11945 let frame = outbound.frame;
11946 match frame.header.ty {
11947 FrameType::Goodbye => {
11948 assert_eq!(
11949 (frame.header.channel, frame.header.epoch),
11950 (self.channel, self.epoch)
11951 );
11952 goodbye = true;
11953 }
11954 FrameType::Push => {
11955 let ClientControlPush::RouteClosed {
11956 reason: r,
11957 module_id,
11958 ..
11959 } = serde_json::from_slice(&frame.body).unwrap()
11960 else {
11961 panic!("unexpected push");
11962 };
11963 assert_eq!(module_id, self.target);
11964 reason = Some(r);
11965 }
11966 other => panic!("unexpected frame {other:?}"),
11967 }
11968 }
11969 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
11970 reason.expect("the client is told why the route closed")
11971 }
11972
11973 fn untouched(&mut self) -> bool {
11974 self.client_rx.try_recv().is_err()
11975 }
11976
11977 fn stamp(&self) -> Option<ScopeStamp> {
11978 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
11979 ModuleControlRequest::RouteBind { scope, .. } => scope,
11980 other => panic!("expected a route.bind, got {other:?}"),
11981 }
11982 }
11983 }
11984
11985 #[tokio::test]
11986 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
11987 ) {
11988 let mut rig = rig().await;
11989 let mut record = session(1);
11990 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
11991 record.child_owners = vec![Principal::Reserved {
11992 module_id: MAGIC.to_string(),
11993 }];
11994 rig.sync(vec![record]).await;
11995 let scope = || Some(rig_selector("s", Some(1)));
11996
11997 rig.bound(Some(OWNER), PLEXUS, scope()).await;
12000 rig.bound(Some(AFT), OTHER, scope()).await;
12001 rig.bound(Some(BROCA), PLEXUS, scope()).await;
12002
12003 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
12007 assert_eq!(
12008 rig.refused(opener, target, scope()).await,
12009 error_codes::SCOPE_NOT_CARRIER,
12010 "{opener:?} -> {target}"
12011 );
12012 }
12013 }
12014
12015 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
12016 ScopeSelector {
12017 owner: Principal::Reserved {
12018 module_id: OWNER.to_string(),
12019 },
12020 scope_ref: scope_ref.to_string(),
12021 scope_epoch,
12022 }
12023 }
12024
12025 #[tokio::test]
12026 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
12027 let mut rig = rig().await;
12028 rig.sync(vec![session(1)]).await;
12029 for opener in [OWNER, AFT] {
12030 assert_eq!(
12031 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
12032 .await,
12033 error_codes::SCOPE_EPOCH_REQUIRED,
12034 "{opener}"
12035 );
12036 }
12037 }
12038
12039 #[tokio::test]
12040 async fn admission_separates_not_synced_not_live_and_ended() {
12041 let mut rig = rig().await;
12042 let code = rig
12044 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12045 .await;
12046 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
12047 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
12048
12049 let ghost = ScopeSelector {
12051 owner: Principal::Reserved {
12052 module_id: "ghost".to_string(),
12053 },
12054 scope_ref: "s".to_string(),
12055 scope_epoch: Some(1),
12056 };
12057 assert_eq!(
12058 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
12059 error_codes::SCOPE_NOT_LIVE
12060 );
12061
12062 rig.sync(vec![session(2)]).await;
12063 assert_eq!(
12064 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
12065 .await,
12066 error_codes::SCOPE_NOT_LIVE
12067 );
12068 for epoch in [1, 3] {
12069 assert_eq!(
12070 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
12071 .await,
12072 error_codes::SCOPE_ENDED,
12073 "epoch {epoch}"
12074 );
12075 }
12076 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
12078 .await;
12079 }
12080
12081 #[tokio::test]
12082 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
12083 let mut rig = rig().await;
12084 rig.sync(vec![session(1)]).await;
12085 let route = rig
12086 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12087 .await;
12088 let stamp = route.stamp().expect("a scoped bind carries the stamp");
12089 assert_eq!(stamp.scope_ref, "s");
12090 assert_eq!(stamp.scope_epoch, 1);
12091 assert_eq!(stamp.kind, ScopeKind::Head);
12092 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
12093 assert!(stamp.attributes.delegates);
12094 assert!(stamp.owner_authorized);
12095 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
12096 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
12097
12098 let (broca, mut broca_rx) = wide_ctx(50);
12101 hello_via_sink(
12102 &rig.handler,
12103 &broca,
12104 &mut broca_rx,
12105 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
12106 )
12107 .await;
12108 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
12109 .await
12110 .unwrap();
12111 let own = ScopeSelector {
12112 owner: Principal::Reserved {
12113 module_id: BROCA.to_string(),
12114 },
12115 scope_ref: "b".to_string(),
12116 scope_epoch: Some(1),
12117 };
12118 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
12119 assert!(!route.stamp().unwrap().owner_authorized);
12120 }
12121
12122 #[tokio::test]
12127 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
12128 let mut rig = rig().await;
12129 rig.sync(vec![session(1)]).await;
12130 let mut cotenant = rig
12131 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
12132 .await;
12133
12134 let mut changed = session(1);
12135 changed.child_owners.push(Principal::Reserved {
12136 module_id: MAGIC.to_string(),
12137 });
12138 let mut ended = None;
12139 for (code, next) in [
12140 (error_codes::SCOPE_CHANGED, vec![changed]),
12141 (error_codes::SCOPE_ENDED, Vec::new()),
12142 ] {
12143 let relayed = rig
12144 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12145 .await;
12146 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
12147 ended = Some(next.is_empty());
12148 rig.sync(next).await;
12149 rig.ack(&relayed).await;
12150 let replies = relayed.task.await.unwrap();
12151 assert_eq!(replies.len(), 1, "{replies:?}");
12152 assert_eq!(parse_error(&replies[0])["code"], code);
12153 assert_eq!(
12154 subc_protocol::error_codes::is_retryable_route_open(code),
12155 code == error_codes::SCOPE_CHANGED
12156 );
12157 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
12158 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12160 let mut goodbyes = Vec::new();
12163 while let Ok(outbound) = plexus_rx.try_recv() {
12164 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
12165 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
12166 }
12167 assert!(
12168 goodbyes.contains(&(bind_channel, bind_epoch)),
12169 "{goodbyes:?}"
12170 );
12171 assert!(rig
12172 .handler
12173 .registry
12174 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
12175 .unwrap()
12176 .is_some());
12177 }
12178 assert_eq!(ended, Some(true));
12179 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
12182 }
12183
12184 #[tokio::test]
12187 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
12188 struct Case {
12189 name: &'static str,
12190 change: fn(&mut ScopeRecord),
12191 closed: [Option<RouteCloseReason>; 4],
12194 }
12195 use RouteCloseReason::*;
12196 let cases = [
12197 Case {
12198 name: "a carrier entry removed",
12199 change: |r| {
12200 r.carriers.retain(|c| {
12201 c.principal
12202 != Principal::Reserved {
12203 module_id: AFT.to_string(),
12204 }
12205 })
12206 },
12207 closed: [None, Some(ScopeCarrierRemoved), None, None],
12208 },
12209 Case {
12210 name: "a target removed from a carrier",
12211 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
12212 closed: [None, None, None, Some(ScopeCarrierRemoved)],
12213 },
12214 Case {
12215 name: "a bare carrier narrowed to targets",
12216 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
12217 closed: [None, Some(ScopeCarrierRemoved), None, None],
12218 },
12219 Case {
12220 name: "delegates turned off",
12221 change: |r| r.attributes.delegates = false,
12222 closed: [Some(ScopeDelegationChanged); 4],
12223 },
12224 Case {
12225 name: "agent_id changed",
12226 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
12227 closed: [Some(ScopeDelegationChanged); 4],
12228 },
12229 Case {
12230 name: "a carrier added, child owners changed, the record re-sent",
12231 change: |r| {
12232 r.carriers.push(carrier(MAGIC, None));
12233 r.child_owners.push(Principal::Reserved {
12234 module_id: MAGIC.to_string(),
12235 });
12236 },
12237 closed: [None; 4],
12238 },
12239 Case {
12240 name: "a target added",
12241 change: |r| {
12242 r.carriers[1]
12243 .targets
12244 .as_mut()
12245 .unwrap()
12246 .push("third".to_string())
12247 },
12248 closed: [None; 4],
12249 },
12250 Case {
12251 name: "delegates turned on",
12252 change: |r| r.attributes.delegates = true,
12253 closed: [None; 4],
12254 },
12255 ];
12256 for case in cases {
12257 let mut rig = rig().await;
12258 rig.sync(vec![session(1)]).await;
12259 let scope = || Some(rig_selector("s", Some(1)));
12260 let mut routes = [
12261 rig.bound(Some(OWNER), PLEXUS, scope()).await,
12262 rig.bound(Some(AFT), PLEXUS, scope()).await,
12263 rig.bound(Some(BROCA), PLEXUS, scope()).await,
12264 rig.bound(Some(BROCA), OTHER, scope()).await,
12265 ];
12266 let mut record = session(1);
12267 (case.change)(&mut record);
12268 rig.sync(vec![record]).await;
12269 for (index, expected) in case.closed.iter().enumerate() {
12270 let route = &mut routes[index];
12271 match expected {
12272 Some(reason) => {
12273 assert!(!rig.live(route), "{}: route {index} still live", case.name);
12274 assert_eq!(
12275 route.closed_reason(),
12276 *reason,
12277 "{}: route {index}",
12278 case.name
12279 );
12280 }
12281 None => {
12282 assert!(rig.live(route), "{}: route {index} closed", case.name);
12283 assert!(
12284 route.untouched(),
12285 "{}: route {index} was told something",
12286 case.name
12287 );
12288 }
12289 }
12290 }
12291 }
12292 }
12293
12294 #[tokio::test]
12295 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
12296 for next in [Vec::new(), vec![session(2)]] {
12298 let mut rig = rig().await;
12299 rig.sync(vec![session(1)]).await;
12300 let mut route = rig
12301 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12302 .await;
12303 rig.sync(next).await;
12304 assert!(!rig.live(&route));
12305 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
12306 }
12307
12308 let mut rig = rig().await;
12311 let mut child = session(1);
12312 child.scope_ref = "child".to_string();
12313 child.kind = ScopeKind::Worker;
12314 child.parent = Some(ScopeParent {
12315 owner: Principal::Reserved {
12316 module_id: OWNER.to_string(),
12317 },
12318 scope_ref: "s".to_string(),
12319 scope_epoch: 1,
12320 });
12321 rig.sync(vec![session(1), child.clone()]).await;
12322 let mut child_route = rig
12323 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
12324 .await;
12325 assert_eq!(
12326 child_route.stamp().unwrap().parent_state,
12327 Some(ParentState::Linked)
12328 );
12329 rig.sync(vec![child]).await;
12330 assert!(!rig.live(&child_route));
12331 assert_eq!(
12332 child_route.closed_reason(),
12333 RouteCloseReason::ScopeParentEnded
12334 );
12335 }
12336
12337 #[tokio::test]
12338 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
12339 ) {
12340 let mut rig = rig().await;
12341 rig.sync(vec![session(1)]).await;
12342 let mut route = rig
12343 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12344 .await;
12345 let before = rig.forwarding.published_scope_tag(OWNER, "s");
12346 rig.sync(vec![session(1)]).await;
12347 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
12348 assert!(rig.live(&route) && route.untouched());
12349
12350 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
12354 .forwarding
12355 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
12356 .unwrap()
12357 else {
12358 panic!("the route is bound");
12359 };
12360 binding.flow.acquire_tagged(9, false).await.unwrap();
12361 let mut widened = session(1);
12362 widened.carriers.push(carrier(MAGIC, None));
12363 rig.sync(vec![widened]).await;
12364 assert!(rig.live(&route) && route.untouched());
12365 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12366 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
12367 assert_eq!(binding.flow.in_flight(), 1);
12370 binding
12371 .flow
12372 .acquire_tagged(10, false)
12373 .await
12374 .expect("the flow is still open");
12375 }
12376
12377 #[tokio::test]
12380 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
12381 let mut rig = rig().await;
12382 rig.sync(vec![session(1)]).await;
12383 let mut on_incumbent = rig
12384 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
12385 .await;
12386
12387 let (candidate, _candidate_rx) = wide_ctx(9);
12390 let registration = rig
12391 .handler
12392 .registry
12393 .register_candidate_with_control_ops(
12394 manifest(PLEXUS, PROTOCOL_VERSION),
12395 PROTOCOL_VERSION,
12396 candidate.connection_id,
12397 module_baseline_control_ops(),
12398 )
12399 .unwrap();
12400 rig.forwarding
12401 .register_candidate_module_connection(
12402 candidate.connection_id,
12403 PLEXUS.to_string(),
12404 PROTOCOL_VERSION,
12405 manifest_concurrency(®istration.manifest),
12406 candidate.egress.clone(),
12407 )
12408 .unwrap();
12409 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
12410 rig.handler
12411 .registry
12412 .promote_candidate(PLEXUS)
12413 .unwrap()
12414 .unwrap();
12415 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
12416
12417 rig.sync(Vec::new()).await;
12418 assert!(!rig.live(&on_incumbent));
12419 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
12420 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
12421 let goodbye = incumbent_rx
12422 .try_recv()
12423 .expect("the superseded endpoint is told")
12424 .frame;
12425 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12426 }
12427 }
12428}
12429
12430#[cfg(test)]
12431mod concurrency_default_exposure_tests {
12432 use super::*;
12433
12434 fn hello_body(role_json: &str) -> Vec<u8> {
12435 format!(
12436 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
12437 )
12438 .into_bytes()
12439 }
12440
12441 fn manifest_from(body: &[u8]) -> ModuleManifest {
12442 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
12443 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
12444 .expect("manifest parses")
12445 }
12446
12447 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
12448
12449 #[test]
12450 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
12451 let body = hello_body(&format!(
12452 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
12453 ));
12454 let manifest = manifest_from(&body);
12455 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
12458 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
12459 }
12460
12461 #[test]
12462 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
12463 let body = hello_body(&format!(
12464 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
12465 ));
12466 let manifest = manifest_from(&body);
12467 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
12468 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
12469 }
12470
12471 #[test]
12472 fn non_management_roles_are_never_reported() {
12473 let body = hello_body(
12474 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
12475 );
12476 let manifest = manifest_from(&body);
12477 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
12478 }
12479}