1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_event_declarations,
23 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
24 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
25 },
26 scope::{
27 ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_DESCRIBE_OP, SCOPE_SYNC_OP,
29 },
30 session::{
31 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
32 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
33 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
34 ROLE_VERSIONS_FIELD,
35 },
36 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
37 Priority, RouteTarget, PROTOCOL_VERSION,
38};
39use tokio::time::{timeout_at, Instant};
40use tracing::{debug, info, warn};
41
42use crate::{
43 capability_requirements::{
44 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
45 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
46 RuntimeModule,
47 },
48 daemon_config::RestartRequiredSection,
49 forwarding::{
50 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
51 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
52 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
53 },
54 observability::{
55 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
56 },
57 provenance::{
58 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
59 },
60 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
61 router::{RouteCtx, RouterError},
62 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
63 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
64 stderr_tail::{CaptureState, TailEntry},
65 supervise::{
66 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
67 SupervisorHandle, SwapHelloAdmission,
68 },
69 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
70};
71
72pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
78
79const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
80const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
81const CAP_PING_PONG: &str = "ping_pong_v1";
82const CAP_SESSION_ATTACH: &str = "session_attach_v1";
83const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
84
85const SUBC_CONTROL_OPS: &[&str] = &[
86 ops::SERVER_DESCRIBE,
87 ops::CATALOG_LIST,
88 ops::ROUTE_OPEN,
89 ops::ROUTE_POLL,
90 ops::ROUTE_CLOSING,
91 ops::ROUTE_CLOSED,
92 ops::SUPERVISOR_LIST,
93 ops::SUPERVISOR_RESTART,
94 ops::SUPERVISOR_SWAP,
95 ops::SUPERVISOR_RELOAD,
96 ops::SUPERVISOR_RESCAN,
97 ops::SUPERVISOR_RELEASE_RESERVED,
98 ops::SUPERVISOR_SET_ENABLED,
99 ops::SUPERVISOR_HEALTH_PROBE,
100 ops::SUPERVISOR_HEALTH,
101 ops::SUPERVISOR_STDERR_TAIL,
102 ops::SUPERVISOR_TERMINALS,
103 ops::SUPERVISOR_ROUTES,
104 ops::SUPERVISOR_PROVENANCE,
105 ops::SUPERVISOR_SPAWN_SNAPSHOT,
106 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
107];
108
109const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
110 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
111 "supervisor.live_roots",
112 SCOPE_SYNC_OP,
113 SCOPE_DESCRIBE_OP,
114 "operator.confirm",
115];
116
117const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
121
122const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
123
124pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
132
133pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
148
149pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
167
168const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
169const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
170
171fn reload_verdict(
172 configured: &Path,
173 spawned_from: Option<&Path>,
174 image: subc_control::RunningImageAgreement,
175) -> PendingReloadVerdict {
176 let path = match spawned_from {
177 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
178 Some(spawned_from) => ReloadPathAgreement::Mismatch {
179 configured: configured.to_path_buf(),
180 spawned_from: spawned_from.to_path_buf(),
181 },
182 None => ReloadPathAgreement::Unavailable {
183 reason: if matches!(
184 image,
185 subc_control::RunningImageAgreement::Unavailable {
186 reason: subc_control::RunningImageUnavailableReason::NotRunning
187 }
188 ) {
189 ReloadPathUnavailableReason::NotRunning
190 } else {
191 ReloadPathUnavailableReason::SpawnedPathUnavailable
192 },
193 },
194 };
195 PendingReloadVerdict { path, image }
196}
197
198#[derive(Clone)]
199struct DaemonProvenanceFacts {
200 build: DaemonBuildProvenance,
201 pid: Option<u32>,
202 started_at_ms: Option<u64>,
203 start_clock: Option<crate::clock::StartClock>,
204 executable_path: Option<PathBuf>,
205 executable_identity: Option<SpawnedFileIdentity>,
206 process_start_time: Option<u64>,
207 probe: ExecutableIdentityProbe,
208}
209
210impl Default for DaemonProvenanceFacts {
211 fn default() -> Self {
212 Self {
213 build: DaemonBuildProvenance {
214 build_git_sha: None,
215 build_lock_digest: None,
216 },
217 pid: None,
218 started_at_ms: None,
219 start_clock: None,
220 executable_path: None,
221 executable_identity: None,
222 process_start_time: None,
223 probe: ExecutableIdentityProbe::default(),
224 }
225 }
226}
227
228#[derive(Debug, Clone)]
229struct SupervisorRescanContext {
230 supervisor: Supervisor,
231 config_path: PathBuf,
232 configured_port: Option<u16>,
233 storage_config: Option<crate::daemon_config::StorageConfig>,
234 admission_facts_carrier_module_id: Option<String>,
235 admission_facts_targets: Option<Vec<String>>,
236 scope_authority_owners: Vec<String>,
237}
238
239const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
260 "reloading",
261 "supervisor_not_live",
262 "registration_not_active",
263 "no_forwarding_connection",
264 "relay_send_failed",
265];
266
267#[derive(Clone)]
269pub struct ControlHandler {
270 registry: Arc<Registry>,
271 forwarding: Arc<ForwardingTable>,
272 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
273 supervisor: SupervisorHandle,
274 subc_capabilities: Arc<[String]>,
275 route_bind_relay_timeout: Duration,
279 route_bind_relay_timeouts: BTreeMap<String, Duration>,
283 route_bind_breakers: RouteBindBreakers,
286 route_bind_concurrency: RouteBindConcurrency,
289 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
294 route_bind_breaker_threshold: u32,
296 route_bind_breaker_cooldown: Duration,
298 health_probe_timeout: Duration,
299 storage_config: Option<crate::daemon_config::StorageConfig>,
302 machine_id: Option<crate::machine_id::MachineId>,
306 admission_facts_carrier_module_id: Option<String>,
307 admission_facts_targets: Option<Vec<String>>,
308 scopes: Arc<RwLock<ScopeTable>>,
312 scope_authority_owners: Vec<String>,
315 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
318 rescan: Option<SupervisorRescanContext>,
319 connected_clients: ConnectedClients,
320 counters: DaemonCounters,
321 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
322 daemon_provenance: DaemonProvenanceFacts,
323 #[cfg(test)]
324 control_dispatch_delay: Option<Duration>,
325 #[cfg(test)]
326 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
327}
328
329impl fmt::Debug for ControlHandler {
330 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
331 f.debug_struct("ControlHandler")
332 .field("registry", &self.registry)
333 .field("forwarding", &self.forwarding)
334 .field("process_liveness", &self.process_liveness.is_some())
335 .field("supervisor", &self.supervisor)
336 .field("subc_capabilities", &self.subc_capabilities)
337 .finish()
338 }
339}
340
341struct RouteOpenRequest {
342 target: RouteTarget,
343 identity: BindIdentity,
344 consumer_identity: Option<ConsumerIdentity>,
345 consumer_capabilities: Option<Vec<String>>,
346 role_versions: Option<BTreeMap<String, String>>,
347 admission_facts: Option<serde_json::Value>,
348 scope: Option<ScopeSelector>,
349}
350
351struct RouteBindReservationGuard {
352 forwarding: Arc<ForwardingTable>,
353 endpoint: ModuleEndpointId,
354 relay_corr: u64,
355 armed: bool,
356}
357
358struct ModuleControlRpcGuard {
359 forwarding: Arc<ForwardingTable>,
360 endpoint: ModuleEndpointId,
361 corr: u64,
362 armed: bool,
363}
364
365impl ModuleControlRpcGuard {
366 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
367 Self {
368 forwarding,
369 endpoint,
370 corr,
371 armed: true,
372 }
373 }
374
375 fn disarm(&mut self) {
376 self.armed = false;
377 }
378}
379
380impl Drop for ModuleControlRpcGuard {
381 fn drop(&mut self) {
382 if self.armed {
383 let _ = self
384 .forwarding
385 .cancel_module_control_rpc(self.endpoint, self.corr);
386 }
387 }
388}
389
390impl RouteBindReservationGuard {
391 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
392 Self {
393 forwarding,
394 endpoint,
395 relay_corr,
396 armed: true,
397 }
398 }
399
400 fn release_and_disarm(&mut self) {
401 if !self.armed {
402 return;
403 }
404 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
405 self.endpoint,
406 self.relay_corr,
407 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
408 ) {
409 send_goodbye_target_best_effort(
410 &self.forwarding.counters(),
411 &target,
412 "canceled route.bind",
413 );
414 }
415 self.armed = false;
416 }
417
418 fn disarm(&mut self) {
419 self.armed = false;
420 }
421}
422
423impl Drop for RouteBindReservationGuard {
424 fn drop(&mut self) {
425 self.release_and_disarm();
426 }
427}
428
429#[derive(Debug, Clone, Default)]
459pub(crate) struct RouteBindBreakers {
460 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
461}
462
463#[derive(Debug, Clone, Default)]
464pub(crate) struct RouteBindConcurrency {
465 modules: Arc<Mutex<HashMap<String, usize>>>,
466}
467
468struct RouteBindConcurrencyGuard {
469 concurrency: RouteBindConcurrency,
470 module_id: String,
471}
472
473impl RouteBindConcurrency {
474 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
477 let mut modules = self
478 .modules
479 .lock()
480 .expect("route.bind concurrency mutex poisoned");
481 let in_flight = modules.entry(module_id.to_string()).or_default();
482 if *in_flight >= limit {
483 return Err(*in_flight);
484 }
485 *in_flight += 1;
486 Ok(RouteBindConcurrencyGuard {
487 concurrency: self.clone(),
488 module_id: module_id.to_string(),
489 })
490 }
491}
492
493impl Drop for RouteBindConcurrencyGuard {
494 fn drop(&mut self) {
495 let mut modules = self
496 .concurrency
497 .modules
498 .lock()
499 .expect("route.bind concurrency mutex poisoned");
500 let remove = {
501 let in_flight = modules
502 .get_mut(&self.module_id)
503 .expect("admitted route.bind has a concurrency entry");
504 *in_flight -= 1;
505 *in_flight == 0
506 };
507 if remove {
508 modules.remove(&self.module_id);
509 }
510 }
511}
512
513#[derive(Debug, Default)]
514struct ModuleBreakerState {
515 consecutive_timeouts: u32,
517 cooldown_until: Option<Instant>,
520 probe_in_flight: Option<Arc<()>>,
525}
526
527enum RouteBindAdmission<'a> {
529 Admitted {
530 guard: RouteBindBreakerGuard<'a>,
531 probe: bool,
534 },
535 Refused {
536 consecutive_timeouts: u32,
537 retry_in: Duration,
541 probe_in_flight: bool,
542 },
543}
544
545struct RouteBindBreakerGuard<'a> {
552 breakers: RouteBindBreakers,
553 module_id: &'a str,
554 probe_token: Option<Arc<()>>,
555 settled: bool,
556}
557
558impl RouteBindBreakerGuard<'_> {
559 fn record_accepted(&mut self) -> bool {
563 self.settled = true;
564 self.breakers.record_accepted(self.module_id)
565 }
566
567 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
570 self.settled = true;
571 self.breakers.record_timeout(
572 self.module_id,
573 self.probe_token.as_ref(),
574 threshold,
575 cooldown,
576 )
577 }
578
579 fn record_inconclusive(&mut self) {
588 self.settled = true;
589 self.breakers
590 .record_inconclusive(self.module_id, self.probe_token.as_ref());
591 }
592}
593
594impl Drop for RouteBindBreakerGuard<'_> {
595 fn drop(&mut self) {
596 if !self.settled {
597 self.breakers
598 .record_inconclusive(self.module_id, self.probe_token.as_ref());
599 }
600 }
601}
602
603struct BreakerOpened {
607 consecutive_timeouts: u32,
608 reopened_after_probe: bool,
611}
612
613impl RouteBindBreakers {
614 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
615 self.modules
616 .lock()
617 .expect("route.bind breaker mutex poisoned")
618 }
619
620 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
623 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
624 probe: probe_token.is_some(),
625 guard: RouteBindBreakerGuard {
626 breakers: self.clone(),
627 module_id,
628 probe_token,
629 settled: false,
630 },
631 };
632
633 let mut modules = self.lock();
634 let Some(state) = modules.get_mut(module_id) else {
635 return admitted(None);
636 };
637 let Some(cooldown_until) = state.cooldown_until else {
638 return admitted(None);
639 };
640 if state.probe_in_flight.is_some() {
641 return RouteBindAdmission::Refused {
642 consecutive_timeouts: state.consecutive_timeouts,
643 retry_in: Duration::ZERO,
644 probe_in_flight: true,
645 };
646 }
647 let now = Instant::now();
648 if now < cooldown_until {
649 return RouteBindAdmission::Refused {
650 consecutive_timeouts: state.consecutive_timeouts,
651 retry_in: cooldown_until - now,
652 probe_in_flight: false,
653 };
654 }
655 let token = Arc::new(());
656 state.probe_in_flight = Some(Arc::clone(&token));
657 admitted(Some(token))
658 }
659
660 fn record_accepted(&self, module_id: &str) -> bool {
661 self.lock()
662 .remove(module_id)
663 .is_some_and(|state| state.cooldown_until.is_some())
664 }
665
666 fn record_timeout(
667 &self,
668 module_id: &str,
669 probe_token: Option<&Arc<()>>,
670 threshold: u32,
671 cooldown: Duration,
672 ) -> Option<BreakerOpened> {
673 let mut modules = self.lock();
674 let state = modules.entry(module_id.to_string()).or_default();
675 let was_open = state.cooldown_until.is_some();
676 let was_probe = Self::owns_probe(state, probe_token);
677 if was_probe {
678 state.probe_in_flight = None;
679 }
680 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
681 if state.consecutive_timeouts < threshold {
682 return None;
683 }
684 state.cooldown_until = Some(Instant::now() + cooldown);
685 Some(BreakerOpened {
686 consecutive_timeouts: state.consecutive_timeouts,
687 reopened_after_probe: was_open && was_probe,
688 })
689 }
690
691 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
692 state
696 .probe_in_flight
697 .as_ref()
698 .zip(token)
699 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
700 }
701
702 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
703 if let Some(state) = self.lock().get_mut(module_id) {
704 if Self::owns_probe(state, probe_token) {
705 state.probe_in_flight = None;
706 }
707 }
708 }
709
710 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
725 self.lock()
726 .remove(module_id)
727 .map(|state| state.consecutive_timeouts)
728 .filter(|discarded| *discarded > 0)
729 }
730
731 fn open_snapshot(&self) -> Option<serde_json::Value> {
738 let now = Instant::now();
739 let modules = self.lock();
740 let open = modules
741 .iter()
742 .filter_map(|(module_id, state)| {
743 let cooldown_until = state.cooldown_until?;
744 Some((
745 module_id.clone(),
746 serde_json::json!({
747 "consecutive_timeouts": state.consecutive_timeouts,
748 "cooldown_remaining_ms":
749 cooldown_until.saturating_duration_since(now).as_millis() as u64,
750 "probe_in_flight": state.probe_in_flight.is_some(),
751 }),
752 ))
753 })
754 .collect::<serde_json::Map<String, serde_json::Value>>();
755 (!open.is_empty()).then_some(serde_json::Value::Object(open))
756 }
757}
758
759impl ControlHandler {
760 pub fn new(registry: Arc<Registry>) -> Self {
761 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
762 }
763
764 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
765 let counters = forwarding.counters();
766 let route_bind_breakers = forwarding.route_bind_breakers();
770 let route_bind_concurrency = forwarding.route_bind_concurrency();
771 let route_outages = forwarding.route_outages();
772 Self {
773 registry,
774 forwarding,
775 process_liveness: None,
776 supervisor: SupervisorHandle::new(),
777 subc_capabilities: Arc::from([
778 CAP_MANIFEST_REGISTRATION.to_string(),
779 CAP_CHANNEL_LIFECYCLE.to_string(),
780 CAP_PING_PONG.to_string(),
781 CAP_SESSION_ATTACH.to_string(),
782 CAP_ADMISSION_FACTS_RELAY.to_string(),
783 CAP_SCOPES_V1.to_string(),
784 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
785 ]),
786 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
787 route_bind_relay_timeouts: BTreeMap::new(),
788 route_bind_breakers,
789 route_bind_concurrency,
790 route_outages,
791 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
792 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
793 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
794 storage_config: None,
795 machine_id: None,
796 admission_facts_carrier_module_id: None,
797 admission_facts_targets: None,
798 scopes: Arc::new(RwLock::new(ScopeTable::new(
799 crate::daemon_config::default_scope_authority_owners(),
800 ))),
801 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
802 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
803 rescan: None,
804 connected_clients: ConnectedClients::new(),
805 counters,
806 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
807 daemon_provenance: DaemonProvenanceFacts::default(),
808 #[cfg(test)]
809 control_dispatch_delay: None,
810 #[cfg(test)]
811 provenance_probe_override: None,
812 }
813 }
814
815 pub fn with_storage_config(
818 mut self,
819 storage_config: Option<crate::daemon_config::StorageConfig>,
820 ) -> Self {
821 self.storage_config = storage_config;
822 self
823 }
824
825 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
828 self.machine_id = machine_id;
829 self
830 }
831
832 pub fn with_admission_facts_config(
836 mut self,
837 carrier_module_id: Option<String>,
838 targets: Option<Vec<String>>,
839 ) -> Self {
840 self.admission_facts_carrier_module_id = carrier_module_id;
841 self.admission_facts_targets = targets;
842 self
843 }
844
845 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
849 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
850 self.scope_authority_owners = owners;
851 self
852 }
853
854 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
857 self.route_bind_relay_timeout = timeout;
858 self
859 }
860
861 pub fn with_route_bind_relay_timeouts(
867 mut self,
868 timeouts: impl IntoIterator<Item = (String, Duration)>,
869 ) -> Self {
870 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
871 self
872 }
873
874 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
880 self.route_bind_relay_timeouts
881 .get(module_id)
882 .copied()
883 .unwrap_or(self.route_bind_relay_timeout)
884 }
885
886 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
894 self.route_bind_breaker_threshold = threshold.max(1);
895 self.route_bind_breaker_cooldown = cooldown;
896 self
897 }
898
899 #[cfg(test)]
900 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
901 self.health_probe_timeout = timeout;
902 self
903 }
904
905 #[cfg(test)]
906 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
907 self.control_dispatch_delay = Some(delay);
908 self
909 }
910
911 pub fn with_process_liveness(
912 mut self,
913 process_liveness: Arc<dyn ModuleProcessLiveness>,
914 ) -> Self {
915 self.process_liveness = Some(process_liveness);
916 self
917 }
918
919 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
920 self.supervisor = supervisor;
921 self
922 }
923
924 pub fn with_daemon_provenance(
925 mut self,
926 pid: u32,
927 started_at_ms: u64,
928 executable_path: Option<PathBuf>,
929 build_git_sha: Option<String>,
930 build_lock_digest: Option<String>,
931 ) -> Self {
932 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
933 let process_start_time = process_start_time(pid);
934 self.daemon_provenance = DaemonProvenanceFacts {
935 build: DaemonBuildProvenance {
936 build_git_sha,
937 build_lock_digest,
938 },
939 pid: Some(pid),
940 started_at_ms: Some(started_at_ms),
941 start_clock: None,
942 executable_path,
943 executable_identity,
944 process_start_time,
945 probe: ExecutableIdentityProbe::default(),
946 };
947 self
948 }
949
950 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
951 self.daemon_provenance.start_clock = Some(clock);
952 self
953 }
954
955 #[cfg(test)]
956 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
957 self.provenance_probe_override = Some(result);
958 self
959 }
960
961 pub fn with_capability_config(
965 self,
966 modules: impl IntoIterator<Item = (String, bool)>,
967 reserved_capabilities: BTreeMap<String, String>,
968 ) -> Self {
969 self.capability_evaluator
970 .configure(modules, reserved_capabilities);
971 self
972 }
973
974 pub fn with_supervisor_rescan(
975 mut self,
976 supervisor: Supervisor,
977 config_path: impl Into<PathBuf>,
978 configured_port: Option<u16>,
979 ) -> Self {
980 self.rescan = Some(SupervisorRescanContext {
981 supervisor,
982 config_path: config_path.into(),
983 configured_port,
984 storage_config: self.storage_config.clone(),
985 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
986 admission_facts_targets: self.admission_facts_targets.clone(),
987 scope_authority_owners: self.scope_authority_owners.clone(),
988 });
989 self
990 }
991
992 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
993 self.connected_clients = connected_clients;
994 self
995 }
996
997 pub fn forwarding(&self) -> Arc<ForwardingTable> {
998 Arc::clone(&self.forwarding)
999 }
1000
1001 pub(crate) fn counters(&self) -> DaemonCounters {
1002 self.counters.clone()
1003 }
1004
1005 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1008 tokio::spawn(async move {
1009 loop {
1010 self.capability_evaluator
1011 .wait_for_change_or_deadline()
1012 .await;
1013 self.refresh_capability_requirements();
1014 }
1015 });
1016 }
1017
1018 fn runtime_capability_snapshot(
1019 &self,
1020 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1021 let runtime = self
1022 .supervisor
1023 .list()
1024 .into_iter()
1025 .map(|module| {
1026 let status = module.status().map_err(|err| {
1027 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1028 })?;
1029 Ok(RuntimeModule {
1030 module_id: status.module_id,
1031 state: status.state,
1032 enabled: status.enabled,
1033 })
1034 })
1035 .collect::<Result<Vec<_>, RouterError>>()?;
1036 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1037 RouterError::backend(
1038 0,
1039 0,
1040 format!("failed to list capability registrations: {err}"),
1041 )
1042 })?;
1043 let registrations = registrations
1044 .into_iter()
1045 .map(|registration| RegisteredModule {
1046 module_id: registration.manifest.module_id,
1047 module_version: registration.manifest.module_version,
1048 capabilities: registration.manifest.capabilities,
1049 })
1050 .collect();
1051 Ok((runtime, registrations))
1052 }
1053
1054 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1061 let cached_registration = RegisteredModule {
1062 module_id: registration.manifest.module_id.clone(),
1063 module_version: registration.manifest.module_version.clone(),
1064 capabilities: registration.manifest.capabilities.clone(),
1065 };
1066 if self.capability_evaluator.record_hello(&cached_registration) {
1067 warn!(
1068 module_id = %cached_registration.module_id,
1069 "capability claims drifted from the cached manifest"
1070 );
1071 }
1072 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1073 self.enforce_capability_denies();
1074 }
1075 self.refresh_capability_requirements();
1076 }
1077
1078 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1082 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1083 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1084 self.supervisor.set_swap_promotion_observer(observer);
1085 }
1086
1087 pub fn refresh_capability_requirements(&self) {
1088 match self.runtime_capability_snapshot() {
1089 Ok((runtime, registrations)) => {
1090 log_requirement_events(
1091 self.capability_evaluator
1092 .evaluate_now(&runtime, ®istrations),
1093 );
1094 }
1095 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1096 }
1097 }
1098
1099 fn enforce_capability_denies(&self) {
1103 let (_, registrations) = match self.registry.list_modules() {
1104 Ok(snapshot) => snapshot,
1105 Err(err) => {
1106 warn!(error = %err, "failed to read registrations for capability deny census");
1107 return;
1108 }
1109 };
1110 let manifests = registrations
1111 .into_iter()
1112 .map(|registration| {
1113 (
1114 registration.manifest.module_id.clone(),
1115 registration.manifest,
1116 )
1117 })
1118 .collect::<BTreeMap<_, _>>();
1119 let census = match self.forwarding.route_census(None) {
1120 Ok(census) => census,
1121 Err(err) => {
1122 warn!(error = %err, "failed to read route census for capability deny enforcement");
1123 return;
1124 }
1125 };
1126
1127 for (target_module_id, routes) in census {
1128 let Some(target_manifest) = manifests.get(&target_module_id) else {
1129 continue;
1130 };
1131 let mut closed_routes = Vec::new();
1132 let mut module_goodbyes = Vec::new();
1133 for route in routes {
1134 let Principal::Reserved {
1135 module_id: opening_module_id,
1136 } = &route.principal
1137 else {
1138 continue;
1139 };
1140 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1141 continue;
1142 };
1143 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1144 continue;
1145 };
1146
1147 match self.forwarding.release_client_route(
1148 route.goodbye_target.connection_id,
1149 route.goodbye_target.channel,
1150 route.goodbye_target.epoch,
1151 ) {
1152 Ok(RouteRelease::Removed(module_goodbye)) => {
1153 warn!(
1154 opening_module_id,
1155 target_module_id,
1156 capability,
1157 "force-closing route because an attested capability deny edge now matches"
1158 );
1159 closed_routes.push(route);
1160 module_goodbyes.push(module_goodbye);
1161 }
1162 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1163 Err(err) => warn!(
1164 opening_module_id,
1165 target_module_id,
1166 capability,
1167 error = %err,
1168 "failed to force-close capability-denied route"
1169 ),
1170 }
1171 }
1172
1173 if closed_routes.is_empty() {
1174 continue;
1175 }
1176 send_route_control_pushes(
1177 &self.forwarding,
1178 closed_routes,
1179 ClientControlPush::RouteClosed {
1180 module_id: target_module_id,
1181 channels: Vec::new(),
1182 reason: RouteCloseReason::CapabilityDenied,
1183 drained: false,
1184 abandoned: 0,
1185 excluded_subscriptions: 0,
1186 terminal: Some(false),
1187 },
1188 );
1189 self.emit_route_goodbyes(module_goodbyes);
1190 }
1191 }
1192
1193 fn not_ready_reason(
1199 &self,
1200 registration: &crate::registry::ModuleRegistration,
1201 ) -> Option<NotReadyReason> {
1202 if !registration.ready {
1203 return Some(NotReadyReason {
1204 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1205 capability: None,
1206 });
1207 }
1208 self.first_unprovided_required_capability(registration)
1209 .map(|capability| NotReadyReason {
1210 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1211 capability: Some(capability),
1212 })
1213 }
1214
1215 fn first_unprovided_required_capability(
1233 &self,
1234 registration: &crate::registry::ModuleRegistration,
1235 ) -> Option<String> {
1236 let required = registration
1237 .manifest
1238 .capabilities
1239 .iter()
1240 .flat_map(|declarations| declarations.requires.iter())
1241 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1242 .map(|requirement| requirement.capability.as_str())
1243 .collect::<BTreeSet<_>>();
1244 if required.is_empty() {
1245 return None;
1246 }
1247 let module_id = registration.manifest.module_id.as_str();
1248 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1249 if required
1250 .iter()
1251 .any(|capability| verdict(capability).is_none())
1252 {
1253 self.refresh_capability_requirements();
1254 }
1255 required
1256 .into_iter()
1257 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1258 .map(str::to_string)
1259 }
1260
1261 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1262 self.capability_evaluator
1263 .statuses()
1264 .into_iter()
1265 .map(capability_requirement_status)
1266 .collect()
1267 }
1268
1269 fn deregister_connection(
1276 &self,
1277 connection_id: ConnectionId,
1278 reason: RegistrationEndReason,
1279 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1280 self.registry
1281 .deregister_connection_with_reason(connection_id, reason)
1282 }
1283
1284 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1285 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1286 return None;
1287 }
1288 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1289 parse_client_control_request(&frame.body)
1290 else {
1291 return None;
1292 };
1293 Some(target_module_id(&target).to_string())
1294 }
1295
1296 pub(crate) fn route_open_capacity_refusal(
1297 &self,
1298 ctx: &RouteCtx,
1299 frame: &Frame,
1300 target_module_id: &str,
1301 in_flight: usize,
1302 limit: usize,
1303 ) -> Result<Frame, RouterError> {
1304 self.route_open_admission_refusal_frame(
1305 ctx,
1306 frame,
1307 target_module_id,
1308 "open_admission_full",
1309 (in_flight, limit),
1310 format!(
1311 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1312 ),
1313 )
1314 }
1315
1316 fn route_open_target_capacity_refusal(
1317 &self,
1318 ctx: &RouteCtx,
1319 frame: &Frame,
1320 target_module_id: &str,
1321 in_flight: usize,
1322 ) -> Result<Frame, RouterError> {
1323 self.route_open_admission_refusal_frame(
1324 ctx,
1325 frame,
1326 target_module_id,
1327 "target_binds_full",
1328 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1329 format!(
1330 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1331 ),
1332 )
1333 }
1334
1335 fn route_open_admission_refusal_frame(
1343 &self,
1344 ctx: &RouteCtx,
1345 frame: &Frame,
1346 target_module_id: &str,
1347 reason: &'static str,
1348 (in_flight, limit): (usize, usize),
1349 message: impl Into<String>,
1350 ) -> Result<Frame, RouterError> {
1351 let code = error_codes::TARGET_UNAVAILABLE;
1352 self.counters.increment_route_open_refused(code);
1353 info!(
1354 target: "control",
1355 code,
1356 reason,
1357 module_id = ?target_module_id,
1358 connection_id = ctx.connection_id.get(),
1359 in_flight,
1360 limit,
1361 "route.open refused"
1362 );
1363 control_error_frame(frame, code, message.into())
1364 }
1365
1366 #[cfg(test)]
1372 pub fn handle_control(
1373 &self,
1374 connection_id: ConnectionId,
1375 frame: Frame,
1376 ) -> Result<Vec<Frame>, RouterError> {
1377 match frame.header.ty {
1378 FrameType::Ping => Ok(vec![pong(&frame)?]),
1379 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1380 FrameType::Goodbye => self.handle_goodbye(connection_id),
1381 ty => Ok(vec![control_error_frame(
1382 &frame,
1383 "unsupported_control_frame",
1384 format!("unsupported channel-0 frame {ty:?}"),
1385 )?]),
1386 }
1387 }
1388
1389 pub async fn handle_control_frame(
1390 &self,
1391 ctx: &RouteCtx,
1392 frame: Frame,
1393 ) -> Result<Vec<Frame>, RouterError> {
1394 self.handle_control_frame_timed(ctx, frame, None).await
1395 }
1396
1397 pub(crate) async fn handle_control_frame_timed(
1398 &self,
1399 ctx: &RouteCtx,
1400 frame: Frame,
1401 dispatch_started_at: Option<StdInstant>,
1402 ) -> Result<Vec<Frame>, RouterError> {
1403 match frame.header.ty {
1404 FrameType::Ping => Ok(vec![pong(&frame)?]),
1405 FrameType::Hello => {
1406 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1407 }
1408 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1409 FrameType::Cancel => {
1410 if self
1413 .forwarding
1414 .operator_confirms()
1415 .cancel(ctx.connection_id, frame.header.corr)
1416 || self
1417 .supervisor
1418 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1419 {
1420 Ok(Vec::new())
1421 } else {
1422 Ok(vec![control_error_frame(
1423 &frame,
1424 "unknown_subscription",
1425 "no supervisor spawn subscription has this correlation id",
1426 )?])
1427 }
1428 }
1429 FrameType::Request => {
1430 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1433 if op
1434 .as_ref()
1435 .is_some_and(|probe| probe.op == "operator.confirm")
1436 {
1437 return self.handle_operator_confirm(ctx, frame);
1438 }
1439 if self
1440 .forwarding
1441 .module_endpoint_for_connection(ctx.connection_id)
1442 .map_err(RouterError::Forwarding)?
1443 .is_some()
1444 {
1445 if !is_known_module_request_op(&frame.body) {
1446 return Ok(vec![control_error_frame(
1447 &frame,
1448 "unsupported_control_frame",
1449 "module-originated channel-0 REQUEST is not supported",
1450 )?]);
1451 }
1452 let request = match parse_module_control_request_from_module(&frame.body) {
1453 Ok(request) => request,
1454 Err((err, ControlRequestBodyError::UnknownOp)) => {
1455 return Ok(vec![control_error_frame(
1456 &frame,
1457 "unsupported_control_frame",
1458 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1459 )?])
1460 }
1461 Err((err, ControlRequestBodyError::InvalidBody)) => {
1462 return Ok(vec![control_error_frame(
1463 &frame,
1464 "invalid_control_body",
1465 format!("malformed module control body: {err}"),
1466 )?])
1467 }
1468 };
1469 let op = module_control_request_op(&request);
1470 let corr = frame.header.corr;
1471 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1472 let result =
1473 self.handle_module_control_request(ctx.connection_id, frame, request);
1474 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1475 return result;
1476 }
1477
1478 if is_known_module_request_op(&frame.body) {
1479 return Ok(vec![control_error_frame(
1480 &frame,
1481 "not_registered",
1482 "catalog.update requires an active module registration owned by this connection",
1483 )?]);
1484 }
1485
1486 let request = match parse_client_control_request(&frame.body) {
1487 Ok(request) => request,
1488 Err((err, ControlRequestBodyError::UnknownOp)) => {
1489 return Ok(vec![control_error_frame(
1490 &frame,
1491 "unknown_control_op",
1492 format!("unknown client control op: {err}"),
1493 )?])
1494 }
1495 Err((err, ControlRequestBodyError::InvalidBody)) => {
1496 return Ok(vec![control_error_frame(
1497 &frame,
1498 "invalid_control_body",
1499 format!("malformed client control body: {err}"),
1500 )?])
1501 }
1502 };
1503 let op = client_control_request_op(&request);
1504 let corr = frame.header.corr;
1505 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1506 #[cfg(test)]
1507 if let Some(delay) = self.control_dispatch_delay {
1508 tokio::time::sleep(delay).await;
1509 }
1510 let result = self
1511 .handle_client_control_request(ctx, frame, request)
1512 .await;
1513 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1514 result
1515 }
1516 FrameType::Push => {
1517 let Some(endpoint) = self
1518 .forwarding
1519 .module_endpoint_for_connection(ctx.connection_id)
1520 .map_err(RouterError::Forwarding)?
1521 else {
1522 return Ok(vec![control_error_frame(
1523 &frame,
1524 "unsupported_control_frame",
1525 "client-originated channel-0 PUSH is not supported",
1526 )?]);
1527 };
1528 self.handle_status_update(endpoint, frame)
1529 }
1530 FrameType::Response | FrameType::Error
1531 if self
1532 .forwarding
1533 .module_endpoint_for_connection(ctx.connection_id)
1534 .map_err(RouterError::Forwarding)?
1535 .is_some() =>
1536 {
1537 self.handle_module_relay_response(ctx.connection_id, frame)
1538 }
1539 ty => Ok(vec![control_error_frame(
1540 &frame,
1541 "unsupported_control_frame",
1542 format!("unsupported channel-0 frame {ty:?}"),
1543 )?]),
1544 }
1545 }
1546
1547 pub fn cleanup_connection(
1548 &self,
1549 connection_id: ConnectionId,
1550 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1551 self.cleanup_connection_with_end_reason(
1552 connection_id,
1553 RegistrationEndReason::ConnectionClosed,
1554 )
1555 }
1556
1557 fn cleanup_connection_with_end_reason(
1558 &self,
1559 connection_id: ConnectionId,
1560 requested_reason: RegistrationEndReason,
1561 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1562 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1563 self.registry
1564 .get_module_by_connection(connection_id)?
1565 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1566 .and_then(|module| module.registration_end_reason().ok().flatten())
1567 .unwrap_or(requested_reason)
1568 } else {
1569 requested_reason
1570 };
1571 let crash_closed = self
1572 .registry
1573 .get_module_by_connection(connection_id)?
1574 .and_then(|registration| {
1575 self.forwarding
1576 .module_endpoint_for_connection(connection_id)
1577 .ok()
1578 .flatten()
1579 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1580 .map(|routes| (registration.manifest.module_id, routes))
1581 });
1582 let crash_closed = crash_closed.map(|(module_id, routes)| {
1583 let terminal = match self.supervisor.get(&module_id) {
1584 None => false,
1585 Some(module) => match module.will_recover_after_connection_loss() {
1586 Ok(will_recover) => !will_recover,
1587 Err(err) => {
1588 warn!(
1589 %module_id,
1590 error = %err,
1591 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1592 );
1593 false
1594 }
1595 },
1596 };
1597 let reason = match self.forwarding.is_daemon_draining() {
1601 Ok(true) => RouteCloseReason::Restart,
1602 Ok(false) => RouteCloseReason::Crash,
1603 Err(err) => {
1604 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1605 RouteCloseReason::Crash
1606 }
1607 };
1608 (module_id, routes, reason, terminal)
1609 });
1610 let registrations = self.deregister_connection(connection_id, end_reason);
1611 let cleanup = if crash_closed.is_some() {
1612 self.forwarding.cleanup_connection_counted(connection_id)
1613 } else {
1614 self.forwarding
1620 .cleanup_connection(connection_id)
1621 .map(|released| crate::forwarding::ConnectionCleanup {
1622 released,
1623 abandoned_relays: 0,
1624 })
1625 };
1626 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1631 let abandoned = cleanup
1632 .as_ref()
1633 .map_or(0, |cleanup| cleanup.abandoned_relays);
1634 send_route_control_pushes(
1635 &self.forwarding,
1636 routes,
1637 ClientControlPush::RouteClosed {
1638 module_id,
1639 channels: Vec::new(),
1640 reason,
1641 drained: false,
1642 abandoned,
1643 excluded_subscriptions: 0,
1644 terminal: Some(terminal),
1645 },
1646 );
1647 }
1648 if let Ok(cleanup) = cleanup {
1649 self.emit_route_goodbyes(cleanup.released);
1650 }
1651 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1655 crate::supervise::notify_registration_release();
1656 self.capability_evaluator.wake_deadline_loop();
1657 self.refresh_capability_requirements();
1658 }
1659 self.supervisor.remove_spawn_subscribers(connection_id);
1660 self.hello_launch_nonces
1663 .lock()
1664 .unwrap_or_else(|poisoned| poisoned.into_inner())
1665 .forget(connection_id);
1666 self.scopes
1667 .write()
1668 .unwrap_or_else(|poisoned| poisoned.into_inner())
1669 .release_connection(connection_id);
1670 registrations
1671 }
1672
1673 pub(crate) fn handle_route_goodbye(
1674 &self,
1675 connection_id: ConnectionId,
1676 route_channel: u16,
1677 route_epoch: u32,
1678 ) -> Result<bool, RouterError> {
1679 debug!(
1680 connection_id = connection_id.get(),
1681 route_channel, route_epoch, "handling route GOODBYE"
1682 );
1683 let RouteRelease::Removed(released_route) = self
1684 .forwarding
1685 .release_client_route(connection_id, route_channel, route_epoch)
1686 .map_err(RouterError::Forwarding)?
1687 else {
1688 return Ok(false);
1689 };
1690 self.emit_route_goodbyes(vec![released_route]);
1691 Ok(true)
1692 }
1693
1694 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1695 for released in released_routes {
1696 let frame = match Frame::build_with_version(
1697 released.negotiated_ver,
1698 FrameType::Goodbye,
1699 control_flags(),
1700 released.channel,
1701 released.epoch,
1702 0,
1703 Vec::new(),
1704 ) {
1705 Ok(frame) => frame,
1706 Err(err) => {
1707 warn!(
1708 route_channel = released.channel,
1709 error = %err,
1710 "failed to build route GOODBYE frame"
1711 );
1712 continue;
1713 }
1714 };
1715 if !released.close_on_delivery_failure() {
1716 crate::forwarding::send_module_route_goodbye(
1717 &self.counters,
1718 &released.sink,
1719 frame,
1720 released.module_id.as_deref(),
1721 "client route released",
1722 );
1723 continue;
1724 }
1725 if let Err(err) = released.sink.try_send(frame) {
1726 warn!(
1727 target_connection_id = released.connection_id.get(),
1728 route_channel = released.channel,
1729 error = %err,
1730 "route GOODBYE was not delivered to client; closing target connection"
1731 );
1732 if self
1733 .forwarding
1734 .escalate_client_delivery_failure(
1735 released.connection_id,
1736 released.channel,
1737 released.epoch,
1738 CloseReason::new(
1739 "route_goodbye_delivery_failed",
1740 format!(
1741 "failed to enqueue route GOODBYE for channel {}: {err}",
1742 released.channel
1743 ),
1744 ),
1745 crate::forwarding::UndeliveredFrame {
1746 module_id: released.module_id.as_deref(),
1747 sink: &released.sink,
1748 },
1749 )
1750 .unwrap_or(false)
1751 {
1752 self.counters.increment_goodbye_relay_client_failed();
1753 }
1754 }
1755 }
1756 }
1757
1758 fn send_abandoned_route_bind_goodbye(
1770 &self,
1771 module_sink: &crate::FrameSink,
1772 negotiated_ver: u8,
1773 module_channel: u16,
1774 module_epoch: u32,
1775 ) {
1776 let frame = match Frame::build_with_version(
1777 negotiated_ver,
1778 FrameType::Goodbye,
1779 control_flags(),
1780 module_channel,
1781 module_epoch,
1782 0,
1783 Vec::new(),
1784 ) {
1785 Ok(frame) => frame,
1786 Err(err) => {
1787 warn!(
1788 route_channel = module_channel,
1789 error = %err,
1790 "failed to build GOODBYE for abandoned route.bind"
1791 );
1792 return;
1793 }
1794 };
1795 crate::forwarding::send_module_route_goodbye(
1796 &self.counters,
1797 module_sink,
1798 frame,
1799 None,
1800 "abandoned route.bind",
1801 );
1802 }
1803
1804 fn handle_hello(
1805 &self,
1806 connection_id: ConnectionId,
1807 sink: Option<crate::FrameSink>,
1808 frame: Frame,
1809 ) -> Result<Vec<Frame>, RouterError> {
1810 debug!(
1811 connection_id = connection_id.get(),
1812 corr = frame.header.corr,
1813 "handling HELLO"
1814 );
1815 if self
1819 .registry
1820 .get_module_by_connection(connection_id)
1821 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
1822 .is_some()
1823 {
1824 return Ok(vec![control_error_frame(
1825 &frame,
1826 "invalid_hello",
1827 "connection is already registered as a module",
1828 )?]);
1829 }
1830 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1831 Ok(value) => value,
1832 Err(err) => {
1833 return Ok(vec![control_error_frame(
1834 &frame,
1835 "invalid_hello",
1836 format!("malformed HELLO body: {err}"),
1837 )?])
1838 }
1839 };
1840 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1841 return Ok(vec![control_error_frame(
1842 &frame,
1843 "invalid_capability_grammar",
1844 err.to_string(),
1845 )?]);
1846 }
1847 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1848 return Ok(vec![control_error_frame(
1849 &frame,
1850 "invalid_manifest",
1851 err.to_string(),
1852 )?]);
1853 }
1854 if let Err(err) = validate_hello_event_declarations(&hello_value) {
1855 return Ok(vec![control_error_frame(
1856 &frame,
1857 "invalid_event_declaration",
1858 err.to_string(),
1859 )?]);
1860 }
1861 if let Some(provenance) = hello_value
1862 .get("manifest")
1863 .and_then(|manifest| manifest.get("provenance"))
1864 {
1865 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1866 return Ok(vec![control_error_frame(
1867 &frame,
1868 "invalid_manifest",
1869 format!("malformed manifest provenance: {err}"),
1870 )?]);
1871 }
1872 }
1873 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1874 Ok(hello) => hello,
1875 Err(err) => {
1876 return Ok(vec![control_error_frame(
1877 &frame,
1878 "invalid_hello",
1879 format!("malformed HELLO body: {err}"),
1880 )?])
1881 }
1882 };
1883
1884 if hello.protocol_ver != hello.manifest.protocol_ver {
1885 return Ok(vec![control_error_frame(
1886 &frame,
1887 "invalid_manifest",
1888 format!(
1889 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1890 hello.protocol_ver, hello.manifest.protocol_ver
1891 ),
1892 )?]);
1893 }
1894
1895 if hello.manifest.module_id.trim().is_empty() {
1896 return Ok(vec![control_error_frame(
1897 &frame,
1898 "invalid_manifest",
1899 "manifest module_id must not be empty",
1900 )?]);
1901 }
1902
1903 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1904 Ok(negotiated_ver) => negotiated_ver,
1905 Err(message) => {
1906 return Ok(vec![control_error_frame(
1907 &frame,
1908 "version_unsupported",
1909 message,
1910 )?])
1911 }
1912 };
1913
1914 let swap_admission = self
1922 .supervisor
1923 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
1924 if swap_admission == SwapHelloAdmission::Refused {
1925 warn!(
1926 module_id = %hello.manifest.module_id,
1927 connection_id = connection_id.get(),
1928 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
1929 );
1930 return Ok(vec![control_error_frame(
1931 &frame,
1932 "swap_token_invalid",
1933 format!(
1934 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
1935 hello.manifest.module_id
1936 ),
1937 )?]);
1938 }
1939 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
1940
1941 if let Some(rejection) = (!swap_candidate)
1949 .then(|| {
1950 self.supervisor.reserved_hello_rejection(
1951 &hello.manifest.module_id,
1952 hello.launch_nonce.as_deref(),
1953 )
1954 })
1955 .flatten()
1956 {
1957 let message = match rejection {
1958 ReservedHelloRejection::Exact { module_id } => format!(
1959 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
1960 ),
1961 ReservedHelloRejection::Prefix {
1962 prefix,
1963 owner_module_id,
1964 } => format!(
1965 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
1966 hello.manifest.module_id
1967 ),
1968 };
1969 return Ok(vec![control_error_frame(
1970 &frame,
1971 "reserved_module",
1972 message,
1973 )?]);
1974 }
1975
1976 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
1977 &hello.manifest.module_id,
1978 hello.manifest.capabilities.as_ref(),
1979 );
1980 if let Some(refusal) = reserved_capability_refusals.first() {
1981 let capability = refusal.capability.clone();
1982 let bound_module = refusal.claimants[0].clone();
1983 log_duplicate_claim_events(reserved_capability_refusals);
1984 return Ok(vec![control_error_frame(
1985 &frame,
1986 "reserved_capability",
1987 format!(
1988 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
1989 capability, bound_module, hello.manifest.module_id
1990 ),
1991 )?]);
1992 }
1993
1994 if self
1997 .forwarding
1998 .connection_has_client_routes(connection_id)
1999 .map_err(RouterError::Forwarding)?
2000 {
2001 return Ok(vec![control_error_frame(
2002 &frame,
2003 "invalid_hello",
2004 "connection has open client routes and cannot also register as a module",
2005 )?]);
2006 }
2007
2008 self.hello_launch_nonces
2013 .lock()
2014 .unwrap_or_else(|poisoned| poisoned.into_inner())
2015 .record(connection_id, hello.launch_nonce.as_deref());
2016 let control_ops = effective_module_control_ops(hello.control_ops);
2017 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2020 if swap_candidate {
2021 return self.register_swap_candidate(
2022 connection_id,
2023 sink,
2024 &frame,
2025 hello.manifest,
2026 negotiated_ver,
2027 control_ops,
2028 hello_ack,
2029 );
2030 }
2031 let registration = match self.registry.register_with_control_ops(
2032 hello.manifest,
2033 negotiated_ver,
2034 connection_id,
2035 control_ops,
2036 ) {
2037 Ok(registration) => registration,
2038 Err(RegistryError::DuplicateModuleId { module_id }) => {
2039 return Ok(vec![control_error_frame(
2040 &frame,
2041 "duplicate_module_id",
2042 format!(
2043 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2044 ),
2045 )?])
2046 }
2047 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2048 return Ok(vec![control_error_frame(
2049 &frame,
2050 "invalid_module_id",
2051 err.to_string(),
2052 )?])
2053 }
2054 Err(err) => {
2055 return Ok(vec![control_error_frame(
2056 &frame,
2057 "registry_error",
2058 err.to_string(),
2059 )?])
2060 }
2061 };
2062
2063 let reply = if let Some(sink) = sink {
2064 let concurrency = manifest_concurrency(®istration.manifest);
2080 if let Err(err) = self.forwarding.register_module_connection_acked(
2081 connection_id,
2082 registration.manifest.module_id.clone(),
2083 negotiated_ver,
2084 concurrency,
2085 sink,
2086 hello_ack,
2087 ) {
2088 if matches!(
2092 self.deregister_connection(
2093 connection_id,
2094 RegistrationEndReason::RegistrationFailed,
2095 ),
2096 Ok(r) if !r.is_empty()
2097 ) {
2098 crate::supervise::notify_registration_release();
2099 }
2100 return Ok(vec![control_error_frame(
2101 &frame,
2102 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2103 "invalid_hello"
2104 } else {
2105 forwarding_error_code(&err)
2106 },
2107 err.to_string(),
2108 )?]);
2109 }
2110 Vec::new()
2111 } else {
2112 vec![hello_ack]
2115 };
2116
2117 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2126 info!(
2127 module_id = %registration.manifest.module_id,
2128 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2129 );
2130 }
2131
2132 self.apply_registration_capabilities(®istration);
2133
2134 info!(
2135 module_id = %registration.manifest.module_id,
2136 module_version = %registration.manifest.module_version,
2137 negotiated_ver,
2138 routable_provider = manifest_provides_routable_role(®istration.manifest),
2139 connection_id = connection_id.get(),
2140 "module registered"
2141 );
2142
2143 Ok(reply)
2144 }
2145
2146 #[allow(clippy::too_many_arguments)]
2156 fn register_swap_candidate(
2157 &self,
2158 connection_id: ConnectionId,
2159 sink: Option<crate::FrameSink>,
2160 frame: &Frame,
2161 manifest: ModuleManifest,
2162 negotiated_ver: u8,
2163 control_ops: Vec<String>,
2164 hello_ack: Frame,
2165 ) -> Result<Vec<Frame>, RouterError> {
2166 let module_id = manifest.module_id.clone();
2167 let registration = match self.registry.register_candidate_with_control_ops(
2168 manifest,
2169 negotiated_ver,
2170 connection_id,
2171 control_ops,
2172 ) {
2173 Ok(registration) => registration,
2174 Err(RegistryError::DuplicateModuleId { module_id }) => {
2175 return Ok(vec![control_error_frame(
2176 frame,
2177 "duplicate_module_id",
2178 format!(
2179 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2180 ),
2181 )?])
2182 }
2183 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2184 return Ok(vec![control_error_frame(
2185 frame,
2186 "invalid_module_id",
2187 err.to_string(),
2188 )?])
2189 }
2190 Err(err) => {
2191 return Ok(vec![control_error_frame(
2192 frame,
2193 "registry_error",
2194 err.to_string(),
2195 )?])
2196 }
2197 };
2198 let reply = if let Some(sink) = sink {
2199 let concurrency = manifest_concurrency(®istration.manifest);
2203 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2204 connection_id,
2205 module_id.clone(),
2206 negotiated_ver,
2207 concurrency,
2208 sink,
2209 hello_ack,
2210 ) {
2211 if matches!(
2212 self.deregister_connection(
2213 connection_id,
2214 RegistrationEndReason::RegistrationFailed,
2215 ),
2216 Ok(r) if !r.is_empty()
2217 ) {
2218 crate::supervise::notify_registration_release();
2219 }
2220 return Ok(vec![control_error_frame(
2221 frame,
2222 forwarding_error_code(&err),
2223 err.to_string(),
2224 )?]);
2225 }
2226 Vec::new()
2227 } else {
2228 vec![hello_ack]
2229 };
2230 self.supervisor.mark_swap_candidate_admitted(&module_id);
2231 info!(
2232 module_id = %module_id,
2233 module_version = %registration.manifest.module_version,
2234 negotiated_ver,
2235 ready = registration.ready,
2236 connection_id = connection_id.get(),
2237 "swap candidate registered; not routable until cutover"
2238 );
2239 Ok(reply)
2240 }
2241
2242 fn build_hello_ack(
2243 &self,
2244 frame: &Frame,
2245 negotiated_ver: u8,
2246 module_id: &str,
2247 ) -> Result<Frame, RouterError> {
2248 let ack = ModuleHelloAckBody {
2249 negotiated_ver,
2250 subc_ops: module_subc_ops(),
2251 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2252 storage: self
2253 .storage_config
2254 .as_ref()
2255 .map(|cfg| cfg.descriptor_for(module_id)),
2256 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2257 };
2258 let body = serde_json::to_vec(&ack).map_err(|err| {
2259 RouterError::backend(
2260 0,
2261 frame.header.corr,
2262 format!("failed to encode HELLO_ACK: {err}"),
2263 )
2264 })?;
2265
2266 Frame::build_with_version(
2267 negotiated_ver,
2268 FrameType::HelloAck,
2269 control_flags(),
2270 0,
2271 0,
2272 frame.header.corr,
2273 body,
2274 )
2275 .map_err(RouterError::FrameBuild)
2276 }
2277
2278 async fn handle_client_control_request(
2279 &self,
2280 ctx: &RouteCtx,
2281 frame: Frame,
2282 request: ClientControlRequest,
2283 ) -> Result<Vec<Frame>, RouterError> {
2284 match request {
2285 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2286 ClientControlRequest::CatalogList { module_id } => {
2287 self.handle_catalog_list(frame, module_id)
2288 }
2289 ClientControlRequest::RouteOpen {
2290 target,
2291 identity,
2292 consumer_identity,
2293 consumer_capabilities,
2294 role_versions,
2295 admission_facts,
2296 scope,
2297 } => {
2298 self.handle_route_open(
2299 ctx,
2300 frame,
2301 RouteOpenRequest {
2302 target,
2303 identity,
2304 consumer_identity,
2305 consumer_capabilities,
2306 role_versions,
2307 admission_facts,
2308 scope,
2309 },
2310 )
2311 .await
2312 }
2313 ClientControlRequest::RoutePoll {
2314 route_channel,
2315 route_epoch,
2316 kind,
2317 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2318 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2319 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2320 self.handle_supervisor_spawn_snapshot(frame)
2321 }
2322 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2323 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2324 }
2325 ClientControlRequest::SupervisorRestart {
2326 module_id,
2327 drain_timeout_ms,
2328 } => {
2329 self.log_supervisor_request_received(
2330 ctx,
2331 frame.header.corr,
2332 ops::SUPERVISOR_RESTART,
2333 Some(&module_id),
2334 None,
2335 )?;
2336 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2337 .await
2338 }
2339 ClientControlRequest::SupervisorSwap {
2340 module_id,
2341 ready_timeout_ms,
2342 } => {
2343 self.log_supervisor_request_received(
2344 ctx,
2345 frame.header.corr,
2346 ops::SUPERVISOR_SWAP,
2347 Some(&module_id),
2348 None,
2349 )?;
2350 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2351 .await
2352 }
2353 ClientControlRequest::SupervisorReload { module_id } => {
2354 self.log_supervisor_request_received(
2355 ctx,
2356 frame.header.corr,
2357 ops::SUPERVISOR_RELOAD,
2358 Some(&module_id),
2359 None,
2360 )?;
2361 self.handle_supervisor_reload(frame, module_id).await
2362 }
2363 ClientControlRequest::SupervisorRescan { preview } => {
2364 if !preview {
2365 self.log_supervisor_request_received(
2366 ctx,
2367 frame.header.corr,
2368 ops::SUPERVISOR_RESCAN,
2369 None,
2370 None,
2371 )?;
2372 }
2373 self.handle_supervisor_rescan(frame, preview).await
2374 }
2375 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2376 self.log_supervisor_request_received(
2377 ctx,
2378 frame.header.corr,
2379 ops::SUPERVISOR_RELEASE_RESERVED,
2380 Some(&module_id),
2381 None,
2382 )?;
2383 self.handle_supervisor_release_reserved(frame, module_id)
2384 .await
2385 }
2386 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2387 self.log_supervisor_request_received(
2388 ctx,
2389 frame.header.corr,
2390 ops::SUPERVISOR_SET_ENABLED,
2391 Some(&module_id),
2392 Some(enabled),
2393 )?;
2394 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2395 .await
2396 }
2397 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2398 self.handle_supervisor_health_probe(frame, module_id).await
2399 }
2400 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2401 ClientControlRequest::SupervisorRoutes { module_id } => {
2402 self.handle_supervisor_routes(frame, module_id)
2403 }
2404 ClientControlRequest::SupervisorProvenance { module_id } => {
2405 self.handle_supervisor_provenance(frame, module_id).await
2406 }
2407 ClientControlRequest::SupervisorStderrTail {
2408 module_id,
2409 max_lines,
2410 max_bytes,
2411 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2412 ClientControlRequest::SupervisorTerminals { module_id } => {
2413 self.handle_supervisor_terminals(frame, module_id).await
2414 }
2415 }
2416 }
2417
2418 fn log_supervisor_request_received(
2419 &self,
2420 ctx: &RouteCtx,
2421 corr: u64,
2422 op: &'static str,
2423 module_id: Option<&str>,
2424 enabled: Option<bool>,
2425 ) -> Result<(), RouterError> {
2426 let caller = self
2427 .registry
2428 .get_module_by_connection(ctx.connection_id)
2429 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2430 .map(|registration| Principal::Reserved {
2431 module_id: registration.manifest.module_id,
2432 })
2433 .unwrap_or(Principal::Direct);
2434 let caller = principal_label(&caller);
2435 info!(
2438 target: "control",
2439 op,
2440 module_id,
2441 enabled,
2442 connection_id = ctx.connection_id.get(),
2443 caller = %caller,
2444 "supervisor request received"
2445 );
2446 Ok(())
2447 }
2448
2449 fn handle_module_control_request(
2450 &self,
2451 connection_id: ConnectionId,
2452 frame: Frame,
2453 request: ModuleControlRequestFromModule,
2454 ) -> Result<Vec<Frame>, RouterError> {
2455 match request {
2456 ModuleControlRequestFromModule::CatalogUpdate {
2457 provides,
2458 capabilities,
2459 ready,
2460 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2461 ModuleControlRequestFromModule::LiveRoots {} => {
2462 let registered = self
2463 .registry
2464 .get_module_by_connection(connection_id)
2465 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2466 let Some(registration) = registered else {
2467 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2468 };
2469 let response = self
2470 .forwarding
2471 .live_roots(®istration.manifest.module_id)
2472 .map_err(RouterError::Forwarding)?;
2473 Ok(vec![control_response_body_frame(
2474 &frame,
2475 &response,
2476 "ModuleControlResponseToModule::LiveRoots",
2477 )?])
2478 }
2479 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2480 self.handle_scope_sync(connection_id, frame, generation, scopes)
2481 }
2482 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2483 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2484 }
2485 }
2486 }
2487
2488 fn handle_operator_confirm(
2489 &self,
2490 ctx: &RouteCtx,
2491 frame: Frame,
2492 ) -> Result<Vec<Frame>, RouterError> {
2493 use crate::operator_confirm::{audit, Outcome};
2494 let registration = self
2495 .registry
2496 .get_module_by_connection(ctx.connection_id)
2497 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2498 let Some(registration) = registration else {
2499 let outcome = Outcome::refusal("not_registered");
2500 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2501 return Ok(vec![outcome.frame(&frame)]);
2502 };
2503 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2504 Ok(request) => request,
2505 Err(_) => {
2506 let outcome = Outcome::refusal("invalid_control_body");
2507 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2508 return Ok(vec![outcome.frame(&frame)]);
2509 }
2510 };
2511 let module_id = registration.manifest.module_id;
2512 let nonce = self
2516 .hello_launch_nonces
2517 .lock()
2518 .unwrap_or_else(|p| p.into_inner())
2519 .nonce(ctx.connection_id)
2520 .map(str::to_owned);
2521 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2522 self.supervisor
2523 .spawned_consumer_authorized(&module_id, nonce)
2524 });
2525 let confirms = self.forwarding.operator_confirms();
2526 self.forwarding
2527 .with_operator_route(
2528 ctx.connection_id,
2529 request.route_channel,
2530 request.route_epoch,
2531 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
2532 )
2533 .map_err(RouterError::Forwarding)
2534 }
2535
2536 fn handle_scope_sync(
2540 &self,
2541 connection_id: ConnectionId,
2542 frame: Frame,
2543 generation: u64,
2544 scopes: Vec<ScopeRecord>,
2545 ) -> Result<Vec<Frame>, RouterError> {
2546 let Some(registration) = self
2547 .registry
2548 .get_module_by_connection(connection_id)
2549 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2550 else {
2551 return Ok(vec![control_error_frame(
2552 &frame,
2553 "not_registered",
2554 "scope.sync requires an active module registration owned by this connection",
2555 )?]);
2556 };
2557 let owner = registration.manifest.module_id;
2558 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2559 let is_current_launch = |connection: ConnectionId| {
2560 self.hello_launch_nonces
2561 .lock()
2562 .unwrap_or_else(|poisoned| poisoned.into_inner())
2563 .presented(connection, current_nonce.as_deref())
2564 };
2565 let mut table = self
2570 .scopes
2571 .write()
2572 .unwrap_or_else(|poisoned| poisoned.into_inner());
2573 let outcome = table.sync(&owner, connection_id, is_current_launch, generation, scopes);
2574 let drained = match &outcome {
2575 Ok(applied) => self
2576 .forwarding
2577 .publish_scope_changes(&applied.tag_changes)
2578 .map_err(RouterError::Forwarding)?,
2579 Err(_) => Vec::new(),
2580 };
2581 drop(table);
2582 match outcome {
2583 Ok(applied) => {
2584 let counts = ScopeOutcomeCounts::of(&applied.results);
2585 info!(
2586 owner = %owner,
2587 generation,
2588 records = applied.results.len(),
2589 created = counts.created,
2590 replaced = counts.replaced,
2591 updated = counts.updated,
2592 unchanged = counts.unchanged,
2593 refused = counts.refused,
2594 ended = applied.ended.len(),
2595 tag_changes = applied.tag_changes.len(),
2596 routes_closed = drained.len(),
2597 "scope sync accepted"
2598 );
2599 for refused in applied
2605 .results
2606 .iter()
2607 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2608 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2609 {
2610 warn!(
2611 owner = %owner,
2612 generation,
2613 scope_ref = %refused.scope_ref,
2614 scope_epoch = refused.scope_epoch,
2615 code = refused.code.as_deref().unwrap_or(""),
2616 "scope record refused"
2617 );
2618 }
2619 self.close_scope_drained_routes(drained);
2620 let response = ModuleControlResponseToModule::ScopeSync {
2621 generation,
2622 results: applied.results,
2623 ended: applied.ended,
2624 };
2625 Ok(vec![control_response_body_frame(
2626 &frame,
2627 &response,
2628 "ModuleControlResponseToModule::ScopeSync",
2629 )?])
2630 }
2631 Err(refusal) => {
2632 info!(
2633 owner = %owner,
2634 generation,
2635 code = refusal.code,
2636 "scope sync refused"
2637 );
2638 Ok(vec![control_error_frame(
2639 &frame,
2640 refusal.code,
2641 refusal.message,
2642 )?])
2643 }
2644 }
2645 }
2646
2647 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2653 if drained.is_empty() {
2654 return;
2655 }
2656 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2657 BTreeMap::new();
2658 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2659 for route in drained {
2660 warn!(
2661 module_id = %route.module_id,
2662 reason = ?route.reason,
2663 client_connection_id = route.client.connection_id.get(),
2664 route_channel = route.client.channel,
2665 "closing route because its scope changed"
2666 );
2667 pushes
2668 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2669 .or_insert_with(|| (route.reason, Vec::new()))
2670 .1
2671 .push(EndpointRoute {
2672 goodbye_target: route.client.clone(),
2673 principal: Principal::Unverified,
2674 bound_at: Instant::now(),
2675 draining: false,
2676 drain_reason: None,
2677 });
2678 goodbyes.push(route.module);
2679 goodbyes.push(route.client);
2680 }
2681 for ((module_id, _), (reason, routes)) in pushes {
2682 send_route_control_pushes(
2683 &self.forwarding,
2684 routes,
2685 ClientControlPush::RouteClosed {
2686 module_id,
2687 channels: Vec::new(),
2688 reason,
2689 drained: false,
2690 abandoned: 0,
2691 excluded_subscriptions: 0,
2692 terminal: Some(false),
2693 },
2694 );
2695 }
2696 self.emit_route_goodbyes(goodbyes);
2697 }
2698
2699 fn handle_scope_describe(
2702 &self,
2703 connection_id: ConnectionId,
2704 frame: Frame,
2705 owner: Principal,
2706 scope_ref: String,
2707 ) -> Result<Vec<Frame>, RouterError> {
2708 let registered = self
2709 .registry
2710 .get_module_by_connection(connection_id)
2711 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2712 if registered.is_none() {
2713 return Ok(vec![control_error_frame(
2714 &frame,
2715 "not_registered",
2716 "scope.describe requires an active module registration owned by this connection",
2717 )?]);
2718 }
2719 let description = self
2720 .scopes
2721 .read()
2722 .unwrap_or_else(|poisoned| poisoned.into_inner())
2723 .describe(&owner, &scope_ref);
2724 let owner_configured = match &owner {
2725 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
2730 _ => false,
2731 };
2732 let response = ModuleControlResponseToModule::ScopeDescribe {
2733 status: description.status,
2734 scope_epoch: description.scope_epoch,
2735 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2736 owner_synced: description.owner_synced,
2737 owner_configured,
2738 scope: description.stamp,
2739 };
2740 Ok(vec![control_response_body_frame(
2741 &frame,
2742 &response,
2743 "ModuleControlResponseToModule::ScopeDescribe",
2744 )?])
2745 }
2746
2747 fn handle_catalog_update(
2748 &self,
2749 connection_id: ConnectionId,
2750 frame: Frame,
2751 provides: Vec<ProviderRole>,
2752 capabilities: Option<CapabilityDeclarations>,
2753 ready: Option<bool>,
2754 ) -> Result<Vec<Frame>, RouterError> {
2755 self.refresh_capability_requirements();
2756 let Some(registration) = self
2757 .registry
2758 .get_module_by_connection(connection_id)
2759 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2760 else {
2761 return Ok(vec![control_error_frame(
2762 &frame,
2763 "not_registered",
2764 "catalog.update requires an active module registration owned by this connection",
2765 )?]);
2766 };
2767
2768 if let Some(message) =
2769 catalog_update_frozen_field_message(®istration.manifest, &provides)
2770 {
2771 return Ok(vec![control_error_frame(
2772 &frame,
2773 "catalog_update_frozen_field",
2774 message,
2775 )?]);
2776 }
2777
2778 let mut candidate = registration.manifest.clone();
2779 candidate.provides = provides.clone();
2780 candidate.capabilities = capabilities
2781 .clone()
2782 .or_else(|| registration.manifest.capabilities.clone());
2783 if let Err(err) = candidate.validate_capability_grammar() {
2784 return Ok(vec![control_error_frame(
2785 &frame,
2786 "invalid_capability_grammar",
2787 err.to_string(),
2788 )?]);
2789 }
2790
2791 let mut conflicts = self
2794 .capability_evaluator
2795 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
2796 if let Some(conflict) = conflicts.first() {
2797 let message = format!(
2798 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2799 conflict.capability, conflict.claimants[0], candidate.module_id
2800 );
2801 for conflict in &mut conflicts {
2802 conflict.source = DuplicateClaimSource::CatalogUpdate;
2803 }
2804 log_duplicate_claim_events(conflicts);
2805 return Ok(vec![control_error_frame(
2806 &frame,
2807 "reserved_capability",
2808 message,
2809 )?]);
2810 }
2811
2812 let updated = self
2813 .registry
2814 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2815 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2816 if updated.is_none() {
2817 return Ok(vec![control_error_frame(
2818 &frame,
2819 "not_registered",
2820 "catalog.update requires an active module registration owned by this connection",
2821 )?]);
2822 }
2823 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2824 log_duplicate_claim_events(
2825 self.capability_evaluator
2826 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2827 );
2828 }
2829 if capability_census_trigger(
2830 registration.manifest.capabilities.as_ref(),
2831 updated
2832 .as_ref()
2833 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2834 ) {
2835 self.enforce_capability_denies();
2836 }
2837 self.refresh_capability_requirements();
2838
2839 let response = ModuleControlResponseToModule::CatalogUpdate {};
2840 control_response_body_frame(
2841 &frame,
2842 &response,
2843 "ModuleControlResponseToModule::CatalogUpdate",
2844 )
2845 .map(|frame| vec![frame])
2846 }
2847
2848 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2849 self.refresh_capability_requirements();
2850 let mut counters = self.counters.snapshot();
2856 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2857 self.forwarding.client_route_concentration(),
2858 counters.as_object_mut(),
2859 ) {
2860 obj.insert(
2861 "client_connections_with_routes".into(),
2862 connections_with_routes.into(),
2863 );
2864 obj.insert("max_routes_on_one_connection".into(), max.into());
2865 }
2866 if let (Some(open_breakers), Some(obj)) = (
2872 self.route_bind_breakers.open_snapshot(),
2873 counters.as_object_mut(),
2874 ) {
2875 obj.insert("route_bind_breakers_open".into(), open_breakers);
2876 }
2877 let response = ClientControlResponse::ServerDescribe {
2878 protocol_ver: PROTOCOL_VERSION,
2879 subc_ops: subc_ops(),
2880 capabilities: self.subc_capabilities.as_ref().to_vec(),
2881 connected_clients: self.connected_clients.count(),
2882 counters: Some(counters),
2883 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
2884 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
2885 capability_requirements: self.capability_requirement_statuses(),
2886 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2887 };
2888 Ok(vec![control_response_body_frame(
2889 &frame,
2890 &response,
2891 "ClientControlResponse::ServerDescribe",
2892 )?])
2893 }
2894
2895 fn handle_catalog_list(
2896 &self,
2897 frame: Frame,
2898 module_id: Option<String>,
2899 ) -> Result<Vec<Frame>, RouterError> {
2900 let (generation, modules) = self.registry.list_modules().map_err(|err| {
2901 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
2902 })?;
2903 let entries = modules
2904 .into_iter()
2905 .filter(|registration| {
2906 module_id
2907 .as_deref()
2908 .map(|wanted| registration.manifest.module_id == wanted)
2909 .unwrap_or(true)
2910 })
2911 .map(|registration| {
2912 let not_ready = self.not_ready_reason(®istration);
2913 let roles = registration.manifest.provides;
2914 CatalogEntry {
2915 module_id: registration.manifest.module_id,
2916 ready: not_ready.is_none(),
2917 not_ready,
2918 module_version: Some(registration.manifest.module_version),
2919 roles,
2920 control_ops: registration.control_ops,
2921 capabilities: registration.manifest.capabilities,
2922 self_signals: registration.manifest.self_signals,
2923 }
2924 })
2925 .collect();
2926 let response = ClientControlResponse::CatalogList {
2927 generation,
2928 modules: entries,
2929 subc_ops: subc_ops(),
2930 };
2931 Ok(vec![control_response_body_frame(
2932 &frame,
2933 &response,
2934 "ClientControlResponse::CatalogList",
2935 )?])
2936 }
2937
2938 fn route_open_principal(
2939 &self,
2940 frame: &Frame,
2941 consumer_identity: Option<ConsumerIdentity>,
2942 ) -> Result<Result<Principal, Frame>, RouterError> {
2943 let Some(consumer_identity) = consumer_identity else {
2944 return Ok(Ok(Principal::Direct));
2945 };
2946
2947 if self.supervisor.spawned_consumer_authorized(
2948 &consumer_identity.module_id,
2949 &consumer_identity.launch_nonce,
2950 ) {
2951 return Ok(Ok(Principal::Reserved {
2952 module_id: consumer_identity.module_id,
2953 }));
2954 }
2955
2956 Ok(Err(control_error_frame(
2957 frame,
2958 "bad_consumer_identity",
2959 format!(
2960 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
2961 consumer_identity.module_id
2962 ),
2963 )?))
2964 }
2965
2966 fn route_open_refusal_frame(
2973 &self,
2974 ctx: &RouteCtx,
2975 frame: &Frame,
2976 module_id: &str,
2977 reason: &'static str,
2978 code: &'static str,
2979 message: impl Into<String>,
2980 ) -> Result<Frame, RouterError> {
2981 self.observe_route_open_refusal(ctx, module_id, reason, code);
2982 control_error_frame(frame, code, message.into())
2983 }
2984
2985 fn route_open_breaker_refusal_frame(
3003 &self,
3004 ctx: &RouteCtx,
3005 frame: &Frame,
3006 module_id: &str,
3007 consecutive_timeouts: u32,
3008 retry_in: Duration,
3009 probe_in_flight: bool,
3010 ) -> Result<Frame, RouterError> {
3011 self.counters
3012 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3013 debug!(
3014 target: "control",
3015 code = "module_timeout",
3016 module_id = ?module_id,
3017 connection_id = ctx.connection_id.get(),
3018 consecutive_timeouts,
3019 retry_in_ms = retry_in.as_millis() as u64,
3020 probe_in_flight,
3021 "route.open refused by open bind-relay breaker"
3022 );
3023 let detail = if probe_in_flight {
3030 "one test route is already being tried; retry once it settles".to_string()
3031 } else {
3032 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3033 };
3034 control_error_frame(
3035 frame,
3036 "module_timeout",
3037 format!(
3038 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3039 timed out in a row); {detail}; its established routes are unaffected"
3040 ),
3041 )
3042 }
3043
3044 fn observe_route_open_refusal(
3057 &self,
3058 ctx: &RouteCtx,
3059 module_id: &str,
3060 reason: &'static str,
3061 code: &'static str,
3062 ) {
3063 self.counters.increment_route_open_refused(code);
3064 info!(
3065 target: "control",
3066 code,
3067 reason,
3068 module_id = ?module_id,
3069 connection_id = ctx.connection_id.get(),
3070 "route.open refused"
3071 );
3072 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3073 self.route_outages.record_not_serving(module_id, reason);
3074 }
3075 }
3076
3077 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3122 self.route_outages.record_accepted(module_id);
3123 self.counters.increment_route_open_accepted(principal);
3124 info!(
3125 target: "control",
3126 principal,
3127 module_id,
3128 connection_id = ctx.connection_id.get(),
3129 "route.open accepted"
3130 );
3131 }
3132
3133 fn supervised_absent_route_open_refusal_frame(
3134 &self,
3135 ctx: &RouteCtx,
3136 frame: &Frame,
3137 module_id: &str,
3138 code: &'static str,
3139 status: &crate::supervise::ModuleStatus,
3140 ) -> Result<Frame, RouterError> {
3141 self.counters.increment_route_open_refused(code);
3142 info!(
3143 target: "control",
3144 code,
3145 reason = "supervised_not_registered",
3146 module_id = ?module_id,
3147 connection_id = ctx.connection_id.get(),
3148 state = %status.state,
3149 enabled = status.enabled,
3150 live = status.live,
3151 "route.open refused"
3152 );
3153 self.route_outages
3157 .record_not_serving(module_id, "supervised_not_registered");
3158 control_error_frame(
3159 frame,
3160 code,
3161 format!(
3162 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3163 status.state, status.enabled, status.live
3164 ),
3165 )
3166 }
3167
3168 async fn handle_route_open(
3169 &self,
3170 ctx: &RouteCtx,
3171 frame: Frame,
3172 request: RouteOpenRequest,
3173 ) -> Result<Vec<Frame>, RouterError> {
3174 let RouteOpenRequest {
3175 target,
3176 mut identity,
3177 consumer_identity,
3178 consumer_capabilities,
3179 role_versions,
3180 admission_facts,
3181 scope,
3182 } = request;
3183 let target_module_id = target_module_id(&target).to_string();
3184 if self
3185 .registry
3186 .get_module_by_connection(ctx.connection_id)
3187 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3188 .is_some()
3189 {
3190 return Ok(vec![control_error_frame(
3191 &frame,
3192 "invalid_request",
3193 "module connections cannot open client routes",
3194 )?]);
3195 }
3196 debug!(
3197 connection_id = ctx.connection_id.get(),
3198 corr = frame.header.corr,
3199 module_id = %target_module_id,
3200 "handling route.open"
3201 );
3202
3203 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3209 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3210 self.observe_route_open_refusal(
3211 ctx,
3212 &target_module_id,
3213 "invalid_role_versions",
3214 error_codes::INVALID_REQUEST,
3215 );
3216 return Ok(vec![control_error_body_frame(
3217 &frame,
3218 ErrorBody {
3219 code: error_codes::INVALID_REQUEST.to_string(),
3220 message: error.to_string(),
3221 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3222 },
3223 )?]);
3224 }
3225
3226 let Some(registration) = self
3245 .registry
3246 .get_module(&target_module_id)
3247 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3248 else {
3249 if let Some((status, warming)) =
3250 self.supervisor_status(&target_module_id, frame.header.corr)?
3251 {
3252 if status.protocol == ModuleProtocol::None {
3259 return Ok(vec![self.route_open_refusal_frame(
3260 ctx,
3261 &frame,
3262 &target_module_id,
3263 "protocol_none",
3264 error_codes::MODULE_NO_PROTOCOL,
3265 format!(
3266 "module_id '{target_module_id}' is declared protocol: none; \
3267 it speaks no subc wire and serves no routes"
3268 ),
3269 )?]);
3270 }
3271 let code = if warming {
3272 "module_warming"
3273 } else {
3274 "target_unavailable"
3275 };
3276 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3277 ctx,
3278 &frame,
3279 &target_module_id,
3280 code,
3281 &status,
3282 )?]);
3283 }
3284 if let Some(removed_ago_ms) =
3285 self.supervisor.removal_tombstone_age_ms(&target_module_id)
3286 {
3287 return Ok(vec![self.route_open_refusal_frame(
3288 ctx,
3289 &frame,
3290 &target_module_id,
3291 "removed",
3292 error_codes::MODULE_REMOVED,
3293 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3294 )?]);
3295 }
3296 return Ok(vec![self.route_open_refusal_frame(
3297 ctx,
3298 &frame,
3299 &target_module_id,
3300 "not_registered",
3301 error_codes::UNKNOWN_MODULE,
3302 format!("module_id '{target_module_id}' is not registered"),
3303 )?]);
3304 };
3305
3306 if !registration.ready {
3310 self.counters
3311 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3312 info!(
3313 target: "control",
3314 code = error_codes::MODULE_WARMING,
3315 module_id = ?target_module_id,
3316 connection_id = ctx.connection_id.get(),
3317 reason = "declared_not_ready",
3318 "route.open refused"
3319 );
3320 self.route_outages
3323 .record_not_serving(&target_module_id, "declared_not_ready");
3324 return Ok(vec![control_error_body_frame(
3325 &frame,
3326 ErrorBody {
3327 code: error_codes::MODULE_WARMING.to_string(),
3328 message: format!(
3329 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3330 ),
3331 detail: Some(serde_json::json!({
3332 "reason": "declared_not_ready"
3333 })),
3334 },
3335 )?]);
3336 }
3337
3338 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3357 self.counters
3358 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3359 info!(
3360 target: "control",
3361 code = error_codes::MODULE_WARMING,
3362 module_id = ?target_module_id,
3363 connection_id = ctx.connection_id.get(),
3364 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3365 capability = %capability,
3366 "route.open refused"
3367 );
3368 return Ok(vec![control_error_body_frame(
3369 &frame,
3370 ErrorBody {
3371 code: error_codes::MODULE_WARMING.to_string(),
3372 message: format!(
3373 "module_id '{target_module_id}' requires capability '{capability}', \
3374 which no registered module provides; retry"
3375 ),
3376 detail: Some(serde_json::json!({
3377 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3378 "capability": capability,
3379 })),
3380 },
3381 )?]);
3382 }
3383
3384 if !target_has_required_role(&target, ®istration.manifest.provides) {
3385 return Ok(vec![self.route_open_refusal_frame(
3386 ctx,
3387 &frame,
3388 &target_module_id,
3389 "role_not_provided",
3390 "target_unavailable",
3391 format!("module_id '{target_module_id}' does not provide the requested target"),
3392 )?]);
3393 }
3394
3395 if registration.state != ChannelState::Active {
3396 return Ok(vec![self.route_open_refusal_frame(
3397 ctx,
3398 &frame,
3399 &target_module_id,
3400 "registration_not_active",
3401 "target_unavailable",
3402 format!("module_id '{target_module_id}' is not active"),
3403 )?]);
3404 }
3405
3406 if self
3407 .forwarding
3408 .module_is_draining(&target_module_id)
3409 .map_err(RouterError::Forwarding)?
3410 {
3411 return Ok(vec![self.route_open_refusal_frame(
3412 ctx,
3413 &frame,
3414 &target_module_id,
3415 "reloading",
3416 "module_reloading",
3417 format!("module_id '{target_module_id}' is reloading"),
3418 )?]);
3419 }
3420
3421 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3422 process_liveness.process_live(&target_module_id) == Some(false)
3423 }) {
3424 if process_liveness.process_replacing(&target_module_id) {
3431 return Ok(vec![self.route_open_refusal_frame(
3432 ctx,
3433 &frame,
3434 &target_module_id,
3435 "reloading",
3436 "module_reloading",
3437 format!("module_id '{target_module_id}' is reloading"),
3438 )?]);
3439 }
3440 return Ok(vec![self.route_open_refusal_frame(
3441 ctx,
3442 &frame,
3443 &target_module_id,
3444 "supervisor_not_live",
3445 "target_unavailable",
3446 format!("module_id '{target_module_id}' is not live"),
3447 )?]);
3448 }
3449
3450 if !self
3451 .forwarding
3452 .has_live_module_connection(&target_module_id)
3453 .map_err(RouterError::Forwarding)?
3454 {
3455 return Ok(vec![self.route_open_refusal_frame(
3456 ctx,
3457 &frame,
3458 &target_module_id,
3459 "no_forwarding_connection",
3460 "target_unavailable",
3461 format!("module_id '{target_module_id}' has no live forwarding connection"),
3462 )?]);
3463 }
3464
3465 if let Some(error) =
3466 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3467 {
3468 self.observe_route_open_refusal(
3469 ctx,
3470 &target_module_id,
3471 "op_not_allowed",
3472 "op_not_allowed",
3473 );
3474 return Ok(vec![error]);
3475 }
3476
3477 let principal = match self.route_open_principal(&frame, consumer_identity)? {
3478 Ok(principal) => principal,
3479 Err(error) => {
3480 self.observe_route_open_refusal(
3481 ctx,
3482 &target_module_id,
3483 "bad_consumer_identity",
3484 "bad_consumer_identity",
3485 );
3486 return Ok(vec![error]);
3487 }
3488 };
3489
3490 if let Principal::Reserved {
3494 module_id: opening_module_id,
3495 } = &principal
3496 {
3497 if let Some(opening_registration) = self
3498 .registry
3499 .get_module(opening_module_id)
3500 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3501 {
3502 if let Some(capability) =
3503 denied_capability(&opening_registration.manifest, ®istration.manifest)
3504 {
3505 warn!(
3506 opening_module_id,
3507 target_module_id,
3508 capability,
3509 "refusing route.open because an attested capability deny edge matches"
3510 );
3511 return Ok(vec![self.route_open_refusal_frame(
3512 ctx,
3513 &frame,
3514 &target_module_id,
3515 "capability_deny_edge",
3516 "capability_forbidden",
3517 format!(
3518 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3519 ),
3520 )?]);
3521 }
3522 }
3523 }
3524
3525 if admission_facts.is_some() {
3526 let carrier_matches = matches!(
3527 &principal,
3528 Principal::Reserved { module_id }
3529 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3530 );
3531 if !carrier_matches {
3532 return Ok(vec![self.route_open_refusal_frame(
3533 ctx,
3534 &frame,
3535 &target_module_id,
3536 "admission_facts_carrier_not_permitted",
3537 "admission_facts_not_permitted",
3538 "admission facts may only be carried by the configured reserved module",
3539 )?]);
3540 }
3541
3542 let target_allowed = self
3543 .admission_facts_targets
3544 .as_ref()
3545 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3546 if !target_allowed {
3547 return Ok(vec![self.route_open_refusal_frame(
3548 ctx,
3549 &frame,
3550 &target_module_id,
3551 "admission_facts_target_not_listed",
3552 "admission_facts_target_not_allowed",
3553 format!(
3554 "admission facts are not permitted for target module_id '{target_module_id}'"
3555 ),
3556 )?]);
3557 }
3558
3559 }
3563
3564 let (bound_scope, scope_stamp) = match scope {
3570 None => (None, None),
3571 Some(selector) => {
3572 let owner_configured = match &selector.owner {
3573 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3579 _ => false,
3580 };
3581 let admitted = self
3582 .scopes
3583 .read()
3584 .unwrap_or_else(|poisoned| poisoned.into_inner())
3585 .admit(&principal, &target_module_id, &selector, owner_configured)
3586 .and_then(|admission| {
3587 crate::scopes::check_target_flow_support(
3588 &admission.stamp,
3589 &target_module_id,
3590 registration.manifest.capabilities.as_ref(),
3591 )?;
3592 Ok(admission)
3593 });
3594 match admitted {
3595 Ok(admission) => (
3596 Some(BoundScope {
3597 owner: admission.owner,
3598 scope_ref: admission.stamp.scope_ref.clone(),
3599 tag: admission.tag,
3600 }),
3601 Some(admission.stamp),
3602 ),
3603 Err(refusal) => {
3604 return Ok(vec![self.route_open_refusal_frame(
3605 ctx,
3606 &frame,
3607 &target_module_id,
3608 refusal.code,
3609 refusal.code,
3610 refusal.message,
3611 )?]);
3612 }
3613 }
3614 }
3615 };
3616
3617 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3639 Ok(project_root) => project_root,
3640 Err(err) => {
3641 return Ok(vec![control_error_frame(
3642 &frame,
3643 "invalid_project_root",
3644 err.to_string(),
3645 )?])
3646 }
3647 };
3648 identity.project_root = project_root.as_path().to_path_buf();
3649
3650 let _concurrency_guard = match self
3661 .route_bind_concurrency
3662 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3663 {
3664 Ok(guard) => guard,
3665 Err(in_flight) => {
3666 return Ok(vec![self.route_open_target_capacity_refusal(
3667 ctx,
3668 &frame,
3669 &target_module_id,
3670 in_flight,
3671 )?]);
3672 }
3673 };
3674
3675 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3678 RouteBindAdmission::Admitted { guard, probe } => {
3679 if probe {
3680 info!(
3681 module_id = %target_module_id,
3682 connection_id = ctx.connection_id.get(),
3683 "route.bind breaker half-open: admitting one probe"
3684 );
3685 }
3686 guard
3687 }
3688 RouteBindAdmission::Refused {
3689 consecutive_timeouts,
3690 retry_in,
3691 probe_in_flight,
3692 } => {
3693 return Ok(vec![self.route_open_breaker_refusal_frame(
3694 ctx,
3695 &frame,
3696 &target_module_id,
3697 consecutive_timeouts,
3698 retry_in,
3699 probe_in_flight,
3700 )?]);
3701 }
3702 };
3703
3704 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3709 let relay_deadline = Instant::now() + route_bind_relay_timeout;
3710 let pending = match self
3711 .forwarding
3712 .begin_route_bind_relay_for(
3713 ctx.connection_id,
3714 ctx.egress.clone(),
3715 response_version(&frame),
3716 frame.header.corr,
3717 &target_module_id,
3718 principal.clone(),
3719 bound_scope,
3720 Some(project_root),
3721 relay_deadline,
3722 )
3723 .await
3724 {
3725 Ok(pending) => pending,
3726 Err(err) => {
3727 return Ok(vec![self.route_open_refusal_frame(
3728 ctx,
3729 &frame,
3730 &target_module_id,
3731 "relay_reservation_failed",
3732 forwarding_error_code(&err),
3733 err.to_string(),
3734 )?])
3735 }
3736 };
3737 let crate::forwarding::PendingRouteBindRelay {
3738 endpoint,
3739 module_sink,
3740 negotiated_ver,
3741 client_channel,
3742 client_epoch,
3743 module_channel,
3744 module_epoch,
3745 corr: relay_corr,
3746 receiver,
3747 } = pending;
3748 let mut reservation =
3749 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3750
3751 if let Some(stamp) = scope_stamp
3756 .as_ref()
3757 .filter(|stamp| stamp.attributes.flow_id.is_some())
3758 {
3759 let relay_registration = self
3760 .registry
3761 .get_module_by_connection(endpoint.connection_id)
3762 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3763 if let Err(refusal) = crate::scopes::check_target_flow_support(
3764 stamp,
3765 &target_module_id,
3766 relay_registration
3767 .as_ref()
3768 .and_then(|registration| registration.manifest.capabilities.as_ref()),
3769 ) {
3770 reservation.release_and_disarm();
3771 return Ok(vec![self.route_open_refusal_frame(
3772 ctx,
3773 &frame,
3774 &target_module_id,
3775 refusal.code,
3776 refusal.code,
3777 refusal.message,
3778 )?]);
3779 }
3780 }
3781
3782 debug!(
3783 connection_id = ctx.connection_id.get(),
3784 client_channel,
3785 client_epoch,
3786 module_channel,
3787 module_epoch,
3788 "reserved route handle pair"
3789 );
3790 let principal_label = principal_label(&principal);
3793 let relay = ModuleControlRequest::RouteBind {
3794 route_channel: module_channel,
3795 epoch: module_epoch,
3796 target,
3797 identity,
3798 principal: Some(principal),
3799 consumer_capabilities,
3800 role_versions,
3801 admission_facts,
3802 scope: scope_stamp,
3803 };
3804 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3805 RouterError::backend(
3806 0,
3807 frame.header.corr,
3808 format!("failed to encode route.bind request: {err}"),
3809 )
3810 })?;
3811 let relay_frame = Frame::build_with_version(
3812 negotiated_ver,
3813 FrameType::Request,
3814 control_flags(),
3815 0,
3816 0,
3817 relay_corr,
3818 relay_body,
3819 )
3820 .map_err(RouterError::FrameBuild)?;
3821
3822 if let Err(err) = module_sink.send(relay_frame).await {
3823 reservation.release_and_disarm();
3824 return Ok(vec![self.route_open_refusal_frame(
3825 ctx,
3826 &frame,
3827 &target_module_id,
3828 "relay_send_failed",
3829 "target_unavailable",
3830 err.to_string(),
3831 )?]);
3832 }
3833
3834 if !self
3835 .forwarding
3836 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3837 .map_err(RouterError::Forwarding)?
3838 {
3839 self.send_abandoned_route_bind_goodbye(
3840 &module_sink,
3841 negotiated_ver,
3842 module_channel,
3843 module_epoch,
3844 );
3845 }
3846
3847 match timeout_at(relay_deadline, receiver).await {
3848 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3849 reservation.disarm();
3850 if breaker.record_accepted() {
3851 info!(
3852 module_id = %target_module_id,
3853 "route.bind breaker closed: the probe was accepted"
3854 );
3855 }
3856 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
3857 Ok(Vec::new())
3858 }
3859 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
3860 reservation.release_and_disarm();
3861 breaker.record_inconclusive();
3865 let scope_code = match body.code.as_str() {
3869 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
3870 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
3871 _ => None,
3872 };
3873 if let Some(code) = scope_code {
3874 self.observe_route_open_refusal(
3875 ctx,
3876 &target_module_id,
3877 "scope_changed_before_commit",
3878 code,
3879 );
3880 return Ok(vec![control_error_body_frame(&frame, body)?]);
3881 }
3882 self.counters
3883 .increment_route_open_refused("module_rejected");
3884 info!(
3885 target: "control",
3886 code = "module_rejected",
3887 module_code = ?body.code,
3888 module_id = ?target_module_id,
3889 connection_id = ctx.connection_id.get(),
3890 "route.open refused"
3891 );
3892 Ok(vec![control_error_body_frame(&frame, body)?])
3893 }
3894 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
3895 reservation.release_and_disarm();
3896 breaker.record_inconclusive();
3897 tracing::warn!(
3904 module_id = %target_module_id,
3905 "route.bind relay abandoned: {message}"
3906 );
3907 Ok(vec![self.route_open_refusal_frame(
3908 ctx,
3909 &frame,
3910 &target_module_id,
3911 "relay_abandoned",
3912 "target_unavailable",
3913 message,
3914 )?])
3915 }
3916 Ok(Err(_)) => {
3917 reservation.release_and_disarm();
3918 breaker.record_inconclusive();
3919 Ok(vec![self.route_open_refusal_frame(
3920 ctx,
3921 &frame,
3922 &target_module_id,
3923 "relay_waiter_canceled",
3924 "target_unavailable",
3925 "route.bind relay waiter was canceled before the module responded",
3926 )?])
3927 }
3928 Err(_) => {
3929 reservation.release_and_disarm();
3930 if let Some(opened) = breaker.record_timeout(
3934 self.route_bind_breaker_threshold,
3935 self.route_bind_breaker_cooldown,
3936 ) {
3937 warn!(
3938 module_id = %target_module_id,
3939 consecutive_timeouts = opened.consecutive_timeouts,
3940 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
3941 reopened_after_probe = opened.reopened_after_probe,
3942 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
3943 );
3944 }
3945 tracing::warn!(
3953 module_id = %target_module_id,
3954 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
3955 "route.bind relay timed out: module did not ack within budget"
3956 );
3957 Ok(vec![self.route_open_refusal_frame(
3958 ctx,
3959 &frame,
3960 &target_module_id,
3961 "relay_timed_out",
3962 "module_timeout",
3963 format!(
3964 "module_id '{target_module_id}' did not answer route.bind within {:?}",
3965 route_bind_relay_timeout
3966 ),
3967 )?])
3968 }
3969 }
3970 }
3971
3972 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3973 let response = ClientControlResponse::SupervisorSpawnSnapshot {
3974 snapshot: self.supervisor.spawn_snapshot(),
3975 };
3976 Ok(vec![control_response_body_frame(
3977 &frame,
3978 &response,
3979 "ClientControlResponse::SupervisorSpawnSnapshot",
3980 )?])
3981 }
3982
3983 fn handle_supervisor_spawn_subscribe(
3984 &self,
3985 ctx: &RouteCtx,
3986 frame: Frame,
3987 since: Option<SpawnCursor>,
3988 ) -> Result<Vec<Frame>, RouterError> {
3989 match self.supervisor.subscribe_spawns(
3990 ctx.connection_id,
3991 frame.header.corr,
3992 response_version(&frame),
3993 since,
3994 ctx.egress.clone(),
3995 ) {
3996 Ok(()) => Ok(Vec::new()),
3997 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
3998 Ok(vec![control_error_body_frame(
3999 &frame,
4000 ErrorBody {
4001 code: "spawn_cursor_incarnation_mismatch".to_string(),
4002 message: "spawn cursor belongs to a different daemon incarnation"
4003 .to_string(),
4004 detail: Some(serde_json::json!({
4005 "current_daemon_incarnation": current
4006 })),
4007 },
4008 )?])
4009 }
4010 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4011 &frame,
4012 ErrorBody {
4013 code: "spawn_cursor_too_old".to_string(),
4014 message: "spawn cursor predates the retained event ring".to_string(),
4015 detail: Some(serde_json::json!({
4016 "oldest_retained_cursor": oldest
4017 })),
4018 },
4019 )?]),
4020 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4021 0,
4022 frame.header.corr,
4023 format!("failed to open supervisor spawn subscription: {error}"),
4024 )),
4025 }
4026 }
4027
4028 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4029 let generation = self
4030 .registry
4031 .generation()
4032 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4033 let mut modules = Vec::new();
4034 for module in self.supervisor.list() {
4035 let status = module.status_for_control("list").map_err(|err| {
4036 RouterError::backend(
4037 0,
4038 frame.header.corr,
4039 format!("failed to read supervisor status: {err}"),
4040 )
4041 })?;
4042 let (configured, _) = module.configuration().map_err(|err| {
4043 RouterError::backend(
4044 0,
4045 frame.header.corr,
4046 format!("failed to read module configuration: {err}"),
4047 )
4048 })?;
4049 let image = module.running_image_agreement().await;
4051 let resources = Some(module.child_resource_usage());
4054 let pending_reload = Some(reload_verdict(
4055 &configured.program,
4056 status.spawned_from.as_deref(),
4057 image,
4058 ));
4059 modules.push(SupervisorEntry {
4060 launch_nonce_env: Some(!cfg!(unix)),
4063 module_id: status.module_id,
4064 state: status.state.to_string(),
4065 enabled: status.enabled,
4066 live: status.live,
4067 protocol: status.protocol,
4068 health: status.health.status,
4069 pending_reload,
4070 last_probe_ms: status.health.last_probe_ms,
4071 last_exit_code: status.last_exit.as_ref().and_then(|e| e.code),
4072 last_exit_signal: status.last_exit.as_ref().and_then(|e| e.signal),
4073 last_exit_ms: status.last_exit.as_ref().map(|e| e.at_ms),
4074 last_exit_kind: status.last_exit.as_ref().map(|e| e.kind.into()),
4075 restart_count: Some(status.restart_count),
4076 max_restarts: Some(status.max_restarts),
4077 lifetime_restarts: Some(status.lifetime_restarts),
4078 spawn_generation: Some(status.spawn_generation),
4079 restart_window_secs: Some(status.restart_window.as_secs()),
4080 drain_timeout_ms: Some(status.drain_timeout.as_millis() as u64),
4081 restart_backoff_ms: Some(status.restart_backoff.as_millis() as u64),
4082 restart_max_backoff_ms: Some(status.restart_max_backoff.as_millis() as u64),
4083 resources,
4084 });
4085 }
4086 let response = ClientControlResponse::SupervisorList {
4087 generation,
4088 modules,
4089 };
4090 Ok(vec![control_response_body_frame(
4091 &frame,
4092 &response,
4093 "ClientControlResponse::SupervisorList",
4094 )?])
4095 }
4096
4097 fn handle_supervisor_stderr_tail(
4098 &self,
4099 frame: Frame,
4100 module_id: String,
4101 max_lines: Option<u32>,
4102 max_bytes: Option<u32>,
4103 ) -> Result<Vec<Frame>, RouterError> {
4104 let Some(module) = self.supervisor.get(&module_id) else {
4105 return Ok(vec![control_error_frame(
4106 &frame,
4107 "unknown_module",
4108 format!("module_id '{module_id}' is not supervised"),
4109 )?]);
4110 };
4111
4112 let snapshot = module.stderr_tail(
4113 max_lines.map(|value| value as usize),
4114 max_bytes.map(|value| value as usize),
4115 );
4116
4117 let response = ClientControlResponse::SupervisorStderrTail {
4118 module_id,
4119 tail: StderrTail {
4120 capture: match snapshot.capture {
4121 CaptureState::Captured => StderrCaptureState::Captured,
4122 CaptureState::Incomplete { reason } => {
4123 StderrCaptureState::Incomplete { reason }
4124 }
4125 CaptureState::NotCaptured { reason } => {
4126 StderrCaptureState::NotCaptured { reason }
4127 }
4128 },
4129 entries: snapshot
4130 .entries
4131 .into_iter()
4132 .map(|entry| match entry {
4133 TailEntry::Line {
4134 text,
4135 truncated,
4136 at_ms,
4137 } => StderrTailEntry::Line {
4138 text,
4139 truncated,
4140 at_ms,
4141 },
4142 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
4143 })
4144 .collect(),
4145 dropped_lines: snapshot.dropped_lines,
4146 },
4147 };
4148 Ok(vec![control_response_body_frame(
4149 &frame,
4150 &response,
4151 "ClientControlResponse::SupervisorStderrTail",
4152 )?])
4153 }
4154
4155 async fn handle_supervisor_terminals(
4156 &self,
4157 frame: Frame,
4158 module_id: String,
4159 ) -> Result<Vec<Frame>, RouterError> {
4160 let Some(module) = self.supervisor.get(&module_id) else {
4161 return Ok(vec![control_error_frame(
4162 &frame,
4163 "unknown_module",
4164 format!("module_id '{module_id}' is not supervised"),
4165 )?]);
4166 };
4167
4168 let terminals = module
4171 .read_durable_terminal_history()
4172 .await
4173 .map_err(|error| {
4174 RouterError::backend(
4175 0,
4176 frame.header.corr,
4177 format!("failed to read terminal history: {error}"),
4178 )
4179 })?;
4180 let response = ClientControlResponse::SupervisorTerminals {
4181 module_id,
4182 terminals,
4183 };
4184 Ok(vec![control_response_body_frame(
4185 &frame,
4186 &response,
4187 "ClientControlResponse::SupervisorTerminals",
4188 )?])
4189 }
4190
4191 fn handle_supervisor_routes(
4192 &self,
4193 frame: Frame,
4194 module_id: Option<String>,
4195 ) -> Result<Vec<Frame>, RouterError> {
4196 let modules = self
4197 .forwarding
4198 .route_census(module_id.as_deref())
4199 .map_err(RouterError::Forwarding)?
4200 .into_iter()
4201 .map(|(module_id, routes)| SupervisorRouteModule {
4202 module_id,
4203 routes: routes
4204 .into_iter()
4205 .map(|route| SupervisorRoute {
4206 consumer: match route.principal {
4207 Principal::Reserved { module_id } => {
4208 SupervisorRouteConsumer::Reserved { module_id }
4209 }
4210 Principal::Direct | Principal::Unverified => {
4211 SupervisorRouteConsumer::Direct {
4212 connection_id: route.goodbye_target.connection_id.get(),
4213 }
4214 }
4215 },
4216 age_ms: Instant::now()
4217 .saturating_duration_since(route.bound_at)
4218 .as_millis()
4219 .try_into()
4220 .unwrap_or(u64::MAX),
4221 draining: route.draining,
4222 drain_reason: route.drain_reason,
4223 })
4224 .collect(),
4225 })
4226 .collect();
4227 let response = ClientControlResponse::SupervisorRoutes { modules };
4228 Ok(vec![control_response_body_frame(
4229 &frame,
4230 &response,
4231 "ClientControlResponse::SupervisorRoutes",
4232 )?])
4233 }
4234
4235 async fn handle_supervisor_provenance(
4236 &self,
4237 frame: Frame,
4238 module_id: Option<String>,
4239 ) -> Result<Vec<Frame>, RouterError> {
4240 let mut selected = if let Some(module_id) = module_id {
4241 let Some(module) = self.supervisor.get(&module_id) else {
4242 return Ok(vec![control_error_frame(
4243 &frame,
4244 "unknown_module",
4245 format!("module_id '{module_id}' is not supervised"),
4246 )?]);
4247 };
4248 vec![module]
4249 } else {
4250 self.supervisor.list()
4251 };
4252
4253 let mut modules = Vec::with_capacity(selected.len());
4254 for module in selected.drain(..) {
4255 let (status, observed_image) = module
4256 .status_and_running_image_agreement()
4257 .await
4258 .map_err(|err| {
4259 RouterError::backend(
4260 0,
4261 frame.header.corr,
4262 format!("failed to read supervisor status: {err}"),
4263 )
4264 })?;
4265 let module_declared = self
4266 .registry
4267 .get_module(&status.module_id)
4268 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4269 .and_then(|registration| registration.manifest.provenance)
4270 .map(|build| ModuleDeclaredProvenance::Reported { build })
4271 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4272 #[cfg(test)]
4273 let running_image = match &self.provenance_probe_override {
4274 Some(result) => result.clone(),
4275 None => observed_image,
4276 };
4277 #[cfg(not(test))]
4278 let running_image = observed_image;
4279 modules.push(SupervisorModuleProvenance {
4280 module_id: status.module_id,
4281 module_declared,
4282 daemon_observed: SupervisorObservedProcess {
4283 pid: status.pid,
4284 spawned_at_ms: status.spawned_at_ms,
4285 spawned_from: status.spawned_from,
4286 running_image,
4287 },
4288 });
4289 }
4290 let daemon = SupervisorDaemonProvenance {
4291 daemon_build: self.daemon_provenance.build.clone(),
4292 daemon_observed: DaemonObservedProcess {
4293 pid: self.daemon_provenance.pid,
4294 started_at_ms: self
4295 .daemon_provenance
4296 .start_clock
4297 .map(|clock| clock.started_at_ms())
4298 .or(self.daemon_provenance.started_at_ms),
4299 running_image: self
4300 .daemon_provenance
4301 .probe
4302 .observe(
4303 self.daemon_provenance.pid,
4304 self.daemon_provenance.executable_path.as_deref(),
4305 self.daemon_provenance.executable_identity,
4306 self.daemon_provenance.process_start_time,
4307 )
4308 .await,
4309 },
4310 };
4311 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
4312 Ok(vec![control_response_body_frame(
4313 &frame,
4314 &response,
4315 "ClientControlResponse::SupervisorProvenance",
4316 )?])
4317 }
4318
4319 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4320 self.refresh_capability_requirements();
4321 let generation = self
4322 .registry
4323 .generation()
4324 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4325 let modules = self
4326 .supervisor
4327 .list()
4328 .into_iter()
4329 .map(|module| {
4330 let status = module.status_for_control("health").map_err(|err| {
4331 RouterError::backend(
4332 0,
4333 frame.header.corr,
4334 format!("failed to read supervisor health: {err}"),
4335 )
4336 })?;
4337 let module_id = status.module_id;
4338 let capability_detail = self
4339 .capability_evaluator
4340 .required_problem_detail(&module_id);
4341 Ok(SupervisorHealthEntry {
4342 module_id,
4343 status: status.health.status,
4344 detail: append_capability_problem_detail(
4345 status.health.detail,
4346 capability_detail,
4347 ),
4348 metrics: status.health.metrics,
4349 consecutive_failures: status.health.consecutive_failures,
4350 late_answer_count: status.health.late_answer_count,
4351 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4352 last_action: status.health.last_action,
4353 last_action_ms: status.health.last_action_ms,
4354 last_probe_ms: status.health.last_probe_ms,
4355 })
4356 })
4357 .collect::<Result<Vec<_>, RouterError>>()?;
4358 let response = ClientControlResponse::SupervisorHealth {
4359 generation,
4360 modules,
4361 };
4362 Ok(vec![control_response_body_frame(
4363 &frame,
4364 &response,
4365 "ClientControlResponse::SupervisorHealth",
4366 )?])
4367 }
4368
4369 async fn handle_supervisor_restart(
4370 &self,
4371 frame: Frame,
4372 module_id: String,
4373 drain_timeout_ms: Option<u64>,
4374 ) -> Result<Vec<Frame>, RouterError> {
4375 let operation_lock = self.supervisor.operation_lock();
4376 let _operation_guard = operation_lock.lock().await;
4377 let Some(module) = self.supervisor.get(&module_id) else {
4378 return Ok(vec![control_error_frame(
4379 &frame,
4380 "unknown_module",
4381 format!("module_id '{module_id}' is not supervised"),
4382 )?]);
4383 };
4384
4385 self.route_outages.mark_operator_action(&module_id);
4386 if let Err(err) = module.restart(drain_timeout_ms).await {
4387 self.route_outages
4388 .operator_action_ended_unrefused(&module_id);
4389 let (code, message) = match err {
4390 crate::supervise::SuperviseError::Disabled { .. } => {
4391 ("module_disabled", err.to_string())
4392 }
4393 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4394 ("swap_in_progress", err.to_string())
4395 }
4396 _ => (
4397 "target_unavailable",
4398 format!("failed to restart module_id '{module_id}': {err}"),
4399 ),
4400 };
4401 return Ok(vec![control_error_frame(&frame, code, message)?]);
4402 }
4403
4404 let response = ClientControlResponse::SupervisorAck {
4405 module_id,
4406 applied: true,
4407 };
4408 Ok(vec![control_response_body_frame(
4409 &frame,
4410 &response,
4411 "ClientControlResponse::SupervisorAck",
4412 )?])
4413 }
4414
4415 async fn handle_supervisor_swap(
4419 &self,
4420 frame: Frame,
4421 module_id: String,
4422 ready_timeout_ms: Option<u64>,
4423 ) -> Result<Vec<Frame>, RouterError> {
4424 let module = {
4431 let operation_lock = self.supervisor.operation_lock();
4432 let _operation_guard = operation_lock.lock().await;
4433 self.supervisor.get(&module_id)
4434 };
4435 let Some(module) = module else {
4436 return Ok(vec![control_error_frame(
4437 &frame,
4438 "unknown_module",
4439 format!("module_id '{module_id}' is not supervised"),
4440 )?]);
4441 };
4442
4443 self.route_outages.mark_operator_action(&module_id);
4444 if let Err(err) = module
4445 .swap(ready_timeout_ms.map(Duration::from_millis))
4446 .await
4447 {
4448 self.route_outages
4449 .operator_action_ended_unrefused(&module_id);
4450 use crate::supervise::SuperviseError;
4451 let message = err.to_string();
4452 let error = match err {
4453 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4454 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4455 code: "swap_refused".to_string(),
4456 message,
4457 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4458 },
4459 SuperviseError::SwapFailed {
4460 arm,
4461 candidate_exit,
4462 ..
4463 } => ErrorBody {
4464 code: "swap_failed".to_string(),
4465 message,
4466 detail: Some(serde_json::json!({
4467 "arm": arm.as_str(),
4468 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4469 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4470 })),
4471 },
4472 _ => ErrorBody::new(
4473 "target_unavailable",
4474 format!("failed to swap module_id '{module_id}': {message}"),
4475 ),
4476 };
4477 return Ok(vec![control_error_body_frame(&frame, error)?]);
4478 }
4479 self.route_outages
4483 .operator_action_ended_unrefused(&module_id);
4484
4485 let response = ClientControlResponse::SupervisorAck {
4486 module_id,
4487 applied: true,
4488 };
4489 Ok(vec![control_response_body_frame(
4490 &frame,
4491 &response,
4492 "ClientControlResponse::SupervisorAck",
4493 )?])
4494 }
4495
4496 async fn handle_supervisor_reload(
4497 &self,
4498 frame: Frame,
4499 module_id: String,
4500 ) -> Result<Vec<Frame>, RouterError> {
4501 let operation_lock = self.supervisor.operation_lock();
4502 let _operation_guard = operation_lock.lock().await;
4503 let Some(module) = self.supervisor.get(&module_id) else {
4504 return Ok(vec![control_error_frame(
4505 &frame,
4506 "unknown_module",
4507 format!("module_id '{module_id}' is not supervised"),
4508 )?]);
4509 };
4510
4511 self.route_outages.mark_operator_action(&module_id);
4512 if let Err(err) = module.reload().await {
4513 self.route_outages
4514 .operator_action_ended_unrefused(&module_id);
4515 let (code, message) = match err {
4516 crate::supervise::SuperviseError::Disabled { .. } => {
4517 ("module_disabled", err.to_string())
4518 }
4519 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4520 ("swap_in_progress", err.to_string())
4521 }
4522 _ => (
4523 "reload_failed",
4524 format!("failed to reload module_id '{module_id}': {err}"),
4525 ),
4526 };
4527 return Ok(vec![control_error_frame(&frame, code, message)?]);
4528 }
4529
4530 let response = ClientControlResponse::SupervisorAck {
4531 module_id,
4532 applied: true,
4533 };
4534 Ok(vec![control_response_body_frame(
4535 &frame,
4536 &response,
4537 "ClientControlResponse::SupervisorAck",
4538 )?])
4539 }
4540
4541 async fn handle_supervisor_rescan(
4542 &self,
4543 frame: Frame,
4544 preview: bool,
4545 ) -> Result<Vec<Frame>, RouterError> {
4546 let Some(context) = self.rescan.clone() else {
4547 return Ok(vec![control_error_frame(
4548 &frame,
4549 "rescan_unavailable",
4550 "the daemon was not started with a reloadable config path".to_string(),
4551 )?]);
4552 };
4553
4554 let operation_lock = self.supervisor.operation_lock();
4555 let _operation_guard = operation_lock.lock().await;
4556 let loaded = match crate::daemon_config::load(&context.config_path) {
4557 Ok(config) => config,
4558 Err(err) => {
4559 return Ok(vec![control_error_frame(
4560 &frame,
4561 "invalid_daemon_config",
4562 format!("supervisor rescan rejected daemon config: {err}"),
4563 )?])
4564 }
4565 };
4566 let Some(config) = loaded else {
4574 return Ok(vec![control_error_frame(
4575 &frame,
4576 "invalid_daemon_config",
4577 format!(
4578 "daemon config not found at {}; refusing to rescan (an absent config would \
4579 retire every supervised module)",
4580 context.config_path.display()
4581 ),
4582 )?]);
4583 };
4584 let (
4585 configured_port,
4586 storage_config,
4587 admission_facts_carrier_module_id,
4588 admission_facts_targets,
4589 scope_authority_owners,
4590 modules,
4591 reserved_capabilities,
4592 ) = (
4593 config.port,
4594 config.storage,
4595 config.admission_facts_carrier_module_id,
4596 config.admission_facts_targets,
4597 config.scope_authority_owners,
4598 config.modules,
4599 config.reserved_capabilities,
4600 );
4601
4602 let mut restart_required = Vec::new();
4611 for section in RestartRequiredSection::ALL {
4612 let changed = match section {
4613 RestartRequiredSection::Port => configured_port != context.configured_port,
4614 RestartRequiredSection::Storage => storage_config != context.storage_config,
4615 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4616 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4617 }
4618 RestartRequiredSection::AdmissionFactsTargets => {
4619 admission_facts_targets != context.admission_facts_targets
4620 }
4621 RestartRequiredSection::ScopeAuthorityOwners => {
4622 scope_authority_owners != context.scope_authority_owners
4623 }
4624 };
4625 if changed {
4626 restart_required.push(section.label().to_string());
4627 }
4628 }
4629 if !restart_required.is_empty() {
4630 warn!(
4631 config_path = %context.config_path.display(),
4632 sections = %restart_required.join(", "),
4633 "daemon config changed outside the modules section; restart the daemon to apply those changes"
4634 );
4635 }
4636
4637 for configured in &modules {
4638 if let Err(err) = validate_spec(&configured.module_spec()) {
4639 return Ok(vec![control_error_frame(
4640 &frame,
4641 "invalid_daemon_config",
4642 format!("supervisor rescan rejected daemon config: {err}"),
4643 )?]);
4644 }
4645 }
4646
4647 let configured_capabilities = modules
4648 .iter()
4649 .map(|module| (module.module_id.clone(), module.enabled))
4650 .collect::<Vec<_>>();
4651 let preview_capability_warnings = if preview {
4652 let (_, registrations) = self.runtime_capability_snapshot()?;
4653 let current_modules = self
4654 .supervisor
4655 .list()
4656 .into_iter()
4657 .map(|module| module.module_id().to_string())
4658 .collect::<BTreeSet<_>>();
4659 let resulting_modules = configured_capabilities.clone();
4660 let removed = current_modules
4661 .into_iter()
4662 .filter(|module_id| {
4663 !resulting_modules
4664 .iter()
4665 .any(|(configured_id, _)| configured_id == module_id)
4666 })
4667 .collect::<Vec<_>>();
4668 self.capability_evaluator.preview_removal_warnings(
4669 resulting_modules,
4670 &removed,
4671 ®istrations,
4672 )
4673 } else {
4674 Vec::new()
4675 };
4676 let result = match self
4677 .reconcile_supervised_modules(&context.supervisor, modules, preview)
4678 .await
4679 {
4680 Ok(result) => result,
4681 Err(message) => {
4682 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4683 }
4684 };
4685 if !preview {
4686 self.capability_evaluator
4687 .configure(configured_capabilities, reserved_capabilities);
4688 self.capability_evaluator.wake_deadline_loop();
4689 self.refresh_capability_requirements();
4690 }
4691 let mut result = result;
4692 result.restart_required = restart_required;
4693 result.capability_warnings = preview_capability_warnings;
4694 let response = ClientControlResponse::SupervisorRescan { result };
4695 Ok(vec![control_response_body_frame(
4696 &frame,
4697 &response,
4698 "ClientControlResponse::SupervisorRescan",
4699 )?])
4700 }
4701
4702 async fn handle_supervisor_release_reserved(
4703 &self,
4704 frame: Frame,
4705 module_id: String,
4706 ) -> Result<Vec<Frame>, RouterError> {
4707 let Some(context) = self.rescan.clone() else {
4708 return Ok(vec![control_error_frame(
4709 &frame,
4710 "release_unavailable",
4711 "reserved-id release requires a daemon started with a reloadable config path",
4712 )?]);
4713 };
4714 let operation_lock = self.supervisor.operation_lock();
4715 let _operation_guard = operation_lock.lock().await;
4716 let loaded = match crate::daemon_config::load(&context.config_path) {
4717 Ok(Some(config)) => config,
4718 Ok(None) => {
4719 return Ok(vec![control_error_frame(
4720 &frame,
4721 "invalid_daemon_config",
4722 format!(
4723 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4724 context.config_path.display()
4725 ),
4726 )?])
4727 }
4728 Err(err) => {
4729 return Ok(vec![control_error_frame(
4730 &frame,
4731 "invalid_daemon_config",
4732 format!("unable to verify reserved-id release against daemon config: {err}"),
4733 )?])
4734 }
4735 };
4736 if loaded
4737 .modules
4738 .iter()
4739 .any(|configured| configured.module_id == module_id)
4740 {
4741 return Ok(vec![control_error_frame(
4742 &frame,
4743 "reserved_module_configured",
4744 format!(
4745 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4746 ),
4747 )?]);
4748 }
4749 if !self.supervisor.release_retained_reserved_gate(&module_id) {
4750 return Ok(vec![control_error_frame(
4751 &frame,
4752 "reserved_gate_not_retained",
4753 format!(
4754 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4755 ),
4756 )?]);
4757 }
4758
4759 let response = ClientControlResponse::SupervisorAck {
4760 module_id,
4761 applied: true,
4762 };
4763 Ok(vec![control_response_body_frame(
4764 &frame,
4765 &response,
4766 "ClientControlResponse::SupervisorAck",
4767 )?])
4768 }
4769
4770 async fn reconcile_supervised_modules(
4779 &self,
4780 supervisor: &Supervisor,
4781 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4782 preview: bool,
4783 ) -> Result<SupervisorRescanResult, String> {
4784 let mut current = BTreeMap::new();
4785 for module in self.supervisor.list() {
4786 let (spec, health) = module.configuration().map_err(|err| {
4787 format!(
4788 "failed to read configuration for module_id '{}': {err}",
4789 module.module_id()
4790 )
4791 })?;
4792 let enabled = module
4793 .status()
4794 .map_err(|err| {
4795 format!(
4796 "failed to read status for module_id '{}': {err}",
4797 module.module_id()
4798 )
4799 })?
4800 .enabled;
4801 current.insert(
4802 module.module_id().to_string(),
4803 (module, spec, health, enabled),
4804 );
4805 }
4806 let configured = configured_modules
4807 .into_iter()
4808 .map(|module| (module.module_id.clone(), module))
4809 .collect::<BTreeMap<_, _>>();
4810
4811 let added = configured
4812 .keys()
4813 .filter(|module_id| !current.contains_key(*module_id))
4814 .cloned()
4815 .collect::<Vec<_>>();
4816 let removed = current
4817 .keys()
4818 .filter(|module_id| !configured.contains_key(*module_id))
4819 .cloned()
4820 .collect::<Vec<_>>();
4821 let mut changed_pending_reload = Vec::new();
4822 let mut configuration_changes = BTreeSet::new();
4823 let mut enabled_changes = BTreeSet::new();
4824 let mut unchanged = 0_u32;
4825
4826 for (module_id, configured_module) in &configured {
4827 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4828 else {
4829 continue;
4830 };
4831 let launch_changed = *current_spec != configured_module.module_spec();
4835 let configuration_changed =
4836 launch_changed || *current_health != configured_module.health;
4837 let enabled_changed = *current_enabled != configured_module.enabled;
4838 if configuration_changed {
4839 configuration_changes.insert(module_id.clone());
4840 }
4841 if launch_changed {
4842 changed_pending_reload.push(module_id.clone());
4843 }
4844 if enabled_changed {
4845 enabled_changes.insert(module_id.clone());
4846 }
4847 if !configuration_changed && !enabled_changed {
4848 unchanged = unchanged.saturating_add(1);
4849 }
4850 }
4851
4852 if preview {
4858 return Ok(SupervisorRescanResult {
4859 added,
4860 removed,
4861 changed_pending_reload,
4862 enabled_changes: enabled_changes.iter().cloned().collect(),
4863 unchanged,
4864 preview: true,
4865 restart_required: Vec::new(),
4869 capability_warnings: Vec::new(),
4870 });
4871 }
4872
4873 for module_id in &removed {
4874 let module = ¤t
4875 .get(module_id)
4876 .expect("removed module came from current supervisor state")
4877 .0;
4878 module.retire().await.map_err(|err| {
4879 format!("failed to retire module_id '{module_id}' during rescan: {err}")
4880 })?;
4881 self.supervisor.record_rescan_removal(module_id);
4915 self.supervisor.retire(module_id);
4916 self.route_outages.forget(module_id);
4917 }
4918
4919 for module_id in configured.keys() {
4920 let Some((module, _, _, _)) = current.get(module_id) else {
4921 continue;
4922 };
4923 let configured_module = configured
4924 .get(module_id)
4925 .expect("configured module id came from configured map");
4926 if configuration_changes.contains(module_id) {
4927 module
4928 .update_configuration(
4929 configured_module.module_spec(),
4930 configured_module.health.clone(),
4931 configured_module.drain_timeout_ms,
4932 )
4933 .await
4934 .map_err(|err| {
4935 format!(
4936 "failed to update module_id '{module_id}' configuration during rescan: {err}"
4937 )
4938 })?;
4939 }
4940 if enabled_changes.contains(module_id) {
4941 self.route_outages.mark_operator_action(module_id);
4944 module
4945 .set_enabled(configured_module.enabled)
4946 .await
4947 .map_err(|err| {
4948 self.route_outages.operator_action_ended_unrefused(module_id);
4949 format!(
4950 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
4951 configured_module.enabled
4952 )
4953 })?;
4954 }
4955 }
4956
4957 for module_id in &added {
4958 let configured_module = configured
4959 .get(module_id)
4960 .expect("added module id came from configured map");
4961 supervisor
4962 .supervise_configured_with_health(
4963 configured_module.module_spec(),
4964 configured_module.enabled,
4965 configured_module.health.clone(),
4966 configured_module.drain_timeout_ms,
4967 configured_module.restart,
4968 )
4969 .map_err(|err| {
4970 format!("failed to add module_id '{module_id}' during rescan: {err}")
4971 })?;
4972 }
4973
4974 Ok(SupervisorRescanResult {
4975 added,
4976 removed,
4977 changed_pending_reload,
4978 enabled_changes: enabled_changes.iter().cloned().collect(),
4979 unchanged,
4980 preview: false,
4981 restart_required: Vec::new(),
4984 capability_warnings: Vec::new(),
4985 })
4986 }
4987
4988 async fn handle_supervisor_set_enabled(
4989 &self,
4990 frame: Frame,
4991 module_id: String,
4992 enabled: bool,
4993 ) -> Result<Vec<Frame>, RouterError> {
4994 let operation_lock = self.supervisor.operation_lock();
4995 let _operation_guard = operation_lock.lock().await;
4996 let Some(module) = self.supervisor.get(&module_id) else {
4997 return Ok(vec![control_error_frame(
4998 &frame,
4999 "unknown_module",
5000 format!("module_id '{module_id}' is not supervised"),
5001 )?]);
5002 };
5003
5004 self.route_outages.mark_operator_action(&module_id);
5007 let applied = match module.set_enabled(enabled).await {
5008 Ok(applied) => applied,
5009 Err(err) => {
5010 self.route_outages
5011 .operator_action_ended_unrefused(&module_id);
5012 return Ok(vec![control_error_frame(
5013 &frame,
5014 "target_unavailable",
5015 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
5016 )?]);
5017 }
5018 };
5019 if !applied {
5020 self.route_outages
5023 .operator_action_ended_unrefused(&module_id);
5024 }
5025
5026 self.capability_evaluator.wake_deadline_loop();
5027 self.refresh_capability_requirements();
5028 let response = ClientControlResponse::SupervisorAck { module_id, applied };
5029 Ok(vec![control_response_body_frame(
5030 &frame,
5031 &response,
5032 "ClientControlResponse::SupervisorAck",
5033 )?])
5034 }
5035
5036 async fn handle_supervisor_health_probe(
5037 &self,
5038 frame: Frame,
5039 module_id: String,
5040 ) -> Result<Vec<Frame>, RouterError> {
5041 self.refresh_capability_requirements();
5042 let Some(registration) = self
5043 .registry
5044 .get_module(&module_id)
5045 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5046 else {
5047 return Ok(vec![control_error_frame(
5048 &frame,
5049 "unknown_module",
5050 format!("module_id '{module_id}' is not registered"),
5051 )?]);
5052 };
5053
5054 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
5067 {
5068 return Ok(vec![control_error_frame(
5069 &frame,
5070 "health_not_advertised",
5071 format!("module_id '{module_id}' did not advertise health.check"),
5072 )?]);
5073 }
5074
5075 let deadline = Instant::now() + self.health_probe_timeout;
5076 let pending = match self.forwarding.begin_module_control_rpc_for(
5077 &module_id,
5078 MODULE_CONTROL_OP_HEALTH_CHECK,
5079 deadline,
5080 ) {
5081 Ok(pending) => pending,
5082 Err(err) => {
5083 return Ok(vec![control_error_frame(
5084 &frame,
5085 forwarding_error_code(&err),
5086 err.to_string(),
5087 )?])
5088 }
5089 };
5090
5091 let PendingModuleControlRpc {
5092 endpoint,
5093 module_sink,
5094 negotiated_ver,
5095 corr: probe_corr,
5096 receiver,
5097 } = pending;
5098 let mut guard =
5099 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
5100 let probe_body =
5101 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5102 RouterError::backend(
5103 0,
5104 frame.header.corr,
5105 format!("failed to encode health.check request: {err}"),
5106 )
5107 })?;
5108 let probe_frame = Frame::build_with_version(
5109 negotiated_ver,
5110 FrameType::Request,
5111 control_flags(),
5112 0,
5113 0,
5114 probe_corr,
5115 probe_body,
5116 )
5117 .map_err(RouterError::FrameBuild)?;
5118
5119 if let Err(err) = module_sink.send(probe_frame).await {
5120 return Ok(vec![control_error_frame(
5121 &frame,
5122 "target_unavailable",
5123 err.to_string(),
5124 )?]);
5125 }
5126
5127 match timeout_at(deadline, receiver).await {
5128 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5129 guard.disarm();
5130 let Some(report) = response.health_report() else {
5131 return Ok(vec![control_error_frame(
5132 &frame,
5133 "invalid_control_body",
5134 "health.check RPC returned a non-health response",
5135 )?]);
5136 };
5137 let HealthReport {
5142 status,
5143 detail,
5144 metrics,
5145 } = report;
5146 let capability_detail = self
5147 .capability_evaluator
5148 .required_problem_detail(&module_id);
5149 let response = ClientControlResponse::SupervisorHealthProbe {
5150 module_id,
5151 status,
5152 detail: append_capability_problem_detail(detail, capability_detail),
5153 metrics,
5154 };
5155 Ok(vec![control_response_body_frame(
5156 &frame,
5157 &response,
5158 "ClientControlResponse::SupervisorHealthProbe",
5159 )?])
5160 }
5161 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
5162 guard.disarm();
5163 Ok(vec![control_error_body_frame(&frame, body)?])
5164 }
5165 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5166 guard.disarm();
5167 Ok(vec![control_error_frame(
5168 &frame,
5169 "target_unavailable",
5170 message,
5171 )?])
5172 }
5173 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5174 guard.disarm();
5175 Ok(vec![control_error_frame(
5176 &frame,
5177 "invalid_control_body",
5178 message,
5179 )?])
5180 }
5181 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5182 guard.disarm();
5183 Ok(vec![control_error_frame(
5184 &frame,
5185 "invalid_control_body",
5186 format!("expected module-control op '{expected}', got '{actual}'"),
5187 )?])
5188 }
5189 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
5190 guard.disarm();
5191 Ok(vec![control_error_frame(
5192 &frame,
5193 "module_timeout",
5194 format!(
5195 "module_id '{module_id}' answered health.check after {:?}",
5196 self.health_probe_timeout
5197 ),
5198 )?])
5199 }
5200 Ok(Err(_)) => Ok(vec![control_error_frame(
5201 &frame,
5202 "target_unavailable",
5203 "health.check waiter was canceled before the module responded",
5204 )?]),
5205 Err(_) => Ok(vec![control_error_frame(
5206 &frame,
5207 "module_timeout",
5208 format!(
5209 "module_id '{module_id}' did not answer health.check within {:?}",
5210 self.health_probe_timeout
5211 ),
5212 )?]),
5213 }
5214 }
5215
5216 fn supervisor_status(
5217 &self,
5218 module_id: &str,
5219 corr: u64,
5220 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5221 self.supervisor
5222 .get(module_id)
5223 .map(|module| {
5224 let warming = module.is_warming_for_control("status").map_err(|err| {
5225 RouterError::backend(
5226 0,
5227 corr,
5228 format!(
5229 "failed to read supervisor warming state for module_id '{module_id}': {err}"
5230 ),
5231 )
5232 })?;
5233 module.status_for_control("status").map_err(|err| {
5234 RouterError::backend(
5235 0,
5236 corr,
5237 format!(
5238 "failed to read supervisor status for module_id '{module_id}': {err}"
5239 ),
5240 )
5241 }).map(|status| (status, warming))
5242 })
5243 .transpose()
5244 }
5245
5246 fn guard_module_control_op(
5247 &self,
5248 frame: &Frame,
5249 module_id: &str,
5250 op: &str,
5251 ) -> Result<Option<Frame>, RouterError> {
5252 if self.module_grants_op(module_id, op, frame.header.corr)? {
5253 return Ok(None);
5254 }
5255
5256 Ok(Some(control_error_frame(
5257 frame,
5258 "op_not_allowed",
5259 format!("module_id '{module_id}' did not grant control op '{op}'"),
5260 )?))
5261 }
5262
5263 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
5264 let Some(registration) = self
5265 .registry
5266 .get_module(module_id)
5267 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
5268 else {
5269 return Ok(false);
5270 };
5271 Ok(module_registration_grants_op(®istration.control_ops, op))
5272 }
5273
5274 fn handle_status_update(
5275 &self,
5276 endpoint: ModuleEndpointId,
5277 frame: Frame,
5278 ) -> Result<Vec<Frame>, RouterError> {
5279 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
5280 Ok(update) => update,
5281 Err(err) => {
5282 if is_known_module_push_op(&frame.body) {
5287 return Ok(vec![control_error_frame(
5288 &frame,
5289 "invalid_control_body",
5290 format!("malformed module control push body: {err}"),
5291 )?]);
5292 }
5293 return Ok(Vec::new());
5294 }
5295 };
5296
5297 match update {
5298 ModuleControlPush::RouteStatus {
5299 route_channel,
5300 route_epoch,
5301 status,
5302 } => {
5303 self.forwarding
5304 .cache_status(endpoint, route_channel, route_epoch, status)
5305 .map_err(RouterError::Forwarding)?;
5306 }
5307 }
5308 Ok(Vec::new())
5309 }
5310
5311 fn handle_route_poll(
5312 &self,
5313 ctx: &RouteCtx,
5314 frame: Frame,
5315 route_channel: u16,
5316 route_epoch: u32,
5317 kind: PollKind,
5318 ) -> Result<Vec<Frame>, RouterError> {
5319 let snapshot = self
5320 .forwarding
5321 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
5322 .map_err(RouterError::Forwarding)?;
5323 let response = match (kind, snapshot) {
5324 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
5325 ClientControlResponse::RoutePoll {
5326 route_channel,
5327 route_epoch,
5328 status,
5329 live: None,
5330 }
5331 }
5332 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5333 route_channel,
5334 route_epoch,
5335 status: None,
5336 live: None,
5337 },
5338 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5339 let live = self
5365 .process_liveness
5366 .as_ref()
5367 .and_then(|source| source.process_live(&module_id))
5368 .unwrap_or(true);
5369 ClientControlResponse::RoutePoll {
5370 route_channel,
5371 route_epoch,
5372 status: None,
5373 live: Some(live),
5374 }
5375 }
5376 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5377 route_channel,
5378 route_epoch,
5379 status: None,
5380 live: Some(false),
5381 },
5382 };
5383
5384 Ok(vec![control_response_body_frame(
5385 &frame,
5386 &response,
5387 "ClientControlResponse::RoutePoll",
5388 )?])
5389 }
5390
5391 pub(crate) fn observe_module_control_completion(
5392 &self,
5393 completion: ModuleControlRpcCompletion,
5394 ) -> bool {
5395 match completion {
5396 ModuleControlRpcCompletion::Unknown => false,
5397 ModuleControlRpcCompletion::Settled => true,
5398 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5399 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5400 info!(
5401 module_id = %module_id,
5402 latency_ms,
5403 "late health.check answer proves the module is alive"
5404 );
5405 match self
5406 .supervisor
5407 .record_late_health_answer(&module_id, latency_ms)
5408 {
5409 Ok(true) => {}
5410 Ok(false) => debug!(
5411 module_id = %module_id,
5412 latency_ms,
5413 "late health.check answer has no active supervisor snapshot"
5414 ),
5415 Err(err) => warn!(
5416 module_id = %module_id,
5417 latency_ms,
5418 error = %err,
5419 "failed to record late health.check answer"
5420 ),
5421 }
5422 true
5423 }
5424 }
5425 }
5426
5427 fn refuse_to_end_module_connection_for_a_client(
5448 &self,
5449 module_connection_id: ConnectionId,
5450 corr: u64,
5451 err: ForwardingError,
5452 ) -> Result<(), RouterError> {
5453 if let ForwardingError::ConnectionClosing { connection_id } = err {
5454 if connection_id != module_connection_id {
5455 warn!(
5456 module_connection_id = module_connection_id.get(),
5457 client_connection_id = connection_id.get(),
5458 corr,
5459 "dropping a route.bind response for a closing client; the module connection keeps serving"
5460 );
5461 return Ok(());
5462 }
5463 }
5464 Err(RouterError::Forwarding(err))
5465 }
5466
5467 fn handle_module_relay_response(
5468 &self,
5469 connection_id: ConnectionId,
5470 frame: Frame,
5471 ) -> Result<Vec<Frame>, RouterError> {
5472 let mut secondary_error = None;
5473 let outcome = match frame.header.ty {
5474 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5475 Ok(probe) if probe.op == "route.bind" => {
5476 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5477 Ok(ModuleControlResponse::RouteBindAck {}) => {
5478 RouteBindRelayOutcome::Accepted
5479 }
5480 Ok(other) => {
5481 let message =
5482 format!("route.bind response carried unexpected body: {other:?}");
5483 secondary_error = Some(control_error_frame(
5484 &frame,
5485 "invalid_control_body",
5486 message.clone(),
5487 )?);
5488 RouteBindRelayOutcome::ModuleGone(message)
5489 }
5490 Err(err) => {
5491 let message = format!("malformed route.bind response body: {err}");
5492 secondary_error = Some(control_error_frame(
5493 &frame,
5494 "invalid_control_body",
5495 message.clone(),
5496 )?);
5497 RouteBindRelayOutcome::ModuleGone(message)
5498 }
5499 }
5500 }
5501 Ok(probe) => {
5502 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5503 {
5504 Ok(response) => ModuleControlRpcOutcome::Response(response),
5505 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5506 "malformed {} response body: {err}",
5507 probe.op
5508 )),
5509 };
5510 let completion = self
5511 .forwarding
5512 .complete_module_control_rpc(
5513 connection_id,
5514 frame.header.corr,
5515 Some(&probe.op),
5516 outcome,
5517 )
5518 .map_err(RouterError::Forwarding)?;
5519 if !self.observe_module_control_completion(completion) {
5520 debug!(
5521 connection_id = connection_id.get(),
5522 corr = frame.header.corr,
5523 op = %probe.op,
5524 "dropping late or unknown module-control RPC response"
5525 );
5526 }
5527 return Ok(Vec::new());
5528 }
5529 Err(err) => {
5530 if let Some(expected_op) = self
5531 .forwarding
5532 .pending_module_control_op(connection_id, frame.header.corr)
5533 .map_err(RouterError::Forwarding)?
5534 {
5535 let completion = self
5536 .forwarding
5537 .complete_module_control_rpc(
5538 connection_id,
5539 frame.header.corr,
5540 None,
5541 ModuleControlRpcOutcome::MalformedResponse(format!(
5542 "malformed {expected_op} response body: {err}"
5543 )),
5544 )
5545 .map_err(RouterError::Forwarding)?;
5546 if !self.observe_module_control_completion(completion) {
5547 debug!(
5548 connection_id = connection_id.get(),
5549 corr = frame.header.corr,
5550 "dropping late malformed module-control RPC response"
5551 );
5552 }
5553 return Ok(Vec::new());
5554 }
5555 let message = format!("malformed route.bind response body: {err}");
5556 secondary_error = Some(control_error_frame(
5557 &frame,
5558 "invalid_control_body",
5559 message.clone(),
5560 )?);
5561 RouteBindRelayOutcome::ModuleGone(message)
5562 }
5563 },
5564 FrameType::Error => {
5565 if self
5566 .forwarding
5567 .pending_module_control_op(connection_id, frame.header.corr)
5568 .map_err(RouterError::Forwarding)?
5569 .is_some()
5570 {
5571 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5572 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5573 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5574 "malformed module-control ERROR body: {err}"
5575 )),
5576 };
5577 let completion = self
5578 .forwarding
5579 .complete_module_control_rpc(
5580 connection_id,
5581 frame.header.corr,
5582 None,
5583 outcome,
5584 )
5585 .map_err(RouterError::Forwarding)?;
5586 if !self.observe_module_control_completion(completion) {
5587 debug!(
5588 connection_id = connection_id.get(),
5589 corr = frame.header.corr,
5590 "dropping late or unknown module-control RPC error"
5591 );
5592 }
5593 return Ok(Vec::new());
5594 }
5595 match serde_json::from_slice::<ErrorBody>(&frame.body) {
5596 Ok(body) => RouteBindRelayOutcome::Rejected(body),
5597 Err(err) => {
5598 let message = format!("malformed route.bind ERROR body: {err}");
5599 secondary_error = Some(control_error_frame(
5600 &frame,
5601 "invalid_control_body",
5602 message.clone(),
5603 )?);
5604 RouteBindRelayOutcome::ModuleGone(message)
5605 }
5606 }
5607 }
5608 ty => {
5609 return Ok(vec![control_error_frame(
5610 &frame,
5611 "unsupported_control_frame",
5612 format!("unsupported module channel-0 frame {ty:?}"),
5613 )?])
5614 }
5615 };
5616
5617 let settled =
5618 self.forwarding
5619 .complete_pending_relay(connection_id, frame.header.corr, outcome);
5620 let completion = match settled {
5621 Ok(completion) => completion,
5622 Err(err) => {
5623 self.refuse_to_end_module_connection_for_a_client(
5624 connection_id,
5625 frame.header.corr,
5626 err,
5627 )?;
5628 return Ok(secondary_error.into_iter().collect());
5629 }
5630 };
5631 if let Some(target) = completion.abandoned.as_ref() {
5632 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5633 }
5634 if !completion.settled {
5635 debug!(
5636 connection_id = connection_id.get(),
5637 corr = frame.header.corr,
5638 frame_type = ?frame.header.ty,
5639 "dropping late or unknown route.bind relay response"
5640 );
5641 }
5642 Ok(secondary_error.into_iter().collect())
5643 }
5644
5645 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5646 debug!(connection_id = connection_id.get(), "handling GOODBYE");
5647 self.cleanup_connection_with_end_reason(
5651 connection_id,
5652 RegistrationEndReason::ExplicitGoodbye,
5653 )
5654 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5655 Ok(Vec::new())
5656 }
5657}
5658
5659impl Default for ControlHandler {
5660 fn default() -> Self {
5661 Self::new(Arc::new(Registry::default()))
5662 }
5663}
5664
5665impl crate::supervise::SwapPromotionObserver for ControlHandler {
5666 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5667 self.apply_registration_capabilities(registration);
5668 }
5669}
5670
5671fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5672 CapabilityRequirementStatus {
5673 consumer: status.consumer,
5674 capability: status.capability,
5675 need: match status.need {
5676 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5677 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5678 },
5679 verdict: status.verdict.as_str().to_string(),
5680 episode_seq: status.episode_seq,
5681 config_satisfiable: status.config_satisfiable,
5682 runtime_available: status.runtime_available,
5683 detail: status.detail,
5684 }
5685}
5686
5687fn append_capability_problem_detail(
5688 detail: Option<String>,
5689 capability_detail: Option<String>,
5690) -> Option<String> {
5691 match (detail, capability_detail) {
5692 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5693 (Some(detail), None) => Some(detail),
5694 (None, Some(capability_detail)) => Some(capability_detail),
5695 (None, None) => None,
5696 }
5697}
5698
5699fn subc_ops() -> Vec<String> {
5700 SUBC_CONTROL_OPS
5701 .iter()
5702 .map(|op| (*op).to_string())
5703 .collect()
5704}
5705
5706fn module_subc_ops() -> Vec<String> {
5707 SUBC_CONTROL_OPS
5708 .iter()
5709 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5710 .map(|op| (*op).to_string())
5711 .collect()
5712}
5713
5714#[cfg(test)]
5715fn module_baseline_control_ops() -> Vec<String> {
5716 MODULE_BASELINE_CONTROL_OPS
5717 .iter()
5718 .map(|op| (*op).to_string())
5719 .collect()
5720}
5721
5722fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5723 let mut seen = HashSet::new();
5724 let mut effective = Vec::new();
5725 for op in MODULE_BASELINE_CONTROL_OPS {
5726 if seen.insert((*op).to_string()) {
5727 effective.push((*op).to_string());
5728 }
5729 }
5730 for op in declared.unwrap_or_default() {
5731 if seen.insert(op.clone()) {
5732 effective.push(op);
5733 }
5734 }
5735 effective
5736}
5737
5738fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5739 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5740}
5741
5742fn target_module_id(target: &RouteTarget) -> &str {
5743 match target {
5744 RouteTarget::ToolProvider { module_id }
5745 | RouteTarget::ManagementSurface { module_id }
5746 | RouteTarget::InternalService { module_id, .. } => module_id,
5747 }
5748}
5749
5750fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5751 roles.iter().any(|role| match (target, role) {
5752 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5753 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5754 (
5755 RouteTarget::InternalService { service_id, .. },
5756 ProviderRole::InternalService {
5757 service_id: provided,
5758 ..
5759 },
5760 ) => service_id == provided,
5761 _ => false,
5762 })
5763}
5764
5765fn is_routable_role(role: &ProviderRole) -> bool {
5766 matches!(
5767 role,
5768 ProviderRole::ToolProvider { .. }
5769 | ProviderRole::ManagementSurface { .. }
5770 | ProviderRole::InternalService { .. }
5771 )
5772}
5773
5774#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5775enum ControlRequestBodyError {
5776 UnknownOp,
5777 InvalidBody,
5778}
5779
5780#[derive(Debug, Deserialize)]
5781struct ControlOpProbe {
5782 op: String,
5783}
5784
5785const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5788
5789fn is_known_module_push_op(body: &[u8]) -> bool {
5790 serde_json::from_slice::<ControlOpProbe>(body)
5791 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5792 .unwrap_or(false)
5793}
5794
5795fn is_known_module_request_op(body: &[u8]) -> bool {
5796 serde_json::from_slice::<ControlOpProbe>(body)
5797 .map(|probe| is_module_to_subc_op(&probe.op))
5798 .unwrap_or(false)
5799}
5800
5801fn is_module_to_subc_op(op: &str) -> bool {
5802 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5803}
5804
5805fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5806 debug!(
5807 op = %op,
5808 connection_id = connection_id.get(),
5809 corr,
5810 "control dispatch"
5811 );
5812}
5813
5814fn log_slow_control_dispatch(
5815 dispatch_started_at: Option<StdInstant>,
5816 op: &'static str,
5817 connection_id: ConnectionId,
5818 corr: u64,
5819) {
5820 let Some(dispatch_started_at) = dispatch_started_at else {
5821 return;
5822 };
5823 let elapsed = dispatch_started_at.elapsed();
5824 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5825 warn!(
5826 op = %op,
5827 connection_id = connection_id.get(),
5828 corr,
5829 elapsed_ms = elapsed.as_millis() as u64,
5830 "slow control dispatch"
5831 );
5832 }
5833}
5834
5835fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5836 match request {
5837 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5838 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5839 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5840 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5841 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5842 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5843 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5844 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5845 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5846 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5847 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5848 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5849 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5850 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5851 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5852 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5853 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
5854 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
5855 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
5856 }
5857}
5858
5859fn principal_label(principal: &Principal) -> String {
5860 match principal {
5861 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
5862 Principal::Direct => "direct".to_string(),
5863 other => format!("{other:?}"),
5864 }
5865}
5866
5867fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
5868 match request {
5869 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
5870 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
5871 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
5872 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
5873 }
5874}
5875
5876fn parse_client_control_request(
5877 body: &[u8],
5878) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
5879 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
5880 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5881 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
5882 ControlRequestBodyError::InvalidBody
5883 }
5884 Ok(_) => ControlRequestBodyError::UnknownOp,
5885 Err(_) => ControlRequestBodyError::InvalidBody,
5886 };
5887 (err, classification)
5888 })
5889}
5890
5891fn parse_module_control_request_from_module(
5892 body: &[u8],
5893) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
5894 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
5895 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
5896 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
5897 Ok(_) => ControlRequestBodyError::UnknownOp,
5898 Err(_) => ControlRequestBodyError::InvalidBody,
5899 };
5900 (err, classification)
5901 })
5902}
5903
5904#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
5905enum ProviderRoleKind {
5906 ToolProvider,
5907 PipelineStage,
5908 ManagementSurface,
5909 InternalService,
5910}
5911
5912fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
5913 match role {
5914 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
5915 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
5916 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
5917 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
5918 }
5919}
5920
5921fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
5922 roles.iter().map(provider_role_kind).collect()
5923}
5924
5925const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
5928
5929#[derive(Debug, Default, PartialEq, Eq)]
5931struct ScopeOutcomeCounts {
5932 created: usize,
5933 replaced: usize,
5934 updated: usize,
5935 unchanged: usize,
5936 refused: usize,
5937}
5938
5939impl ScopeOutcomeCounts {
5940 fn of(results: &[ScopeRecordResult]) -> Self {
5941 let mut counts = Self::default();
5942 for result in results {
5943 let slot = match result.outcome {
5944 ScopeRecordOutcome::Created => &mut counts.created,
5945 ScopeRecordOutcome::Replaced => &mut counts.replaced,
5946 ScopeRecordOutcome::Updated => &mut counts.updated,
5947 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
5948 ScopeRecordOutcome::Refused => &mut counts.refused,
5949 };
5950 *slot += 1;
5951 }
5952 counts
5953 }
5954}
5955
5956#[cfg(test)]
5957mod scope_outcome_count_tests {
5958 use super::*;
5959
5960 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
5961 ScopeRecordResult {
5962 scope_ref: "r".to_string(),
5963 scope_epoch: 1,
5964 outcome,
5965 code: None,
5966 message: None,
5967 version: None,
5968 parent_state: None,
5969 }
5970 }
5971
5972 #[test]
5975 fn every_outcome_is_counted_in_its_own_field() {
5976 let results = [
5977 result(ScopeRecordOutcome::Created),
5978 result(ScopeRecordOutcome::Created),
5979 result(ScopeRecordOutcome::Replaced),
5980 result(ScopeRecordOutcome::Updated),
5981 result(ScopeRecordOutcome::Unchanged),
5982 result(ScopeRecordOutcome::Refused),
5983 result(ScopeRecordOutcome::Refused),
5984 result(ScopeRecordOutcome::Refused),
5985 ];
5986 assert_eq!(
5987 ScopeOutcomeCounts::of(&results),
5988 ScopeOutcomeCounts {
5989 created: 2,
5990 replaced: 1,
5991 updated: 1,
5992 unchanged: 1,
5993 refused: 3,
5994 }
5995 );
5996 }
5997}
5998
5999fn capability_census_trigger(
6003 old: Option<&CapabilityDeclarations>,
6004 new: Option<&CapabilityDeclarations>,
6005) -> bool {
6006 let old_provides = old
6007 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
6008 .unwrap_or_default();
6009 let old_denies = old
6010 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
6011 .unwrap_or_default();
6012 let new = new.cloned().unwrap_or(CapabilityDeclarations {
6013 provides: Vec::new(),
6014 requires: Vec::new(),
6015 must_never_reach: Vec::new(),
6016 });
6017
6018 new.provides
6019 .iter()
6020 .any(|capability| !old_provides.contains(capability))
6021 || new
6022 .must_never_reach
6023 .iter()
6024 .any(|capability| !old_denies.contains(capability))
6025}
6026
6027fn denied_capability<'a>(
6030 opening_manifest: &'a ModuleManifest,
6031 target_manifest: &ModuleManifest,
6032) -> Option<&'a str> {
6033 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
6034 let target_capabilities = target_manifest.capabilities.as_ref()?;
6035 opening_capabilities
6036 .must_never_reach
6037 .iter()
6038 .find(|denied| {
6039 target_capabilities
6040 .provides
6041 .iter()
6042 .any(|provided| provided == *denied)
6043 })
6044 .map(String::as_str)
6045}
6046
6047fn catalog_update_frozen_field_message(
6048 registered: &ModuleManifest,
6049 provides: &[ProviderRole],
6050) -> Option<String> {
6051 let old_has_provides = !registered.provides.is_empty();
6052 let new_has_provides = !provides.is_empty();
6053 if old_has_provides != new_has_provides {
6054 return Some(format!(
6055 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
6056 registered.module_id
6057 ));
6058 }
6059
6060 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
6061 return Some(format!(
6062 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
6063 registered.module_id
6064 ));
6065 }
6066
6067 let registered_concurrency = manifest_concurrency(registered);
6068 let mut candidate = registered.clone();
6069 candidate.provides = provides.to_vec();
6070 let candidate_concurrency = manifest_concurrency(&candidate);
6071 if candidate_concurrency != registered_concurrency {
6072 return Some(format!(
6073 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
6074 registered.module_id, registered_concurrency, candidate_concurrency
6075 ));
6076 }
6077
6078 None
6081}
6082
6083fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
6084 manifest.provides.iter().any(is_routable_role)
6085}
6086
6087fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
6093 manifest
6094 .provides
6095 .iter()
6096 .find_map(|provider| match provider {
6097 ProviderRole::ToolProvider { concurrency, .. }
6098 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
6099 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
6100 })
6101 .unwrap_or(Concurrency::ModuleManaged)
6102}
6103
6104fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
6111 let has_management_surface = manifest
6112 .provides
6113 .iter()
6114 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
6115 if !has_management_surface {
6116 return false;
6117 }
6118 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
6119 return false;
6120 };
6121 let Some(provides) = raw
6122 .get("manifest")
6123 .and_then(|manifest| manifest.get("provides"))
6124 .and_then(serde_json::Value::as_array)
6125 else {
6126 return false;
6127 };
6128 provides.iter().any(|role| {
6133 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
6134 && role.get("concurrency").is_none()
6135 })
6136}
6137
6138fn negotiate_version(peer_version: u8) -> Result<u8, String> {
6139 if peer_version != PROTOCOL_VERSION {
6140 return Err(format!(
6141 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
6142 ));
6143 }
6144 Ok(PROTOCOL_VERSION)
6145}
6146
6147fn pong(frame: &Frame) -> Result<Frame, RouterError> {
6148 Frame::build_with_version(
6149 response_version(frame),
6150 FrameType::Pong,
6151 frame.header.flags,
6152 0,
6153 0,
6154 frame.header.corr,
6155 Vec::new(),
6156 )
6157 .map_err(RouterError::FrameBuild)
6158}
6159
6160fn control_error_frame(
6161 frame: &Frame,
6162 code: &'static str,
6163 message: impl Into<String>,
6164) -> Result<Frame, RouterError> {
6165 control_error_body_frame(
6166 frame,
6167 ErrorBody {
6168 code: code.to_string(),
6169 message: message.into(),
6170 detail: None,
6171 },
6172 )
6173}
6174
6175fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
6176 let body = serde_json::to_vec(&error).map_err(|err| {
6177 RouterError::backend(
6178 0,
6179 frame.header.corr,
6180 format!("failed to encode control ERROR: {err}"),
6181 )
6182 })?;
6183
6184 Frame::build_with_version(
6185 response_version(frame),
6186 FrameType::Error,
6187 control_flags(),
6188 0,
6189 0,
6190 frame.header.corr,
6191 body,
6192 )
6193 .map_err(RouterError::FrameBuild)
6194}
6195
6196fn control_response_body_frame<T: Serialize>(
6197 frame: &Frame,
6198 reply: &T,
6199 label: &'static str,
6200) -> Result<Frame, RouterError> {
6201 let body = serde_json::to_vec(reply).map_err(|err| {
6202 RouterError::backend(
6203 0,
6204 frame.header.corr,
6205 format!("failed to encode {label}: {err}"),
6206 )
6207 })?;
6208
6209 Frame::build_with_version(
6210 response_version(frame),
6211 FrameType::Response,
6212 control_flags(),
6213 0,
6214 0,
6215 frame.header.corr,
6216 body,
6217 )
6218 .map_err(RouterError::FrameBuild)
6219}
6220
6221fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6251 match err {
6252 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6253 ForwardingError::NoModuleConnection => "target_unavailable",
6254 ForwardingError::ModuleReloading { .. } => "module_reloading",
6255 ForwardingError::ClientRouteChannelExhausted { .. }
6256 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6257 ForwardingError::StaleModuleEndpoint
6258 | ForwardingError::UnknownReservation { .. }
6259 | ForwardingError::ConnectionClosing { .. }
6260 | ForwardingError::ClientEgressClosed { .. }
6261 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
6262 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
6265 ForwardingError::RelayCorrelationExhausted
6266 | ForwardingError::RouteOpenBuild(_)
6267 | ForwardingError::Poisoned => "forwarding_error",
6268 }
6269}
6270
6271fn response_version(frame: &Frame) -> u8 {
6272 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
6273 frame.header.ver
6274 } else {
6275 PROTOCOL_VERSION
6276 }
6277}
6278
6279fn control_flags() -> Flags {
6280 Flags::new(false, Priority::Passive, false)
6281}
6282
6283fn send_goodbye_target_best_effort(
6288 counters: &DaemonCounters,
6289 target: &GoodbyeTarget,
6290 context: &'static str,
6291) {
6292 let Ok(frame) = Frame::build_with_version(
6293 target.negotiated_ver,
6294 FrameType::Goodbye,
6295 control_flags(),
6296 target.channel,
6297 target.epoch,
6298 0,
6299 Vec::new(),
6300 ) else {
6301 return;
6302 };
6303 crate::forwarding::send_module_route_goodbye(
6304 counters,
6305 &target.sink,
6306 frame,
6307 target.module_id.as_deref(),
6308 context,
6309 );
6310}
6311
6312pub(crate) fn send_route_control_pushes(
6313 forwarding: &ForwardingTable,
6314 routes: Vec<EndpointRoute>,
6315 push: ClientControlPush,
6316) {
6317 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
6318 for route in routes {
6319 let target = route.goodbye_target;
6320 if let Some((existing, channels)) = targets
6321 .iter_mut()
6322 .find(|(existing, _)| existing.connection_id == target.connection_id)
6323 {
6324 debug_assert_eq!(
6325 existing.negotiated_ver, target.negotiated_ver,
6326 "one connection cannot negotiate multiple frame versions"
6327 );
6328 if !channels.contains(&target.channel) {
6329 channels.push(target.channel);
6330 }
6331 continue;
6332 }
6333 let channel = target.channel;
6334 targets.push((target, vec![channel]));
6335 }
6336 for (target, mut channels) in targets {
6337 channels.sort_unstable();
6338 let mut push = push.clone();
6339 match &mut push {
6340 ClientControlPush::RouteClosing {
6341 channels: covered, ..
6342 }
6343 | ClientControlPush::RouteClosed {
6344 channels: covered, ..
6345 } => *covered = channels,
6346 }
6347 let body = match serde_json::to_vec(&push) {
6348 Ok(body) => body,
6349 Err(err) => {
6350 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
6351 continue;
6352 }
6353 };
6354 let frame = match Frame::build_with_version(
6355 target.negotiated_ver,
6356 FrameType::Push,
6357 control_flags(),
6358 0,
6359 0,
6360 0,
6361 body.clone(),
6362 ) {
6363 Ok(frame) => frame,
6364 Err(err) => {
6365 warn!(
6366 route_channel = target.channel,
6367 error = %err,
6368 "failed to build route lifecycle control PUSH frame"
6369 );
6370 continue;
6371 }
6372 };
6373 if let Err(err) = target.sink.try_send(frame) {
6374 if target.close_on_delivery_failure() {
6375 warn!(
6376 target_connection_id = target.connection_id.get(),
6377 route_channel = target.channel,
6378 error = %err,
6379 "route lifecycle control PUSH was not delivered to client; closing target connection"
6380 );
6381 let _ = forwarding.escalate_client_delivery_failure(
6382 target.connection_id,
6383 target.channel,
6384 target.epoch,
6385 CloseReason::new(
6386 "route_lifecycle_push_delivery_failed",
6387 format!(
6388 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6389 target.channel
6390 ),
6391 ),
6392 crate::forwarding::UndeliveredFrame {
6393 module_id: target.module_id.as_deref(),
6394 sink: &target.sink,
6395 },
6396 );
6397 }
6398 }
6399 }
6400}
6401
6402#[cfg(test)]
6403mod tests {
6404 #[cfg(unix)]
6405 #[tokio::test]
6406 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
6407 let dir = subc_test_support::TestTempDir::new("rescan-live-health");
6408 let path = dir.join("subc.jsonc");
6409 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
6410 "program":"/bin/sleep","args":["60"],"protocol":"none",
6411 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
6412 }}}).to_string()).unwrap();
6413 let mut configured = crate::daemon_config::load(&path)
6414 .unwrap()
6415 .unwrap()
6416 .modules
6417 .pop()
6418 .unwrap();
6419 let registry = std::sync::Arc::new(crate::Registry::default());
6420 let handle = crate::SupervisorHandle::new();
6421 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
6422 .with_handle(handle.clone());
6423 let module = supervisor
6424 .supervise_configured_with_health(
6425 configured.module_spec(),
6426 true,
6427 configured.health.clone(),
6428 None,
6429 configured.restart,
6430 )
6431 .unwrap();
6432 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
6433 let before = module.status().unwrap().pid;
6434 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
6435 configured.health.cadence = std::time::Duration::from_secs(3600);
6436 let health_only = handler
6437 .reconcile_supervised_modules(&supervisor, vec![configured.clone()], false)
6438 .await
6439 .unwrap();
6440 assert!(
6441 health_only.changed_pending_reload.is_empty(),
6442 "health policy is already applied live"
6443 );
6444 assert_eq!(module.status().unwrap().pid, before);
6445 assert_eq!(
6446 module.configuration().unwrap().1.http,
6447 configured.health.http
6448 );
6449 configured.args = vec!["61".into()];
6450 let launch = handler
6451 .reconcile_supervised_modules(&supervisor, vec![configured], false)
6452 .await
6453 .unwrap();
6454 assert_eq!(launch.changed_pending_reload, ["stock"]);
6455 assert_eq!(
6456 module.status().unwrap().pid,
6457 before,
6458 "a launch edit is stored until reload"
6459 );
6460 module.drain().await.unwrap();
6461 }
6462 use std::{
6463 collections::BTreeMap,
6464 fmt,
6465 path::PathBuf,
6466 sync::{Arc, Mutex},
6467 time::Duration,
6468 };
6469 use subc_test_support::TestTempDir;
6470
6471 use serde_json::{json, Value};
6472 use subc_protocol::{
6473 manifest::{
6474 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6475 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6476 },
6477 session::HealthStatus,
6478 FrameType,
6479 };
6480
6481 use super::*;
6482 use crate::{
6483 forwarding::{DataRoute, DataRouteState},
6484 registry::ChannelState,
6485 router::FrameSink,
6486 stderr_tail::DEFAULT_MAX_LINE_BYTES,
6487 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6488 RouteCtx, Router,
6489 };
6490 use tokio::{
6491 sync::mpsc,
6492 time::{sleep, Instant},
6493 };
6494 use tracing::{
6495 field::{Field, Visit},
6496 Event, Subscriber,
6497 };
6498 use tracing_subscriber::{layer::Context, prelude::*, Layer};
6499
6500 fn fake_aft_stub_path() -> PathBuf {
6519 let mut path = std::env::current_exe().expect("current_exe available in tests");
6520 path.pop(); path.pop(); path.push(if cfg!(windows) {
6523 "fake-aft-stub.exe"
6524 } else {
6525 "fake-aft-stub"
6526 });
6527 assert!(
6528 path.exists(),
6529 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6530 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6531 path.display()
6532 );
6533 path
6534 }
6535
6536 fn client_retries(code: &str) -> bool {
6541 subc_protocol::error_codes::is_retryable_route_open(code)
6542 }
6543
6544 #[test]
6555 fn retryability_of_forwarding_codes_matches_the_failure() {
6556 let transient = [
6559 ForwardingError::NoModuleConnection,
6560 ForwardingError::ModuleReloading {
6561 module_id: "m".into(),
6562 },
6563 ForwardingError::StaleModuleEndpoint,
6564 ForwardingError::UnknownReservation {
6565 client_channel: 1,
6566 module_channel: 1,
6567 },
6568 ForwardingError::ConnectionClosing {
6569 connection_id: ConnectionId::new(1),
6570 },
6571 ForwardingError::ClientEgressClosed {
6572 connection_id: ConnectionId::new(1),
6573 },
6574 ForwardingError::ModuleEgressUnavailable {
6575 connection_id: ConnectionId::new(1),
6576 },
6577 ];
6578 for err in transient {
6579 let code = forwarding_error_code(&err);
6580 assert!(
6581 client_retries(code),
6582 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6583 );
6584 }
6585
6586 let permanent = [
6591 ForwardingError::ConnectionRoleConflict {
6592 connection_id: ConnectionId::new(1),
6593 },
6594 ForwardingError::ClientRouteChannelExhausted {
6595 connection_id: ConnectionId::new(1),
6596 },
6597 ForwardingError::ModuleRouteChannelExhausted {
6598 endpoint: ModuleEndpointId {
6599 connection_id: ConnectionId::new(1),
6600 generation: 1,
6601 },
6602 },
6603 ForwardingError::RelayCorrelationExhausted,
6604 ForwardingError::RouteOpenBuild("x".into()),
6605 ForwardingError::Poisoned,
6606 ];
6607 for err in permanent {
6608 let code = forwarding_error_code(&err);
6609 assert!(
6610 !client_retries(code),
6611 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6612 );
6613 }
6614 }
6615
6616 #[tokio::test]
6644 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6645 let handler = ControlHandler::default();
6646 let frame =
6647 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6648
6649 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6653 assert_eq!(
6654 stamped,
6655 Principal::Direct,
6656 "a caller that proved nothing must not be stamped as a supervised module"
6657 );
6658
6659 let forged = handler
6664 .route_open_principal(
6665 &frame,
6666 Some(ConsumerIdentity {
6667 module_id: "aft".to_string(),
6668 launch_nonce: "not-a-real-nonce".to_string(),
6669 }),
6670 )
6671 .unwrap();
6672 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6673 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6674 }
6675
6676 #[test]
6693 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6694 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6695 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6696 let ClientControlRequest::RouteOpen {
6697 consumer_identity, ..
6698 } = parsed
6699 else {
6700 panic!("route.open body must parse as RouteOpen");
6701 };
6702 assert_eq!(
6703 consumer_identity,
6704 Some(ConsumerIdentity {
6705 module_id: "aft".to_string(),
6706 launch_nonce: "n".to_string(),
6707 }),
6708 "the wire field name must reach the value route_open_principal reads"
6709 );
6710 }
6711
6712 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6713 ModuleManifest::builder(module_id, "0.1.0")
6714 .protocol_ver(protocol_ver)
6715 .provides(vec![ProviderRole::ToolProvider {
6716 tools: vec![Tool {
6717 name: "read".to_string(),
6718 description: None,
6719 execution_mode: ExecutionMode::Pure,
6720 schema: json!({"type": "object"}),
6721 }],
6722 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6723 concurrency: Concurrency::ModuleManaged,
6724 emits_push: true,
6725 sub_supervises: true,
6726 }])
6727 .build()
6728 }
6729
6730 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6731 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6732 }
6733
6734 fn hello_frame_with_control_ops(
6735 module_id: &str,
6736 protocol_ver: u8,
6737 corr: u64,
6738 control_ops: Option<Vec<String>>,
6739 ) -> Frame {
6740 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6741 }
6742
6743 fn hello_frame_with_nonce(
6744 module_id: &str,
6745 protocol_ver: u8,
6746 corr: u64,
6747 launch_nonce: Option<&str>,
6748 ) -> Frame {
6749 hello_frame_full(
6750 module_id,
6751 protocol_ver,
6752 corr,
6753 None,
6754 launch_nonce.map(ToOwned::to_owned),
6755 )
6756 }
6757
6758 fn hello_frame_full(
6759 module_id: &str,
6760 protocol_ver: u8,
6761 corr: u64,
6762 control_ops: Option<Vec<String>>,
6763 launch_nonce: Option<String>,
6764 ) -> Frame {
6765 let body = serde_json::to_vec(&ModuleHelloBody {
6766 manifest: manifest(module_id, protocol_ver),
6767 protocol_ver,
6768 control_ops,
6769 launch_nonce,
6770 })
6771 .unwrap();
6772 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6773 }
6774
6775 fn non_routable_hello_frame_with_control_ops(
6776 module_id: &str,
6777 corr: u64,
6778 control_ops: Option<Vec<String>>,
6779 ) -> Frame {
6780 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6781 manifest.provides.clear();
6782 let body = serde_json::to_vec(&ModuleHelloBody {
6783 manifest,
6784 protocol_ver: PROTOCOL_VERSION,
6785 control_ops,
6786 launch_nonce: None,
6787 })
6788 .unwrap();
6789 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6790 }
6791
6792 fn capability_grammar_hello_frame(
6793 capabilities: Value,
6794 runtime_computed: Option<Value>,
6795 corr: u64,
6796 ) -> Frame {
6797 let mut body = serde_json::to_value(ModuleHelloBody {
6798 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6799 protocol_ver: PROTOCOL_VERSION,
6800 control_ops: None,
6801 launch_nonce: None,
6802 })
6803 .expect("HELLO body serializes");
6804 body["manifest"]["capabilities"] = capabilities;
6805 if let Some(runtime_computed) = runtime_computed {
6806 body["runtime_computed"] = runtime_computed;
6807 }
6808 Frame::build(
6809 FrameType::Hello,
6810 control_flags(),
6811 0,
6812 0,
6813 corr,
6814 serde_json::to_vec(&body).expect("HELLO body reserializes"),
6815 )
6816 .expect("HELLO frame builds")
6817 }
6818
6819 fn channel_request(channel: u16, corr: u64) -> Frame {
6820 Frame::build(
6821 FrameType::Request,
6822 Flags::new(true, Priority::Interactive, false),
6823 channel,
6824 0,
6825 corr,
6826 b"opaque".to_vec(),
6827 )
6828 .unwrap()
6829 }
6830
6831 fn route_ctx(
6832 connection_id: ConnectionId,
6833 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
6834 let (tx, rx) = mpsc::channel(8);
6835 (
6836 RouteCtx {
6837 connection_id,
6838 egress: FrameSink::new(tx),
6839 },
6840 rx,
6841 )
6842 }
6843
6844 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
6845 serde_json::from_slice(&frame.body).unwrap()
6846 }
6847
6848 async fn hello_via_sink(
6853 handler: &ControlHandler,
6854 ctx: &RouteCtx,
6855 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
6856 hello: Frame,
6857 ) -> Frame {
6858 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
6859 assert!(
6860 replies.is_empty(),
6861 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
6862 );
6863 let ack = rx
6864 .try_recv()
6865 .expect("HELLO_ACK is queued on the module sink")
6866 .frame;
6867 assert_eq!(ack.header.ty, FrameType::HelloAck);
6868 ack
6869 }
6870
6871 fn parse_error(frame: &Frame) -> Value {
6872 serde_json::from_slice(&frame.body).unwrap()
6873 }
6874
6875 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
6876 serde_json::from_slice(&frame.body).unwrap()
6877 }
6878
6879 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
6880 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
6881 route_channel,
6882 route_epoch: 0,
6883 kind,
6884 })
6885 .unwrap();
6886 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6887 }
6888
6889 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
6890 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
6891 module_id: module_id.to_string(),
6892 })
6893 .unwrap();
6894 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6895 }
6896
6897 fn route_open_frame(corr: u64, module_id: &str, project_root: TestTempDir) -> Frame {
6898 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
6899 }
6900
6901 fn route_open_frame_with_consumer_capabilities(
6902 corr: u64,
6903 module_id: &str,
6904 project_root: TestTempDir,
6905 consumer_capabilities: Option<Vec<String>>,
6906 ) -> Frame {
6907 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6908 target: RouteTarget::ToolProvider {
6909 module_id: module_id.to_string(),
6910 },
6911 identity: BindIdentity::new(
6912 project_root.path().to_path_buf(),
6913 "unit".to_string(),
6914 "session".to_string(),
6915 ),
6916 consumer_identity: None,
6917 consumer_capabilities,
6918 role_versions: None,
6919 admission_facts: None,
6920 scope: None,
6921 })
6922 .unwrap();
6923 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6924 }
6925
6926 fn route_open_frame_with_role_versions(
6927 corr: u64,
6928 module_id: &str,
6929 project_root: TestTempDir,
6930 role_versions: Option<BTreeMap<String, String>>,
6931 ) -> Frame {
6932 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6933 target: RouteTarget::ToolProvider {
6934 module_id: module_id.to_string(),
6935 },
6936 identity: BindIdentity::new(
6937 project_root.path().to_path_buf(),
6938 "unit".to_string(),
6939 format!("session-{corr}"),
6940 ),
6941 consumer_identity: None,
6942 consumer_capabilities: None,
6943 role_versions,
6944 admission_facts: None,
6945 scope: None,
6946 })
6947 .unwrap();
6948 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6949 }
6950
6951 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
6952 entries
6953 .iter()
6954 .map(|(role, version)| (role.to_string(), version.to_string()))
6955 .collect()
6956 }
6957
6958 fn route_open_frame_with_admission_facts(
6959 corr: u64,
6960 module_id: &str,
6961 project_root: TestTempDir,
6962 consumer_identity: Option<subc_control::ConsumerIdentity>,
6963 facts: Option<Value>,
6964 ) -> Frame {
6965 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
6966 target: RouteTarget::ToolProvider {
6967 module_id: module_id.to_string(),
6968 },
6969 identity: BindIdentity::new(
6970 project_root.path().to_path_buf(),
6971 "unit".to_string(),
6972 format!("session-{corr}"),
6973 ),
6974 consumer_identity,
6975 consumer_capabilities: None,
6976 role_versions: None,
6977 admission_facts: facts,
6978 scope: None,
6979 })
6980 .unwrap();
6981 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
6982 }
6983
6984 #[derive(Clone, Default)]
6985 struct EventCapture {
6986 events: Arc<Mutex<Vec<CapturedEvent>>>,
6987 }
6988
6989 #[derive(Clone, Debug)]
6990 struct CapturedEvent {
6991 target: String,
6992 level: tracing::Level,
6993 fields: BTreeMap<String, String>,
6994 }
6995
6996 impl EventCapture {
6997 fn events(&self) -> Vec<CapturedEvent> {
6998 self.events.lock().unwrap().clone()
6999 }
7000 }
7001
7002 impl<S> Layer<S> for EventCapture
7003 where
7004 S: Subscriber,
7005 {
7006 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
7007 let mut visitor = EventFieldVisitor::default();
7008 event.record(&mut visitor);
7009 self.events.lock().unwrap().push(CapturedEvent {
7010 target: event.metadata().target().to_string(),
7011 level: *event.metadata().level(),
7012 fields: visitor.fields,
7013 });
7014 }
7015 }
7016
7017 #[derive(Default)]
7018 struct EventFieldVisitor {
7019 fields: BTreeMap<String, String>,
7020 }
7021
7022 impl Visit for EventFieldVisitor {
7023 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
7024 self.fields
7025 .insert(field.name().to_string(), format!("{value:?}"));
7026 }
7027 }
7028
7029 fn health_response(corr: u64, status: HealthStatus) -> Frame {
7030 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
7031 status,
7032 detail: Some("warming".to_string()),
7033 metrics: Some(json!({"queue_depth": 3})),
7034 })
7035 .unwrap();
7036 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7037 }
7038
7039 fn route_bind_ack(corr: u64) -> Frame {
7040 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
7041 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7042 }
7043
7044 fn unique_project_root(label: &str) -> TestTempDir {
7045 TestTempDir::new(label)
7046 }
7047
7048 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
7049 match parse_route_poll(frame) {
7050 ClientControlResponse::RoutePoll {
7051 status: None,
7052 live: Some(live),
7053 ..
7054 } => assert_eq!(live, expected_live),
7055 other => panic!("unexpected route.poll response: {other:?}"),
7056 }
7057 }
7058
7059 fn bind_liveness_route(
7060 registry: &Registry,
7061 forwarding: &ForwardingTable,
7062 module_id: &str,
7063 ) -> (RouteCtx, u16, u32) {
7064 let module_connection = ConnectionId::new(101);
7065 let client_connection = ConnectionId::new(202);
7066 let registration = registry
7067 .register_with_control_ops(
7068 manifest(module_id, PROTOCOL_VERSION),
7069 PROTOCOL_VERSION,
7070 module_connection,
7071 module_baseline_control_ops(),
7072 )
7073 .unwrap();
7074 let (module_tx, _module_rx) = mpsc::channel(8);
7075 let endpoint = forwarding
7076 .register_module_connection(
7077 module_connection,
7078 module_id.to_string(),
7079 PROTOCOL_VERSION,
7080 manifest_concurrency(®istration.manifest),
7081 FrameSink::new(module_tx),
7082 )
7083 .unwrap();
7084 let (client_ctx, _client_rx) = route_ctx(client_connection);
7085 let pending = forwarding
7086 .begin_route_bind_relay_for_test(
7087 client_connection,
7088 client_ctx.egress.clone(),
7089 1,
7090 module_id,
7091 )
7092 .unwrap();
7093 assert_eq!(pending.endpoint, endpoint);
7094 let route_channel = pending.client_channel;
7095 let route_epoch = pending.client_epoch;
7096 forwarding
7097 .complete_pending_relay(
7098 module_connection,
7099 pending.corr,
7100 RouteBindRelayOutcome::Accepted,
7101 )
7102 .unwrap();
7103 (client_ctx, route_channel, route_epoch)
7104 }
7105
7106 struct FakeProcessLiveness {
7107 live: Option<bool>,
7108 }
7109
7110 impl ModuleProcessLiveness for FakeProcessLiveness {
7111 fn process_live(&self, _module_id: &str) -> Option<bool> {
7112 self.live
7113 }
7114 }
7115
7116 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7117 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
7118 {
7119 let registry = Arc::new(Registry::default());
7120 let supervisor_handle = SupervisorHandle::new();
7121 let supervisor = Supervisor::new_for_test(
7122 Arc::clone(®istry),
7123 RestartPolicy::new(1, Duration::from_millis(10)),
7124 )
7125 .with_handle(supervisor_handle.clone());
7126 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
7127 let module = supervisor
7128 .spawn(ModuleSpec {
7129 module_id: "stderr-tail-wire".to_string(),
7130 program: fake_aft_stub_path(),
7131 args: Vec::new(),
7132 env: vec![
7133 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
7134 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
7135 ],
7136 reserved: false,
7137 reserved_prefixes: Vec::new(),
7138 protocol: ModuleProtocol::Subc,
7139 overlap: Default::default(),
7140 })
7141 .unwrap();
7142
7143 let deadline = Instant::now() + Duration::from_secs(5);
7144 loop {
7145 let tail = module.stderr_tail(None, None);
7146 if tail
7147 .entries
7148 .iter()
7149 .any(|entry| matches!(entry, TailEntry::ProcessStart))
7150 && tail.entries.iter().any(|entry| {
7151 matches!(
7152 entry,
7153 TailEntry::Line {
7154 truncated: true,
7155 ..
7156 }
7157 )
7158 })
7159 {
7160 break;
7161 }
7162 assert!(
7163 Instant::now() < deadline,
7164 "module did not produce a truncated line and restart boundary: {tail:?}"
7165 );
7166 sleep(Duration::from_millis(10)).await;
7167 }
7168
7169 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7170 let request = ClientControlRequest::SupervisorStderrTail {
7171 module_id: "stderr-tail-wire".to_string(),
7172 max_lines: None,
7173 max_bytes: None,
7174 };
7175 let frame = Frame::build(
7176 FrameType::Request,
7177 control_flags(),
7178 0,
7179 0,
7180 1,
7181 serde_json::to_vec(&request).unwrap(),
7182 )
7183 .unwrap();
7184 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7185 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7186 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
7187 serde_json::from_slice(&responses[0].body).unwrap()
7188 else {
7189 panic!("expected supervisor.stderr_tail response");
7190 };
7191
7192 assert!(
7193 tail.entries
7194 .iter()
7195 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
7196 "the control response lost the restart boundary"
7197 );
7198 let Some(StderrTailEntry::Line {
7199 text,
7200 truncated,
7201 at_ms,
7202 }) = tail.entries.iter().find(|entry| {
7203 matches!(
7204 entry,
7205 StderrTailEntry::Line {
7206 truncated: true,
7207 ..
7208 }
7209 )
7210 })
7211 else {
7212 panic!("the control response lost the truncated line");
7213 };
7214 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
7215 assert!(*truncated);
7216 assert!(
7217 at_ms.is_some(),
7218 "the control response lost the line's capture time"
7219 );
7220 }
7221
7222 #[tokio::test(flavor = "current_thread")]
7227 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
7228 let dir = TestTempDir::new("terminals-off-worker");
7229 let journal_path = dir.join("terminals.jsonl");
7230 let registry = Arc::new(Registry::default());
7231 let supervisor_handle = SupervisorHandle::new();
7232 let supervisor =
7233 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7234 .with_handle(supervisor_handle.clone())
7235 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7236 let module = supervisor
7237 .spawn(ModuleSpec {
7238 module_id: "terminal-off-worker".to_string(),
7239 program: fake_aft_stub_path(),
7240 args: Vec::new(),
7241 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7242 reserved: false,
7243 reserved_prefixes: Vec::new(),
7244 protocol: ModuleProtocol::Subc,
7245 overlap: Default::default(),
7246 })
7247 .unwrap();
7248 let deadline = Instant::now() + Duration::from_secs(5);
7249 while module.terminal_history().entries.len() != 2 {
7250 assert!(Instant::now() < deadline, "module did not record two exits");
7251 sleep(Duration::from_millis(10)).await;
7252 }
7253
7254 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
7255 let handler =
7256 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
7257 let frame = Frame::build(
7258 FrameType::Request,
7259 control_flags(),
7260 0,
7261 0,
7262 1,
7263 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
7264 module_id: "terminal-off-worker".to_string(),
7265 })
7266 .unwrap(),
7267 )
7268 .unwrap();
7269 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7270 let spawned_at = std::time::Instant::now();
7271 let read = tokio::spawn({
7272 let handler = Arc::clone(&handler);
7273 async move { handler.handle_control_frame(&ctx, frame).await }
7274 });
7275 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
7278 .await
7279 .unwrap()
7280 .expect("the history read reached its pause");
7281 let elapsed = spawned_at.elapsed();
7282 assert!(
7283 elapsed < Duration::from_secs(2) && !read.is_finished(),
7284 "this task could not run while the history read was paused \
7285 (resumed after {elapsed:?}, read finished: {})",
7286 read.is_finished()
7287 );
7288
7289 drop(release);
7290 let responses = read.await.unwrap().unwrap();
7291 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7292 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
7293 panic!("expected supervisor.terminals response");
7294 };
7295 assert_eq!(terminals.entries.len(), 2);
7296 assert_eq!(terminals.journal_skipped_lines, 0);
7297 assert_eq!(terminals.journal_read_errors, 0);
7298 }
7299
7300 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7301 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
7302 let registry = Arc::new(Registry::default());
7303 let supervisor_handle = SupervisorHandle::new();
7304 let supervisor =
7305 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7306 .with_handle(supervisor_handle.clone());
7307 let module = supervisor
7308 .spawn(ModuleSpec {
7309 module_id: "terminal-golden".to_string(),
7310 program: fake_aft_stub_path(),
7311 args: Vec::new(),
7312 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7313 reserved: false,
7314 reserved_prefixes: Vec::new(),
7315 protocol: ModuleProtocol::Subc,
7316 overlap: Default::default(),
7317 })
7318 .unwrap();
7319
7320 let deadline = Instant::now() + Duration::from_secs(5);
7321 while module.terminal_history().entries.len() != 2 {
7322 assert!(
7323 Instant::now() < deadline,
7324 "module did not retain two terminal exits: {:?}",
7325 module.terminal_history()
7326 );
7327 sleep(Duration::from_millis(10)).await;
7328 }
7329
7330 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7331 let request = ClientControlRequest::SupervisorTerminals {
7332 module_id: "terminal-golden".to_string(),
7333 };
7334 let frame = Frame::build(
7335 FrameType::Request,
7336 control_flags(),
7337 0,
7338 0,
7339 1,
7340 serde_json::to_vec(&request).unwrap(),
7341 )
7342 .unwrap();
7343 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7344 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7345 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7346 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
7347 panic!("expected supervisor.terminals response");
7348 };
7349 assert_eq!(terminals.entries.len(), 2);
7350 assert_eq!(terminals.dropped, 0);
7351
7352 let mut rendered = serde_json::to_value(response).unwrap();
7353 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
7356 for (index, entry) in rendered["entries"]
7357 .as_array_mut()
7358 .expect("terminal response entries array")
7359 .iter_mut()
7360 .enumerate()
7361 {
7362 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
7363 }
7364
7365 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7366 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
7367 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
7368 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7369 std::fs::write(&golden_path, &serialized).unwrap();
7370 }
7371 let expected: Value =
7372 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
7373 assert_eq!(rendered, expected);
7374 }
7375
7376 #[test]
7377 fn hello_registers_manifest_and_returns_ack() {
7378 let registry = Arc::new(Registry::default());
7379 let handler = ControlHandler::new(Arc::clone(®istry));
7380 let conn = ConnectionId::new(1);
7381
7382 let responses = handler
7383 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
7384 .unwrap();
7385
7386 assert_eq!(responses.len(), 1);
7387 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7388 assert_eq!(responses[0].header.channel, 0);
7389 assert_eq!(responses[0].header.corr, 7);
7390 let ack = parse_ack(&responses[0]);
7391 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
7392 assert!(ack
7393 .subc_capabilities
7394 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
7395 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
7396 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
7397 assert!(ack
7398 .subc_ops
7399 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
7400 assert!(ack
7401 .subc_ops
7402 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
7403
7404 let registration = registry.get_module("aft").unwrap().unwrap();
7405 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
7406 assert_eq!(registration.state, ChannelState::Active);
7407 assert_eq!(registration.connection_id, conn);
7408 assert_eq!(registration.control_ops, module_baseline_control_ops());
7409 }
7410
7411 #[test]
7412 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7413 let invalid_identifiers = [
7414 ("case_change", "credentials-Provider/v1"),
7415 ("leading_zero", "credentials-provider/v01"),
7416 ("trailing_hyphen", "credentials-provider-/v1"),
7417 ("consecutive_hyphens", "credentials--provider/v1"),
7418 ("uppercase", "Credentials-provider/v1"),
7419 ("missing_v", "credentials-provider/1"),
7420 ("whitespace", "credentials provider/v1"),
7421 ("zero_version", "credentials-provider/v0"),
7422 ("out_of_range_version", "credentials-provider/v4294967296"),
7423 (
7424 "overlength_name",
7425 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7426 ),
7427 ];
7428 let mut cases = invalid_identifiers
7429 .into_iter()
7430 .map(|(name, identifier)| {
7431 (
7432 format!("identifier_{name}"),
7433 "capabilities.provides[0]".to_string(),
7434 identifier.to_string(),
7435 json!({ "provides": [identifier] }),
7436 None,
7437 )
7438 })
7439 .collect::<Vec<_>>();
7440 cases.extend([
7441 (
7442 "unknown_need".to_string(),
7443 "capabilities.requires[0].need".to_string(),
7444 "deferred".to_string(),
7445 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7446 None,
7447 ),
7448 (
7449 "duplicate_provides".to_string(),
7450 "capabilities.provides[1]".to_string(),
7451 "credentials-provider/v1".to_string(),
7452 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7453 None,
7454 ),
7455 (
7456 "duplicate_must_never_reach".to_string(),
7457 "capabilities.must_never_reach[1]".to_string(),
7458 "credentials-provider/v1".to_string(),
7459 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7460 None,
7461 ),
7462 (
7463 "duplicate_requires_same_need".to_string(),
7464 "capabilities.requires[1]".to_string(),
7465 "credentials-provider/v1".to_string(),
7466 json!({ "requires": [
7467 { "capability": "credentials-provider/v1", "need": "required" },
7468 { "capability": "credentials-provider/v1", "need": "required" }
7469 ] }),
7470 None,
7471 ),
7472 (
7473 "duplicate_requires_conflicting_need".to_string(),
7474 "capabilities.requires[1]".to_string(),
7475 "credentials-provider/v1".to_string(),
7476 json!({ "requires": [
7477 { "capability": "credentials-provider/v1", "need": "required" },
7478 { "capability": "credentials-provider/v1", "need": "optional" }
7479 ] }),
7480 None,
7481 ),
7482 (
7483 "capabilities_root_pointer".to_string(),
7484 "runtime_computed[0]".to_string(),
7485 "/capabilities".to_string(),
7486 json!({}),
7487 Some(json!(["/capabilities"])),
7488 ),
7489 (
7490 "capabilities_descendant_pointer".to_string(),
7491 "runtime_computed[0]".to_string(),
7492 "/capabilities/provides".to_string(),
7493 json!({}),
7494 Some(json!(["/capabilities/provides"])),
7495 ),
7496 (
7497 "malformed_pointer_without_leading_slash".to_string(),
7498 "runtime_computed[0]".to_string(),
7499 "capabilities".to_string(),
7500 json!({}),
7501 Some(json!(["capabilities"])),
7502 ),
7503 (
7504 "malformed_pointer_escape".to_string(),
7505 "runtime_computed[0]".to_string(),
7506 "/roles/~2/tools".to_string(),
7507 json!({}),
7508 Some(json!(["/roles/~2/tools"])),
7509 ),
7510 (
7511 "unknown_capabilities_field".to_string(),
7512 "capabilities.future".to_string(),
7513 "<array>".to_string(),
7514 json!({ "future": [] }),
7515 None,
7516 ),
7517 ]);
7518
7519 for (index, (name, field, value, capabilities, runtime_computed)) in
7520 cases.into_iter().enumerate()
7521 {
7522 let registry = Arc::new(Registry::default());
7523 let handler = ControlHandler::new(Arc::clone(®istry));
7524 let response = handler
7525 .handle_control(
7526 ConnectionId::new((index + 1) as u64),
7527 capability_grammar_hello_frame(
7528 capabilities,
7529 runtime_computed,
7530 index as u64 + 1,
7531 ),
7532 )
7533 .expect("invalid HELLO returns a refusal");
7534
7535 assert_eq!(response.len(), 1, "{name} must emit one refusal");
7536 let error = parse_error(&response[0]);
7537 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7538 let message = error["message"]
7539 .as_str()
7540 .expect("error message is a string");
7541 assert!(
7542 message.contains(&field),
7543 "{name}: field missing from {message}"
7544 );
7545 assert!(
7546 message.contains(&value),
7547 "{name}: value missing from {message}"
7548 );
7549 assert_eq!(
7550 registry
7551 .active_registration_count()
7552 .expect("registry reads"),
7553 0,
7554 "{name}: refused HELLO must not create a catalog entry"
7555 );
7556 }
7557 }
7558
7559 #[test]
7560 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7561 let registry = Arc::new(Registry::default());
7562 let handler = ControlHandler::new(Arc::clone(®istry));
7563 let capabilities = json!({
7564 "provides": ["credentials-provider/v1"],
7565 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7566 "must_never_reach": ["federation-transport/v1"]
7567 });
7568 let response = handler
7569 .handle_control(
7570 ConnectionId::new(99),
7571 capability_grammar_hello_frame(
7572 capabilities.clone(),
7573 Some(json!(["/roles/0/tools"])),
7574 99,
7575 ),
7576 )
7577 .expect("valid HELLO registers");
7578 assert_eq!(response[0].header.ty, FrameType::HelloAck);
7579
7580 let request = Frame::build(
7581 FrameType::Request,
7582 control_flags(),
7583 0,
7584 0,
7585 100,
7586 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7587 .expect("catalog request serializes"),
7588 )
7589 .expect("catalog request frame builds");
7590 let response = handler
7591 .handle_catalog_list(request, None)
7592 .expect("catalog list succeeds");
7593 let ClientControlResponse::CatalogList { modules, .. } =
7594 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7595 else {
7596 panic!("catalog request must return catalog.list");
7597 };
7598 assert_eq!(modules.len(), 1);
7599 assert_eq!(
7600 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7601 capabilities
7602 );
7603 }
7604
7605 #[test]
7606 fn catalog_list_mirrors_management_operation_description() {
7607 let registry = Arc::new(Registry::default());
7608 let handler = ControlHandler::new(Arc::clone(®istry));
7609 let description = "List managed records and return their identifiers and metadata.";
7610 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7611 manifest.provides = vec![ProviderRole::ManagementSurface {
7612 operations: vec![ManagementOperation {
7613 name: "records.list".to_string(),
7614 kind: ManagementOperationKind::Query,
7615 description: Some(description.to_string()),
7616 }],
7617 config_schema: json!({"type": "object"}),
7618 observability: vec![ObservabilitySurface {
7619 name: "records.stats".to_string(),
7620 kind: ObservabilityKind::Snapshot,
7621 }],
7622 identity_scope: vec![IdentityScope::Project],
7623 concurrency: Concurrency::ModuleManaged,
7624 }];
7625 registry
7626 .register_with_control_ops(
7627 manifest,
7628 PROTOCOL_VERSION,
7629 ConnectionId::new(99),
7630 Vec::new(),
7631 )
7632 .expect("described management manifest registers");
7633
7634 let request = Frame::build(
7635 FrameType::Request,
7636 control_flags(),
7637 0,
7638 0,
7639 100,
7640 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7641 .expect("catalog request serializes"),
7642 )
7643 .expect("catalog request frame builds");
7644 let response = handler
7645 .handle_catalog_list(request, None)
7646 .expect("catalog list succeeds");
7647 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7648 assert_eq!(
7649 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7650 "catalog.list must preserve the declared operation description verbatim"
7651 );
7652 }
7653
7654 #[test]
7655 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7656 let registry = Arc::new(Registry::default());
7657 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7658 [("vault".to_string(), true), ("squatter".to_string(), true)],
7659 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7660 );
7661 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7662 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7663 provides: vec!["credentials-provider/v1".to_string()],
7664 requires: Vec::new(),
7665 must_never_reach: Vec::new(),
7666 });
7667 let frame = Frame::build(
7668 FrameType::Hello,
7669 control_flags(),
7670 0,
7671 0,
7672 77,
7673 serde_json::to_vec(&ModuleHelloBody {
7674 manifest: squatter,
7675 protocol_ver: PROTOCOL_VERSION,
7676 control_ops: None,
7677 launch_nonce: None,
7678 })
7679 .expect("HELLO serializes"),
7680 )
7681 .expect("HELLO frame builds");
7682 let response = handler
7683 .handle_control(ConnectionId::new(77), frame)
7684 .expect("reserved claim receives a typed refusal");
7685 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7686 assert_eq!(
7687 registry
7688 .active_registration_count()
7689 .expect("registry reads"),
7690 0,
7691 "a reserved capability refusal must not leave a catalog entry"
7692 );
7693 }
7694
7695 #[test]
7696 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
7697 for settlement in ["timeout", "inconclusive", "drop"] {
7698 let breakers = RouteBindBreakers::default();
7699 let RouteBindAdmission::Admitted {
7700 guard: mut old,
7701 probe: false,
7702 } = breakers.admit("prov")
7703 else {
7704 panic!("ordinary relay admitted")
7705 };
7706 let RouteBindAdmission::Admitted {
7707 guard: mut opener, ..
7708 } = breakers.admit("prov")
7709 else {
7710 panic!("second relay admitted")
7711 };
7712 assert!(
7713 !opener
7714 .record_timeout(1, Duration::ZERO)
7715 .unwrap()
7716 .reopened_after_probe
7717 );
7718 let RouteBindAdmission::Admitted {
7719 guard: mut probe,
7720 probe: true,
7721 } = breakers.admit("prov")
7722 else {
7723 panic!("one cooldown probe admitted")
7724 };
7725 match settlement {
7726 "timeout" => assert!(
7727 !old.record_timeout(1, Duration::ZERO)
7728 .unwrap()
7729 .reopened_after_probe
7730 ),
7731 "inconclusive" => old.record_inconclusive(),
7732 "drop" => drop(old),
7733 _ => unreachable!(),
7734 }
7735 assert!(
7736 matches!(
7737 breakers.admit("prov"),
7738 RouteBindAdmission::Refused {
7739 probe_in_flight: true,
7740 ..
7741 }
7742 ),
7743 "{settlement} of a pre-open relay cannot release the real probe"
7744 );
7745 assert!(
7746 probe
7747 .record_timeout(1, Duration::ZERO)
7748 .unwrap()
7749 .reopened_after_probe
7750 );
7751 assert!(matches!(
7752 breakers.admit("prov"),
7753 RouteBindAdmission::Admitted { probe: true, .. }
7754 ));
7755 }
7756 let breakers = RouteBindBreakers::default();
7757 let admit = || match breakers.admit("prov") {
7758 RouteBindAdmission::Admitted { guard, .. } => guard,
7759 _ => panic!("relay admitted"),
7760 };
7761 admit().record_timeout(1, Duration::ZERO);
7762 let mut old_probe = admit();
7763 breakers.reset_for_new_module_connection("prov");
7764 admit().record_timeout(1, Duration::ZERO);
7765 let _new_probe = admit();
7766 old_probe.record_inconclusive();
7767 assert!(matches!(
7768 breakers.admit("prov"),
7769 RouteBindAdmission::Refused {
7770 probe_in_flight: true,
7771 ..
7772 }
7773 ));
7774 }
7775
7776 #[tokio::test]
7777 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
7778 for candidate in [false, true] {
7779 let registry = Arc::new(Registry::default());
7780 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7781 [("vault".to_string(), true), ("squatter".to_string(), true)],
7782 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7783 );
7784 let conn = ConnectionId::new(77);
7785 let (ctx, mut rx) = route_ctx(conn);
7786 let initial = capability_manifest("squatter", &[], &[]);
7787 if candidate {
7788 registry
7789 .register_candidate_with_control_ops(
7790 initial.clone(),
7791 PROTOCOL_VERSION,
7792 conn,
7793 module_baseline_control_ops(),
7794 )
7795 .unwrap();
7796 handler
7797 .forwarding
7798 .register_candidate_module_connection(
7799 conn,
7800 "squatter".to_string(),
7801 PROTOCOL_VERSION,
7802 manifest_concurrency(&initial),
7803 ctx.egress.clone(),
7804 )
7805 .unwrap();
7806 } else {
7807 hello_via_sink(
7808 &handler,
7809 &ctx,
7810 &mut rx,
7811 hello_frame_with_manifest(initial.clone(), 1),
7812 )
7813 .await;
7814 }
7815 let response = handler
7816 .handle_control_frame(
7817 &ctx,
7818 catalog_update_with_capabilities_frame(
7819 2,
7820 capability_manifest("squatter", &["credentials-provider/v1"], &[])
7821 .capabilities
7822 .unwrap(),
7823 ),
7824 )
7825 .await
7826 .unwrap();
7827 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7828 assert_eq!(
7829 registry
7830 .get_module_by_connection(conn)
7831 .unwrap()
7832 .unwrap()
7833 .manifest,
7834 initial
7835 );
7836 }
7837 }
7838
7839 #[test]
7840 fn server_describe_surfaces_required_capability_verdict_fields() {
7841 let registry = Arc::new(Registry::default());
7842 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7843 [
7844 ("consumer".to_string(), true),
7845 ("provider".to_string(), false),
7846 ],
7847 BTreeMap::new(),
7848 );
7849 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
7850 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7851 provides: Vec::new(),
7852 requires: vec![subc_protocol::manifest::CapabilityRequirement {
7853 capability: "credentials-provider/v1".to_string(),
7854 need: subc_protocol::manifest::CapabilityNeed::Required,
7855 }],
7856 must_never_reach: Vec::new(),
7857 });
7858 let hello = Frame::build(
7859 FrameType::Hello,
7860 control_flags(),
7861 0,
7862 0,
7863 78,
7864 serde_json::to_vec(&ModuleHelloBody {
7865 manifest: consumer,
7866 protocol_ver: PROTOCOL_VERSION,
7867 control_ops: None,
7868 launch_nonce: None,
7869 })
7870 .expect("HELLO serializes"),
7871 )
7872 .expect("HELLO frame builds");
7873 handler
7874 .handle_control(ConnectionId::new(78), hello)
7875 .expect("consumer registers");
7876 let describe = Frame::build(
7877 FrameType::Request,
7878 control_flags(),
7879 0,
7880 0,
7881 79,
7882 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
7883 .expect("request serializes"),
7884 )
7885 .expect("describe frame builds");
7886 let response = handler
7887 .handle_server_describe(describe)
7888 .expect("server.describe succeeds");
7889 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
7890 let requirement = &rendered["capability_requirements"][0];
7891 assert_eq!(requirement["consumer"], "consumer");
7892 assert_eq!(requirement["verdict"], "never_provided");
7893 assert_eq!(requirement["episode_seq"], 1);
7894 assert_eq!(requirement["config_satisfiable"], false);
7895 assert_eq!(requirement["runtime_available"], false);
7896 assert!(requirement["detail"]
7897 .as_str()
7898 .expect("detail string")
7899 .contains("credentials-provider/v1"));
7900 }
7901
7902 #[test]
7903 fn catalog_list_omits_capabilities_for_legacy_manifest() {
7904 let registry = Arc::new(Registry::default());
7905 let handler = ControlHandler::new(Arc::clone(®istry));
7906 let hello = handler
7907 .handle_control(
7908 ConnectionId::new(101),
7909 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
7910 )
7911 .expect("legacy HELLO registers");
7912 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
7913
7914 let request = Frame::build(
7915 FrameType::Request,
7916 control_flags(),
7917 0,
7918 0,
7919 102,
7920 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7921 .expect("catalog request serializes"),
7922 )
7923 .expect("catalog request frame builds");
7924 let response = handler
7925 .handle_catalog_list(request, None)
7926 .expect("catalog list succeeds");
7927 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7928 assert!(
7929 body["modules"][0].get("capabilities").is_none(),
7930 "legacy manifest must retain an absent capabilities field on catalog.list"
7931 );
7932 }
7933
7934 #[test]
7935 fn hello_ack_omits_storage_when_no_storage_config() {
7936 let registry = Arc::new(Registry::default());
7937 let handler = ControlHandler::new(Arc::clone(®istry));
7938 let responses = handler
7939 .handle_control(
7940 ConnectionId::new(1),
7941 hello_frame("aft", PROTOCOL_VERSION, 7),
7942 )
7943 .unwrap();
7944 let ack = parse_ack(&responses[0]);
7945 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
7946 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
7947 }
7948
7949 #[tokio::test]
7950 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
7951 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
7952 let registry = Arc::new(Registry::default());
7953 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
7954 let responses = handler
7955 .handle_control(
7956 ConnectionId::new(1),
7957 hello_frame("aft", PROTOCOL_VERSION, 7),
7958 )
7959 .unwrap();
7960 let ack = parse_ack(&responses[0]);
7961 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
7962
7963 let described = handler
7964 .handle_control_frame(
7965 &route_ctx(ConnectionId::new(2)).0,
7966 Frame::build(
7967 FrameType::Request,
7968 control_flags(),
7969 0,
7970 0,
7971 9,
7972 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
7973 )
7974 .unwrap(),
7975 )
7976 .await
7977 .unwrap();
7978 let ClientControlResponse::ServerDescribe { machine_id, .. } =
7979 serde_json::from_slice(&described[0].body).unwrap()
7980 else {
7981 panic!("server.describe answered with another shape");
7982 };
7983 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
7984 }
7985
7986 #[test]
7987 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
7988 let registry = Arc::new(Registry::default());
7991 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
7992 crate::daemon_config::StorageConfig::Sqlite {
7993 data_home: std::path::PathBuf::from("/data"),
7994 },
7995 ));
7996
7997 let responses = handler
7998 .handle_control(
7999 ConnectionId::new(1),
8000 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
8001 )
8002 .unwrap();
8003 let ack = parse_ack(&responses[0]);
8004 assert_eq!(
8005 ack.storage,
8006 Some(serde_json::json!({
8007 "module_id": "alfonso-routing",
8008 "storage_namespace": "default",
8009 "isolation": { "kind": "module" },
8010 "backend": {
8011 "backend": "sqlite",
8012 "path": "/data/cortexkit/alfonso-routing/store.db"
8013 }
8014 })),
8015 "the delivered descriptor is the module's own sqlite store path"
8016 );
8017 }
8018
8019 #[test]
8020 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
8021 let registry = Arc::new(Registry::default());
8022 let handler = ControlHandler::new(Arc::clone(®istry));
8023 let conn = ConnectionId::new(1);
8024 let responses = handler
8025 .handle_control(
8026 conn,
8027 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8028 )
8029 .unwrap();
8030 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8031 let registration = registry.get_module("aft").unwrap().unwrap();
8032 assert_eq!(registration.control_ops, module_baseline_control_ops());
8033
8034 let frame =
8035 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
8036 assert!(handler
8037 .guard_module_control_op(&frame, "aft", "route.bind")
8038 .unwrap()
8039 .is_none());
8040 let error = handler
8041 .guard_module_control_op(&frame, "aft", "test.synthetic")
8042 .unwrap()
8043 .expect("synthetic ungranted op should be rejected");
8044 assert_eq!(error.header.ty, FrameType::Error);
8045 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
8046 }
8047
8048 #[test]
8049 fn hello_control_ops_some_adds_optional_grants() {
8050 let registry = Arc::new(Registry::default());
8051 let handler = ControlHandler::new(Arc::clone(®istry));
8052 handler
8053 .handle_control(
8054 ConnectionId::new(1),
8055 hello_frame_with_control_ops(
8056 "aft",
8057 PROTOCOL_VERSION,
8058 7,
8059 Some(vec![
8060 "future.synthetic".to_string(),
8061 "route.bind".to_string(),
8062 ]),
8063 ),
8064 )
8065 .unwrap();
8066 let registration = registry.get_module("aft").unwrap().unwrap();
8067 assert_eq!(
8068 registration.control_ops,
8069 vec![
8070 "route.bind".to_string(),
8071 "route.status".to_string(),
8072 "future.synthetic".to_string(),
8073 ]
8074 );
8075 let frame =
8076 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
8077 assert!(handler
8078 .guard_module_control_op(&frame, "aft", "future.synthetic")
8079 .unwrap()
8080 .is_none());
8081 }
8082
8083 #[tokio::test]
8084 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
8085 let registry = Arc::new(Registry::default());
8086 let forwarding = Arc::new(ForwardingTable::default());
8087 let handler =
8088 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8089 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
8090 hello_via_sink(
8091 &handler,
8092 &module_ctx,
8093 &mut module_rx,
8094 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8095 )
8096 .await;
8097
8098 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
8099 let responses = handler
8100 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
8101 .await
8102 .unwrap();
8103 assert_eq!(responses.len(), 1);
8104 assert_eq!(responses[0].header.ty, FrameType::Error);
8105 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
8106 assert!(module_rx.try_recv().is_err());
8107 }
8108
8109 #[tokio::test]
8110 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
8111 let registry = Arc::new(Registry::default());
8112 let forwarding = Arc::new(ForwardingTable::default());
8113 let handler =
8114 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8115 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
8116 hello_via_sink(
8117 &handler,
8118 &module_ctx,
8119 &mut module_rx,
8120 hello_frame_with_control_ops(
8121 "aft",
8122 PROTOCOL_VERSION,
8123 7,
8124 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8125 ),
8126 )
8127 .await;
8128
8129 let project_root = unique_project_root("demux");
8130 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
8131 let route_handler = handler.clone();
8132 let route_task = tokio::spawn(async move {
8133 route_handler
8134 .handle_control_frame(
8135 &route_client_ctx,
8136 route_open_frame(100, "aft", project_root),
8137 )
8138 .await
8139 .unwrap()
8140 });
8141 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8142 .await
8143 .unwrap()
8144 .unwrap();
8145 assert!(matches!(
8146 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
8147 ModuleControlRequest::RouteBind { .. }
8148 ));
8149
8150 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
8151 let health_handler = handler.clone();
8152 let health_task = tokio::spawn(async move {
8153 health_handler
8154 .handle_control_frame(
8155 &health_client_ctx,
8156 supervisor_health_probe_frame(101, "aft"),
8157 )
8158 .await
8159 .unwrap()
8160 });
8161 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8162 .await
8163 .unwrap()
8164 .unwrap();
8165 assert_eq!(
8166 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8167 ModuleControlRequest::HealthCheck {}
8168 );
8169
8170 handler
8171 .handle_control_frame(
8172 &module_ctx,
8173 health_response(health_frame.header.corr, HealthStatus::Degraded),
8174 )
8175 .await
8176 .unwrap();
8177 let health_response = health_task.await.unwrap();
8178 assert_eq!(health_response.len(), 1);
8179 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8180 ClientControlResponse::SupervisorHealthProbe {
8181 module_id,
8182 status,
8183 detail,
8184 metrics,
8185 } => {
8186 assert_eq!(module_id, "aft");
8187 assert_eq!(status, HealthStatus::Degraded);
8188 assert_eq!(detail.as_deref(), Some("warming"));
8189 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
8190 }
8191 other => panic!("unexpected health response: {other:?}"),
8192 }
8193
8194 handler
8195 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8196 .await
8197 .unwrap();
8198 let route_response = route_task.await.unwrap();
8199 assert!(route_response.is_empty());
8200 let published = route_client_rx.recv().await.unwrap();
8201 assert!(matches!(
8202 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8203 ClientControlResponse::RouteOpen { .. }
8204 ));
8205 }
8206
8207 async fn relay_route_open(
8212 handler: &ControlHandler,
8213 client_connection: ConnectionId,
8214 client_egress: &FrameSink,
8215 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8216 corr: u64,
8217 module_id: &str,
8218 project_root_label: &str,
8219 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
8220 let ctx = RouteCtx {
8221 connection_id: client_connection,
8222 egress: client_egress.clone(),
8223 };
8224 let handler = handler.clone();
8225 let project_root = unique_project_root(project_root_label);
8226 let module_id = module_id.to_string();
8227 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
8228 let task = tokio::spawn(async move {
8229 let _guard = tracing::dispatcher::set_default(&dispatch);
8230 handler
8231 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8232 .await
8233 .unwrap()
8234 });
8235 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8236 .await
8237 .expect("module receives the relayed route.bind")
8238 .expect("module egress is open");
8239 (task, bind.frame)
8240 }
8241
8242 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8243 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8244 ModuleControlRequest::RouteBind {
8245 route_channel,
8246 epoch,
8247 ..
8248 } => (route_channel, epoch),
8249 other => panic!("expected a route.bind request, got {other:?}"),
8250 }
8251 }
8252
8253 fn published_route(frame: &Frame) -> (u16, u32) {
8254 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
8255 ClientControlResponse::RouteOpen {
8256 route_channel,
8257 route_epoch,
8258 } => (route_channel, route_epoch),
8259 other => panic!("expected a route.open response, got {other:?}"),
8260 }
8261 }
8262
8263 #[tokio::test]
8283 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
8284 let registry = Arc::new(Registry::default());
8285 let forwarding = Arc::new(ForwardingTable::default());
8286 let handler =
8287 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8288
8289 let module_connection = ConnectionId::new(30);
8290 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8291 hello_via_sink(
8292 &handler,
8293 &module_ctx,
8294 &mut module_rx,
8295 hello_frame("aft", PROTOCOL_VERSION, 7),
8296 )
8297 .await;
8298
8299 let dying_client = ConnectionId::new(31);
8300 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
8301
8302 let (first_task, first_bind) = relay_route_open(
8305 &handler,
8306 dying_client,
8307 &dying_ctx.egress,
8308 &mut module_rx,
8309 100,
8310 "aft",
8311 "closing-first",
8312 )
8313 .await;
8314 handler
8315 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
8316 .await
8317 .unwrap();
8318 assert!(first_task.await.unwrap().is_empty());
8319 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
8320
8321 let (second_task, second_bind) = relay_route_open(
8323 &handler,
8324 dying_client,
8325 &dying_ctx.egress,
8326 &mut module_rx,
8327 101,
8328 "aft",
8329 "closing-second",
8330 )
8331 .await;
8332 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
8333
8334 assert!(forwarding
8337 .escalate_client_delivery_failure(
8338 dying_client,
8339 first_channel,
8340 first_epoch,
8341 CloseReason::new(
8342 "module_to_client_delivery_failed",
8343 "client egress refused a module frame",
8344 ),
8345 crate::forwarding::UndeliveredFrame {
8346 module_id: None,
8347 sink: &dying_ctx.egress,
8348 },
8349 )
8350 .unwrap());
8351 assert!(!dying_ctx.egress.is_closed());
8352
8353 let ack = handler
8355 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
8356 .await;
8357 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
8358 if module_loop_error.is_some() {
8359 handler.cleanup_connection(module_connection).unwrap();
8363 }
8364 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8368 .await
8369 .ok()
8370 .flatten();
8371
8372 assert!(
8374 registry
8375 .get_module_by_connection(module_connection)
8376 .unwrap()
8377 .is_some(),
8378 "one client's closing connection ended the shared module connection: \
8379 {module_loop_error:?}"
8380 );
8381 let cotenant = ConnectionId::new(32);
8383 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
8384 let (cotenant_task, cotenant_bind) = relay_route_open(
8385 &handler,
8386 cotenant,
8387 &cotenant_ctx.egress,
8388 &mut module_rx,
8389 102,
8390 "aft",
8391 "closing-cotenant",
8392 )
8393 .await;
8394 handler
8395 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
8396 .await
8397 .unwrap();
8398 assert!(cotenant_task.await.unwrap().is_empty());
8399 let (cotenant_channel, cotenant_epoch) =
8400 published_route(&cotenant_rx.recv().await.unwrap());
8401 assert!(matches!(
8402 forwarding
8403 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
8404 .unwrap(),
8405 DataRoute::Client(DataRouteState::Bound(_))
8406 ));
8407
8408 let goodbye = post_ack_module_frame
8411 .expect("module receives a GOODBYE for the abandoned route channel");
8412 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8413 assert_eq!(goodbye.header.channel, abandoned_channel);
8414 assert_eq!(goodbye.header.epoch, abandoned_epoch);
8415
8416 assert!(dying_rx.try_recv().is_err());
8420 let second_response = second_task.await.unwrap();
8421 assert_eq!(second_response.len(), 1);
8422 assert_eq!(
8423 parse_error(&second_response[0])["code"],
8424 "target_unavailable"
8425 );
8426 }
8427
8428 #[test]
8435 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
8436 let handler = ControlHandler::default();
8437 let module_connection = ConnectionId::new(30);
8438 let client_connection = ConnectionId::new(31);
8439
8440 handler
8441 .refuse_to_end_module_connection_for_a_client(
8442 module_connection,
8443 77,
8444 ForwardingError::ConnectionClosing {
8445 connection_id: client_connection,
8446 },
8447 )
8448 .expect("a closing client must never end the module connection");
8449
8450 assert!(matches!(
8451 handler.refuse_to_end_module_connection_for_a_client(
8452 module_connection,
8453 78,
8454 ForwardingError::ConnectionClosing {
8455 connection_id: module_connection,
8456 },
8457 ),
8458 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
8459 connection_id
8460 })) if connection_id == module_connection
8461 ));
8462 assert!(matches!(
8463 handler.refuse_to_end_module_connection_for_a_client(
8464 module_connection,
8465 79,
8466 ForwardingError::Poisoned,
8467 ),
8468 Err(RouterError::Forwarding(ForwardingError::Poisoned))
8469 ));
8470 assert!(matches!(
8471 handler.refuse_to_end_module_connection_for_a_client(
8472 module_connection,
8473 80,
8474 ForwardingError::StaleModuleEndpoint,
8475 ),
8476 Err(RouterError::Forwarding(
8477 ForwardingError::StaleModuleEndpoint
8478 ))
8479 ));
8480 }
8481
8482 #[tokio::test]
8495 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
8496 let registry = Arc::new(Registry::default());
8497 let forwarding = Arc::new(ForwardingTable::default());
8498 let supervisor = SupervisorHandle::new();
8499 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8500 let handler =
8501 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8502 .with_supervisor(supervisor);
8503
8504 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8505 hello_via_sink(
8506 &handler,
8507 &target_ctx,
8508 &mut target_rx,
8509 hello_frame("target", PROTOCOL_VERSION, 1),
8510 )
8511 .await;
8512
8513 let wrong_nonce = handler
8517 .handle_control_frame(
8518 &route_ctx(ConnectionId::new(91)).0,
8519 route_open_frame_with_admission_facts(
8520 20,
8521 "target",
8522 unique_project_root("admission-facts"),
8523 Some(subc_control::ConsumerIdentity {
8524 module_id: "fed".to_string(),
8525 launch_nonce: "not-the-real-nonce".to_string(),
8526 }),
8527 None,
8528 ),
8529 )
8530 .await
8531 .unwrap();
8532 assert_eq!(
8533 parse_error(&wrong_nonce[0])["code"],
8534 "bad_consumer_identity",
8535 "a mismatched launch nonce must be refused, not stamped Reserved"
8536 );
8537
8538 let never_spawned = handler
8542 .handle_control_frame(
8543 &route_ctx(ConnectionId::new(92)).0,
8544 route_open_frame_with_admission_facts(
8545 21,
8546 "target",
8547 unique_project_root("admission-facts"),
8548 Some(subc_control::ConsumerIdentity {
8549 module_id: "never-spawned".to_string(),
8550 launch_nonce: "any-nonce".to_string(),
8551 }),
8552 None,
8553 ),
8554 )
8555 .await
8556 .unwrap();
8557 assert_eq!(
8558 parse_error(&never_spawned[0])["code"],
8559 "bad_consumer_identity",
8560 "an unspawned module_id must be refused rather than accepted for lack of a record"
8561 );
8562 }
8563
8564 #[tokio::test]
8587 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
8588 let registry = Arc::new(Registry::default());
8589 let forwarding = Arc::new(ForwardingTable::default());
8590 let supervisor = SupervisorHandle::new();
8591 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8592 let handler =
8593 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8594 .with_supervisor(supervisor);
8595
8596 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
8597 hello_via_sink(
8598 &handler,
8599 &target_ctx,
8600 &mut target_rx,
8601 hello_frame("target", PROTOCOL_VERSION, 1),
8602 )
8603 .await;
8604
8605 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
8606 let route_handler = handler.clone();
8607 let route_task = tokio::spawn(async move {
8608 route_handler
8609 .handle_control_frame(
8610 &client_ctx,
8611 route_open_frame_with_admission_facts(
8612 30,
8613 "target",
8614 unique_project_root("admission-facts"),
8615 Some(subc_control::ConsumerIdentity {
8616 module_id: "fed".to_string(),
8617 launch_nonce: "fed-nonce".to_string(),
8618 }),
8619 None,
8620 ),
8621 )
8622 .await
8623 .unwrap()
8624 });
8625
8626 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8633 .await
8634 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
8635 .expect("module control channel closed before route.bind");
8636 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8637 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
8638 panic!("expected route.bind")
8639 };
8640 assert_eq!(
8641 principal,
8642 Some(Principal::Reserved {
8643 module_id: "fed".to_string()
8644 }),
8645 "a correctly attested consumer must be stamped Reserved for its own id"
8646 );
8647
8648 handler
8649 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8650 .await
8651 .unwrap();
8652 assert!(route_task.await.unwrap().is_empty());
8653 assert!(
8654 matches!(
8655 serde_json::from_slice::<ClientControlResponse>(
8656 &client_rx.recv().await.unwrap().body
8657 )
8658 .unwrap(),
8659 ClientControlResponse::RouteOpen { .. }
8660 ),
8661 "the route must actually open, not merely avoid an error"
8662 );
8663 }
8664
8665 #[tokio::test(start_paused = true)]
8666 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
8667 let registry = Arc::new(Registry::default());
8668 let forwarding = Arc::new(ForwardingTable::default());
8669 let supervisor = SupervisorHandle::new();
8670 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8671 let handler =
8672 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8673 .with_supervisor(supervisor);
8674
8675 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
8676 hello_via_sink(
8677 &handler,
8678 &target_ctx,
8679 &mut target_rx,
8680 hello_frame("target", PROTOCOL_VERSION, 1),
8681 )
8682 .await;
8683
8684 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
8685 let direct_handler = handler.clone();
8686 let direct_open = tokio::spawn(async move {
8687 direct_handler
8688 .handle_control_frame(
8689 &direct_ctx,
8690 route_open_frame(2, "target", unique_project_root("route-census-direct")),
8691 )
8692 .await
8693 .unwrap()
8694 });
8695 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8696 .await
8697 .expect("no direct route.bind within 5s")
8698 .expect("target control channel closed before direct route.bind");
8699 handler
8700 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
8701 .await
8702 .unwrap();
8703 assert!(direct_open.await.unwrap().is_empty());
8704 let _ = direct_rx.recv().await.unwrap();
8705
8706 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
8707 let reserved_handler = handler.clone();
8708 let reserved_open = tokio::spawn(async move {
8709 reserved_handler
8710 .handle_control_frame(
8711 &reserved_ctx,
8712 route_open_frame_with_admission_facts(
8713 3,
8714 "target",
8715 unique_project_root("admission-facts"),
8716 Some(ConsumerIdentity {
8717 module_id: "fed".to_string(),
8718 launch_nonce: "fed-nonce".to_string(),
8719 }),
8720 None,
8721 ),
8722 )
8723 .await
8724 .unwrap()
8725 });
8726 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8727 .await
8728 .expect("no reserved route.bind within 5s")
8729 .expect("target control channel closed before reserved route.bind");
8730 handler
8731 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
8732 .await
8733 .unwrap();
8734 assert!(reserved_open.await.unwrap().is_empty());
8735 let _ = reserved_rx.recv().await.unwrap();
8736
8737 forwarding
8738 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
8739 .unwrap();
8740 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
8741 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
8742 module_id: Some("target".to_string()),
8743 })
8744 .unwrap();
8745 let census_frame =
8746 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
8747 let response = handler
8748 .handle_control_frame(&census_ctx, census_frame)
8749 .await
8750 .unwrap()
8751 .pop()
8752 .unwrap();
8753 let actual: Value = serde_json::from_slice(&response.body).unwrap();
8754 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
8755 assert!(matches!(
8756 decoded,
8757 ClientControlResponse::SupervisorRoutes { .. }
8758 ));
8759 let routes = actual["modules"][0]["routes"].as_array().unwrap();
8760 assert_eq!(routes.len(), 2);
8761 assert!(routes.iter().all(|route| route["draining"] == true));
8762 assert!(
8765 routes.iter().all(|route| route["drain_reason"] == "reload"),
8766 "draining routes must name the drain's reason: {routes:?}"
8767 );
8768 assert!(routes.iter().any(|route| {
8769 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
8770 }));
8771 assert!(routes.iter().any(|route| {
8772 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
8773 }));
8774
8775 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8776 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
8777 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8778 std::fs::write(
8779 &golden_path,
8780 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
8781 )
8782 .unwrap();
8783 }
8784 let expected: Value =
8785 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
8786 assert_eq!(actual, expected);
8787 }
8788
8789 async fn query_live_roots(
8790 handler: &ControlHandler,
8791 module_ctx: &RouteCtx,
8792 ) -> ModuleControlResponseToModule {
8793 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
8794 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
8795 let response = handler
8796 .handle_control_frame(module_ctx, frame)
8797 .await
8798 .unwrap()
8799 .pop()
8800 .unwrap();
8801 serde_json::from_slice(&response.body).unwrap()
8802 }
8803
8804 #[tokio::test(start_paused = true)]
8805 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
8806 let registry = Arc::new(Registry::default());
8807 let forwarding = Arc::new(ForwardingTable::default());
8808 let handler = ControlHandler::with_forwarding(registry, forwarding);
8809 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
8810 hello_via_sink(
8811 &handler,
8812 &target_ctx,
8813 &mut target_rx,
8814 hello_frame("target", PROTOCOL_VERSION, 1),
8815 )
8816 .await;
8817 let root = unique_project_root("live-roots-known");
8818 let path = ProjectRootId::from_path_allowing_missing(root.path())
8819 .unwrap()
8820 .as_path()
8821 .to_path_buf();
8822 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
8823 let open_handler = handler.clone();
8824 let opened = tokio::spawn(async move {
8825 open_handler
8826 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
8827 .await
8828 .unwrap()
8829 });
8830 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8831 .await
8832 .unwrap()
8833 .unwrap();
8834 handler
8835 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
8836 .await
8837 .unwrap();
8838 assert!(opened.await.unwrap().is_empty());
8839 let _ = client_rx.recv().await.unwrap();
8840
8841 let root = unique_project_root("live-roots-pending");
8842 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
8843 .unwrap()
8844 .as_path()
8845 .to_path_buf();
8846 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
8847 let open_handler = handler.clone();
8848 let pending = tokio::spawn(async move {
8849 open_handler
8850 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
8851 .await
8852 .unwrap()
8853 });
8854 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8855 .await
8856 .unwrap()
8857 .unwrap();
8858 let actual = query_live_roots(&handler, &target_ctx).await;
8859 let ModuleControlResponseToModule::LiveRoots {
8860 roots,
8861 unknown_root_bindings,
8862 total_bindings,
8863 } = actual
8864 else {
8865 panic!("expected live roots")
8866 };
8867 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
8868 assert_eq!(unknown_root_bindings, 0);
8869 assert_eq!(
8870 roots.len(),
8871 2,
8872 "root-known arm must retain each canonical root"
8873 );
8874 assert_eq!(
8875 total_bindings,
8876 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8877 );
8878 let counts = roots
8879 .iter()
8880 .map(|root| (root.project_root.clone(), root.bound, root.pending))
8881 .collect::<Vec<_>>();
8882 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
8883 expected.sort_by(|a, b| a.0.cmp(&b.0));
8884 assert_eq!(
8885 counts, expected,
8886 "roots must sort by path and count pending separately"
8887 );
8888 handler
8889 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
8890 .await
8891 .unwrap();
8892 assert!(pending.await.unwrap().is_empty());
8893 }
8894
8895 #[tokio::test(start_paused = true)]
8896 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
8897 let forwarding = Arc::new(ForwardingTable::default());
8898 let handler =
8899 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8900 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
8901 hello_via_sink(
8902 &handler,
8903 &target_ctx,
8904 &mut target_rx,
8905 hello_frame("target", PROTOCOL_VERSION, 1),
8906 )
8907 .await;
8908 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
8909 let pending = forwarding
8910 .begin_route_bind_relay_for_test(
8911 client_ctx.connection_id,
8912 client_ctx.egress.clone(),
8913 2,
8914 "target",
8915 )
8916 .unwrap();
8917 forwarding
8918 .complete_pending_relay(
8919 target_ctx.connection_id,
8920 pending.corr,
8921 RouteBindRelayOutcome::Accepted,
8922 )
8923 .unwrap();
8924 let actual = query_live_roots(&handler, &target_ctx).await;
8925 let ModuleControlResponseToModule::LiveRoots {
8926 roots,
8927 unknown_root_bindings,
8928 total_bindings,
8929 } = actual
8930 else {
8931 panic!("expected live roots")
8932 };
8933 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
8934 assert_eq!(
8935 unknown_root_bindings, 1,
8936 "unknown-root arm must not read as no bindings"
8937 );
8938 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
8939 assert_eq!(
8940 total_bindings,
8941 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
8942 );
8943 }
8944
8945 #[tokio::test(start_paused = true)]
8953 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
8954 let forwarding = Arc::new(ForwardingTable::default());
8955 let handler =
8956 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
8957 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
8958 let replies = handler
8959 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
8960 .await
8961 .unwrap();
8962 let queued_by_hello = module_rx.len();
8963
8964 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
8965 let open_handler = handler.clone();
8966 let open = tokio::spawn(async move {
8967 open_handler
8968 .handle_control_frame(
8969 &client_ctx,
8970 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
8971 )
8972 .await
8973 .unwrap()
8974 });
8975 let mut spins = 0;
8977 while module_rx.len() == queued_by_hello {
8978 spins += 1;
8979 assert!(spins < 10_000, "route.open never queued a route.bind");
8980 tokio::task::yield_now().await;
8981 }
8982
8983 for reply in replies {
8985 module_ctx.egress.send(reply).await.unwrap();
8986 }
8987
8988 let first = module_rx.recv().await.unwrap().frame;
8989 assert_eq!(
8990 first.header.ty,
8991 FrameType::HelloAck,
8992 "the first frame a registering module reads must be its HELLO_ACK"
8993 );
8994 assert_eq!(first.header.corr, 7);
8995 let second = module_rx.recv().await.unwrap().frame;
8996 assert_eq!(second.header.ty, FrameType::Request);
8997 assert!(
8998 matches!(
8999 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
9000 ModuleControlRequest::RouteBind { .. }
9001 ),
9002 "the route.bind follows the ack"
9003 );
9004 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
9005
9006 handler
9007 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
9008 .await
9009 .unwrap();
9010 assert!(open.await.unwrap().is_empty());
9011 let _ = client_rx.recv().await.unwrap();
9012 }
9013
9014 #[tokio::test(start_paused = true)]
9015 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
9016 let handler = ControlHandler::with_forwarding(
9017 Arc::new(Registry::default()),
9018 Arc::new(ForwardingTable::default()),
9019 );
9020 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
9021 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
9022 hello_via_sink(
9023 &handler,
9024 &first_ctx,
9025 &mut first_rx,
9026 hello_frame("first", PROTOCOL_VERSION, 1),
9027 )
9028 .await;
9029 hello_via_sink(
9030 &handler,
9031 &second_ctx,
9032 &mut second_rx,
9033 hello_frame("second", PROTOCOL_VERSION, 2),
9034 )
9035 .await;
9036 let root = unique_project_root("second-only");
9037 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
9038 let cloned = handler.clone();
9039 let open = tokio::spawn(async move {
9040 cloned
9041 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
9042 .await
9043 .unwrap()
9044 });
9045 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
9046 .await
9047 .unwrap()
9048 .unwrap();
9049 let first = query_live_roots(&handler, &first_ctx).await;
9050 let second = query_live_roots(&handler, &second_ctx).await;
9051 assert!(
9052 matches!(
9053 first,
9054 ModuleControlResponseToModule::LiveRoots {
9055 total_bindings: 0,
9056 ..
9057 }
9058 ),
9059 "cross-module scope must not expose another module's roots"
9060 );
9061 assert!(
9062 matches!(
9063 second,
9064 ModuleControlResponseToModule::LiveRoots {
9065 total_bindings: 1,
9066 ..
9067 }
9068 ),
9069 "second module must see its pending route"
9070 );
9071 handler
9072 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
9073 .await
9074 .unwrap();
9075 assert!(open.await.unwrap().is_empty());
9076 }
9077
9078 #[tokio::test(start_paused = true)]
9079 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
9080 let handler = ControlHandler::with_forwarding(
9081 Arc::new(Registry::default()),
9082 Arc::new(ForwardingTable::default()),
9083 );
9084 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
9085 hello_via_sink(
9086 &handler,
9087 &target_ctx,
9088 &mut target_rx,
9089 hello_frame("target", PROTOCOL_VERSION, 1),
9090 )
9091 .await;
9092 let actual = query_live_roots(&handler, &target_ctx).await;
9093 let ModuleControlResponseToModule::LiveRoots {
9094 roots,
9095 unknown_root_bindings,
9096 total_bindings,
9097 } = actual
9098 else {
9099 panic!("expected live roots")
9100 };
9101 assert!(roots.is_empty());
9102 assert_eq!(unknown_root_bindings, 0);
9103 assert_eq!(total_bindings, 0);
9104 assert_eq!(
9105 total_bindings,
9106 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9107 );
9108 }
9109
9110 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
9117 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9118 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
9119 let text = std::fs::read_to_string(&path)
9120 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
9121 let vectors: Vec<(String, Value)> = text
9122 .lines()
9123 .filter(|line| !line.trim().is_empty())
9124 .map(|line| {
9125 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
9126 let id = entry["corpus_id"]
9127 .as_str()
9128 .expect("every vector carries a corpus_id")
9129 .to_string();
9130 (id, entry["package"].clone())
9131 })
9132 .collect();
9133 assert_eq!(
9137 vectors.len(),
9138 3,
9139 "vendored fed corpus changed size; re-sync from subc-federation"
9140 );
9141
9142 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
9157 let richest = vectors
9158 .iter()
9159 .filter_map(|(_, package)| package.as_object())
9160 .map(|object| {
9161 object
9162 .keys()
9163 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
9164 .count()
9165 })
9166 .max()
9167 .unwrap_or(0);
9168 assert!(
9169 richest >= 2,
9170 "vendored corpus no longer carries a package with unmodelled fields, \
9171 so the relay test can no longer distinguish a verbatim relay from a lossy one"
9172 );
9173
9174 vectors
9175 }
9176
9177 #[tokio::test]
9185 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
9186 for (corpus_id, package) in fed_admission_facts_vectors() {
9187 let registry = Arc::new(Registry::default());
9188 let forwarding = Arc::new(ForwardingTable::default());
9189 let supervisor = SupervisorHandle::new();
9190 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9191 let handler =
9192 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9193 .with_supervisor(supervisor)
9194 .with_admission_facts_config(
9195 Some("fed".to_string()),
9196 Some(vec!["target".to_string()]),
9197 );
9198
9199 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9200 hello_via_sink(
9201 &handler,
9202 &target_ctx,
9203 &mut target_rx,
9204 hello_frame("target", PROTOCOL_VERSION, 1),
9205 )
9206 .await;
9207
9208 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
9209 let route_handler = handler.clone();
9210 let expected = package.clone();
9211 let route_task = tokio::spawn(async move {
9212 route_handler
9213 .handle_control_frame(
9214 &client_ctx,
9215 route_open_frame_with_admission_facts(
9216 20,
9217 "target",
9218 unique_project_root("admission-facts"),
9219 Some(subc_control::ConsumerIdentity {
9220 module_id: "fed".to_string(),
9221 launch_nonce: "fed-nonce".to_string(),
9222 }),
9223 Some(package),
9224 ),
9225 )
9226 .await
9227 .unwrap()
9228 });
9229
9230 let bind_frame = target_rx.recv().await.unwrap();
9231 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9232 let ModuleControlRequest::RouteBind {
9233 admission_facts, ..
9234 } = bind
9235 else {
9236 panic!("{corpus_id}: expected route.bind")
9237 };
9238 assert_eq!(
9239 admission_facts,
9240 Some(expected),
9241 "{corpus_id}: relay must not add, drop or reshape any field"
9242 );
9243
9244 handler
9245 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9246 .await
9247 .unwrap();
9248 route_task.await.unwrap();
9249 }
9250 }
9251
9252 #[tokio::test]
9253 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
9254 let registry = Arc::new(Registry::default());
9255 let forwarding = Arc::new(ForwardingTable::default());
9256 let supervisor = SupervisorHandle::new();
9257 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9258 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
9259 let handler =
9260 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9261 .with_supervisor(supervisor)
9262 .with_admission_facts_config(
9263 Some("fed".to_string()),
9264 Some(vec!["target".to_string()]),
9265 );
9266
9267 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
9268 hello_via_sink(
9269 &handler,
9270 &target_ctx,
9271 &mut target_rx,
9272 hello_frame("target", PROTOCOL_VERSION, 1),
9273 )
9274 .await;
9275 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
9276 hello_via_sink(
9277 &handler,
9278 &other_ctx,
9279 &mut other_rx,
9280 hello_frame("other", PROTOCOL_VERSION, 2),
9281 )
9282 .await;
9283
9284 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
9285 let expected_facts = facts.clone();
9286 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
9287 let route_handler = handler.clone();
9288 let route_task = tokio::spawn(async move {
9289 route_handler
9290 .handle_control_frame(
9291 &client_ctx,
9292 route_open_frame_with_admission_facts(
9293 10,
9294 "target",
9295 unique_project_root("admission-facts"),
9296 Some(subc_control::ConsumerIdentity {
9297 module_id: "fed".to_string(),
9298 launch_nonce: "fed-nonce".to_string(),
9299 }),
9300 Some(facts.clone()),
9301 ),
9302 )
9303 .await
9304 .unwrap()
9305 });
9306 let bind_frame = target_rx.recv().await.unwrap();
9307 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9308 let ModuleControlRequest::RouteBind {
9309 admission_facts, ..
9310 } = bind
9311 else {
9312 panic!("expected route.bind")
9313 };
9314 assert_eq!(admission_facts, Some(expected_facts));
9315 handler
9316 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9317 .await
9318 .unwrap();
9319 assert!(route_task.await.unwrap().is_empty());
9320 assert!(matches!(
9321 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
9322 .unwrap(),
9323 ClientControlResponse::RouteOpen { .. }
9324 ));
9325
9326 let direct = handler
9327 .handle_control_frame(
9328 &route_ctx(ConnectionId::new(73)).0,
9329 route_open_frame_with_admission_facts(
9330 11,
9331 "target",
9332 unique_project_root("admission-facts"),
9333 None,
9334 Some(json!({"x": 1})),
9335 ),
9336 )
9337 .await
9338 .unwrap();
9339 assert_eq!(
9340 parse_error(&direct[0])["code"],
9341 "admission_facts_not_permitted"
9342 );
9343
9344 let different_reserved = handler
9345 .handle_control_frame(
9346 &route_ctx(ConnectionId::new(77)).0,
9347 route_open_frame_with_admission_facts(
9348 15,
9349 "target",
9350 unique_project_root("admission-facts"),
9351 Some(subc_control::ConsumerIdentity {
9352 module_id: "other".to_string(),
9353 launch_nonce: "other-nonce".to_string(),
9354 }),
9355 Some(json!({"x": 1})),
9356 ),
9357 )
9358 .await
9359 .unwrap();
9360 assert_eq!(
9361 parse_error(&different_reserved[0])["code"],
9362 "admission_facts_not_permitted"
9363 );
9364
9365 let other_target = handler
9366 .handle_control_frame(
9367 &route_ctx(ConnectionId::new(74)).0,
9368 route_open_frame_with_admission_facts(
9369 12,
9370 "other",
9371 unique_project_root("admission-facts"),
9372 Some(subc_control::ConsumerIdentity {
9373 module_id: "fed".to_string(),
9374 launch_nonce: "fed-nonce".to_string(),
9375 }),
9376 Some(json!({"x": 1})),
9377 ),
9378 )
9379 .await
9380 .unwrap();
9381 assert_eq!(
9382 parse_error(&other_target[0])["code"],
9383 "admission_facts_target_not_allowed"
9384 );
9385
9386 let nonexistent = handler
9387 .handle_control_frame(
9388 &route_ctx(ConnectionId::new(75)).0,
9389 route_open_frame_with_admission_facts(
9390 13,
9391 "missing",
9392 unique_project_root("admission-facts"),
9393 None,
9394 Some(json!({"x": 1})),
9395 ),
9396 )
9397 .await
9398 .unwrap();
9399 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
9400
9401 let described = handler
9402 .handle_control_frame(
9403 &route_ctx(ConnectionId::new(76)).0,
9404 Frame::build(
9405 FrameType::Request,
9406 control_flags(),
9407 0,
9408 0,
9409 14,
9410 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9411 )
9412 .unwrap(),
9413 )
9414 .await
9415 .unwrap();
9416 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9417 serde_json::from_slice(&described[0].body).unwrap()
9418 else {
9419 panic!("expected server.describe response")
9420 };
9421 assert!(capabilities
9422 .iter()
9423 .any(|cap| cap == "admission_facts_relay_v1"));
9424 }
9425
9426 #[tokio::test]
9427 async fn admission_facts_without_configured_carrier_are_rejected() {
9428 let registry = Arc::new(Registry::default());
9429 let forwarding = Arc::new(ForwardingTable::default());
9430 let handler = ControlHandler::with_forwarding(registry, forwarding);
9431 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
9432 hello_via_sink(
9433 &handler,
9434 &target_ctx,
9435 &mut target_rx,
9436 hello_frame("target", PROTOCOL_VERSION, 1),
9437 )
9438 .await;
9439
9440 let responses = handler
9441 .handle_control_frame(
9442 &route_ctx(ConnectionId::new(79)).0,
9443 route_open_frame_with_admission_facts(
9444 16,
9445 "target",
9446 unique_project_root("admission-facts"),
9447 None,
9448 Some(json!({"x": 1})),
9449 ),
9450 )
9451 .await
9452 .unwrap();
9453 assert_eq!(
9454 parse_error(&responses[0])["code"],
9455 "admission_facts_not_permitted"
9456 );
9457 }
9458
9459 #[tokio::test]
9460 async fn route_open_relays_consumer_capabilities_verbatim() {
9461 let registry = Arc::new(Registry::default());
9462 let forwarding = Arc::new(ForwardingTable::default());
9463 let handler =
9464 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9465 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
9466 hello_via_sink(
9467 &handler,
9468 &module_ctx,
9469 &mut module_rx,
9470 hello_frame("aft", PROTOCOL_VERSION, 7),
9471 )
9472 .await;
9473
9474 let expected = vec!["elicitation".to_string(), "roots".to_string()];
9475 let expected_for_request = expected.clone();
9476 let project_root = unique_project_root("consumer-capabilities-present");
9477 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
9478 let route_handler = handler.clone();
9479 let route_task = tokio::spawn(async move {
9480 route_handler
9481 .handle_control_frame(
9482 &client_ctx,
9483 route_open_frame_with_consumer_capabilities(
9484 401,
9485 "aft",
9486 project_root,
9487 Some(expected_for_request),
9488 ),
9489 )
9490 .await
9491 .unwrap()
9492 });
9493 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9494 .await
9495 .unwrap()
9496 .unwrap();
9497 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9498 let ModuleControlRequest::RouteBind {
9499 consumer_capabilities,
9500 ..
9501 } = bind
9502 else {
9503 panic!("expected route.bind request, got {bind:?}");
9504 };
9505 assert_eq!(consumer_capabilities, Some(expected.clone()));
9506
9507 handler
9508 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9509 .await
9510 .unwrap();
9511 let route_response = route_task.await.unwrap();
9512 assert!(route_response.is_empty());
9513 let published = client_rx.recv().await.unwrap();
9514 assert!(matches!(
9515 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9516 ClientControlResponse::RouteOpen { .. }
9517 ));
9518 }
9519
9520 #[tokio::test]
9521 async fn route_open_without_consumer_capabilities_relays_none() {
9522 let registry = Arc::new(Registry::default());
9523 let forwarding = Arc::new(ForwardingTable::default());
9524 let handler =
9525 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9526 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
9527 hello_via_sink(
9528 &handler,
9529 &module_ctx,
9530 &mut module_rx,
9531 hello_frame("aft", PROTOCOL_VERSION, 7),
9532 )
9533 .await;
9534
9535 let project_root = unique_project_root("consumer-capabilities-absent");
9536 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
9537 let route_handler = handler.clone();
9538 let route_task = tokio::spawn(async move {
9539 route_handler
9540 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
9541 .await
9542 .unwrap()
9543 });
9544 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9545 .await
9546 .unwrap()
9547 .unwrap();
9548 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9549 let ModuleControlRequest::RouteBind {
9550 consumer_capabilities,
9551 ..
9552 } = bind
9553 else {
9554 panic!("expected route.bind request, got {bind:?}");
9555 };
9556 assert_eq!(consumer_capabilities, None);
9557
9558 handler
9559 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9560 .await
9561 .unwrap();
9562 let route_response = route_task.await.unwrap();
9563 assert!(route_response.is_empty());
9564 let published = client_rx.recv().await.unwrap();
9565 assert!(matches!(
9566 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9567 ClientControlResponse::RouteOpen { .. }
9568 ));
9569 }
9570
9571 async fn bind_role_versions_for(
9575 sent: Option<BTreeMap<String, String>>,
9576 connection: u64,
9577 ) -> Option<BTreeMap<String, String>> {
9578 let registry = Arc::new(Registry::default());
9579 let forwarding = Arc::new(ForwardingTable::default());
9580 let handler =
9581 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9582 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
9583 hello_via_sink(
9584 &handler,
9585 &module_ctx,
9586 &mut module_rx,
9587 hello_frame("aft", PROTOCOL_VERSION, 7),
9588 )
9589 .await;
9590 let project_root = unique_project_root("role-versions");
9591 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
9592 let route_handler = handler.clone();
9593 let route_task = tokio::spawn(async move {
9594 route_handler
9595 .handle_control_frame(
9596 &client_ctx,
9597 route_open_frame_with_role_versions(403, "aft", project_root, sent),
9598 )
9599 .await
9600 .unwrap()
9601 });
9602 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9603 .await
9604 .expect("a well-formed route.open reaches the module as a bind")
9605 .unwrap();
9606 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9607 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
9608 panic!("expected route.bind request, got {bind:?}");
9609 };
9610 handler
9611 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9612 .await
9613 .unwrap();
9614 assert!(route_task.await.unwrap().is_empty());
9615 let published = client_rx.recv().await.unwrap();
9616 assert!(matches!(
9617 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9618 ClientControlResponse::RouteOpen { .. }
9619 ));
9620 role_versions
9621 }
9622
9623 #[tokio::test]
9624 async fn route_open_relays_role_versions_verbatim() {
9625 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
9626 assert_eq!(
9627 bind_role_versions_for(Some(sent.clone()), 141).await,
9628 Some(sent)
9629 );
9630 }
9631
9632 #[tokio::test]
9635 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
9636 assert_eq!(bind_role_versions_for(None, 143).await, None);
9637 assert_eq!(
9638 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
9639 None
9640 );
9641 }
9642
9643 #[tokio::test]
9644 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
9645 let registry = Arc::new(Registry::default());
9646 let forwarding = Arc::new(ForwardingTable::default());
9647 let handler =
9648 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9649 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
9650 hello_via_sink(
9651 &handler,
9652 &module_ctx,
9653 &mut module_rx,
9654 hello_frame("aft", PROTOCOL_VERSION, 7),
9655 )
9656 .await;
9657
9658 let nine: BTreeMap<String, String> = (0..9)
9659 .map(|index| (format!("role-{index}"), "v1".to_string()))
9660 .collect();
9661 for (label, malformed) in [
9662 (
9663 "invalid role name",
9664 role_versions(&[("Tool_Provider", "v1")]),
9665 ),
9666 ("invalid version", role_versions(&[("tool-provider", "v0")])),
9667 ("nine entries", nine),
9668 ] {
9669 let responses = tokio::time::timeout(
9672 Duration::from_secs(1),
9673 handler.handle_control_frame(
9674 &route_ctx(ConnectionId::new(148)).0,
9675 route_open_frame_with_role_versions(
9676 404,
9677 "aft",
9678 unique_project_root("role-versions-malformed"),
9679 Some(malformed),
9680 ),
9681 ),
9682 )
9683 .await
9684 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
9685 .unwrap();
9686 assert_eq!(responses.len(), 1, "{label}");
9687 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
9688 let error = parse_error(&responses[0]);
9689 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
9690 assert_eq!(
9691 error["detail"]["field"], "role_versions",
9692 "{label}: {error}"
9693 );
9694 assert!(
9695 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
9696 "{label}: a malformed declaration is terminal"
9697 );
9698 assert!(
9699 module_rx.try_recv().is_err(),
9700 "{label}: the module must never see a bind"
9701 );
9702 }
9703 }
9704
9705 #[tokio::test]
9709 async fn route_role_versions_capability_is_advertised() {
9710 let handler = ControlHandler::new(Arc::new(Registry::default()));
9711 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
9712 let ack = hello_via_sink(
9713 &handler,
9714 &ctx,
9715 &mut rx,
9716 hello_frame("m", PROTOCOL_VERSION, 1),
9717 )
9718 .await;
9719 let ack = parse_ack(&ack);
9720 assert!(
9721 ack.subc_capabilities
9722 .iter()
9723 .any(|c| c == "route-role-versions/v1"),
9724 "{:?}",
9725 ack.subc_capabilities
9726 );
9727
9728 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
9729 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
9730 let reply = handler
9731 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
9732 .await
9733 .unwrap()
9734 .pop()
9735 .unwrap();
9736 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9737 serde_json::from_slice(&reply.body).unwrap()
9738 else {
9739 panic!("not a server.describe reply");
9740 };
9741 assert!(
9742 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
9743 "{capabilities:?}"
9744 );
9745 }
9746
9747 #[tokio::test]
9748 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
9749 let registry = Arc::new(Registry::default());
9750 let forwarding = Arc::new(ForwardingTable::default());
9751 let handler =
9752 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9753 .with_health_probe_timeout(Duration::from_secs(5));
9754 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
9755 hello_via_sink(
9756 &handler,
9757 &module_ctx,
9758 &mut module_rx,
9759 non_routable_hello_frame_with_control_ops(
9760 "mcp",
9761 300,
9762 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9763 ),
9764 )
9765 .await;
9766 assert!(registry
9767 .get_module("mcp")
9768 .unwrap()
9769 .unwrap()
9770 .manifest
9771 .provides
9772 .is_empty());
9773
9774 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
9775 let route_response = handler
9776 .handle_control_frame(
9777 &route_client_ctx,
9778 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
9779 )
9780 .await
9781 .unwrap();
9782 assert_eq!(route_response[0].header.ty, FrameType::Error);
9783 assert_eq!(
9784 parse_error(&route_response[0])["code"],
9785 "target_unavailable"
9786 );
9787 assert!(parse_error(&route_response[0])["message"]
9788 .as_str()
9789 .unwrap()
9790 .contains("does not provide the requested target"));
9791 assert!(module_rx.try_recv().is_err());
9792
9793 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
9794 let health_handler = handler.clone();
9795 let health_task = tokio::spawn(async move {
9796 health_handler
9797 .handle_control_frame(
9798 &health_client_ctx,
9799 supervisor_health_probe_frame(302, "mcp"),
9800 )
9801 .await
9802 .unwrap()
9803 });
9804 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9805 .await
9806 .unwrap()
9807 .unwrap();
9808 assert_eq!(
9809 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9810 ModuleControlRequest::HealthCheck {}
9811 );
9812 handler
9813 .handle_control_frame(
9814 &module_ctx,
9815 health_response(health_frame.header.corr, HealthStatus::Ok),
9816 )
9817 .await
9818 .unwrap();
9819 let health_response = health_task.await.unwrap();
9820 assert_eq!(health_response[0].header.ty, FrameType::Response);
9821 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9822 ClientControlResponse::SupervisorHealthProbe {
9823 module_id, status, ..
9824 } => {
9825 assert_eq!(module_id, "mcp");
9826 assert_eq!(status, HealthStatus::Ok);
9827 }
9828 other => panic!("unexpected health response: {other:?}"),
9829 }
9830
9831 forwarding
9836 .cleanup_connection(module_ctx.connection_id)
9837 .unwrap();
9838 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
9839 let cleanup_response = tokio::time::timeout(
9840 Duration::from_millis(200),
9841 handler.handle_control_frame(
9842 &cleanup_probe_ctx,
9843 supervisor_health_probe_frame(303, "mcp"),
9844 ),
9845 )
9846 .await
9847 .expect("probe should fail immediately when the control lane is gone")
9848 .unwrap();
9849 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
9850 assert_eq!(
9851 parse_error(&cleanup_response[0])["code"],
9852 "target_unavailable"
9853 );
9854 assert!(parse_error(&cleanup_response[0])["message"]
9855 .as_str()
9856 .unwrap()
9857 .contains("no module connection"));
9858
9859 handler
9860 .cleanup_connection(module_ctx.connection_id)
9861 .unwrap();
9862 }
9863
9864 #[tokio::test]
9865 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
9866 let registry = Arc::new(Registry::default());
9867 let supervisor_handle = SupervisorHandle::new();
9868 let supervisor =
9869 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
9870 .with_handle(supervisor_handle.clone())
9871 .with_connection_file_path(
9872 std::env::temp_dir()
9873 .join(format!("subc-route-open-warming-{}", std::process::id())),
9874 );
9875 let module = supervisor
9876 .supervise_configured(
9877 ModuleSpec {
9878 module_id: "warming".to_string(),
9879 program: fake_aft_stub_path(),
9880 args: Vec::new(),
9881 env: Vec::new(),
9882 reserved: false,
9883 reserved_prefixes: Vec::new(),
9884 protocol: ModuleProtocol::Subc,
9885 overlap: Default::default(),
9886 },
9887 true,
9888 )
9889 .unwrap();
9890 assert_eq!(module.state().unwrap(), ModuleState::Running);
9891
9892 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9893 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
9894 let response = handler
9895 .handle_control_frame(
9896 &ctx,
9897 route_open_frame(304, "warming", unique_project_root("warming")),
9898 )
9899 .await
9900 .unwrap();
9901 module.stop().await.unwrap();
9902
9903 assert_eq!(response[0].header.ty, FrameType::Error);
9904 let error = parse_error(&response[0]);
9905 assert_eq!(error["code"], "module_warming");
9906 assert!(error["message"]
9907 .as_str()
9908 .unwrap()
9909 .contains("state=running, enabled=true, live=false"));
9910 }
9911
9912 #[test]
9913 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
9914 let handler = ControlHandler::new(Arc::new(Registry::default()));
9915 let capture = EventCapture::default();
9916 let _subscriber =
9917 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9918 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
9919 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
9920 let pending = (0..limit).collect::<Vec<_>>();
9921 let response = handler
9922 .route_open_capacity_refusal(
9923 &ctx,
9924 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
9925 "busy",
9926 pending.len(),
9927 limit,
9928 )
9929 .unwrap();
9930 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9931 let event = capture
9932 .events()
9933 .into_iter()
9934 .find(|event| {
9935 event.target == "control"
9936 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
9937 })
9938 .expect("connection admission refusal event");
9939 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9940 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9941 }
9942
9943 #[test]
9944 fn route_open_target_cap_logs_admission_reason_and_capacity() {
9945 let handler = ControlHandler::new(Arc::new(Registry::default()));
9946 let capture = EventCapture::default();
9947 let _subscriber =
9948 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9949 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
9950 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
9951 let guards = (0..limit)
9952 .map(|_| {
9953 handler
9954 .route_bind_concurrency
9955 .try_admit("busy", limit)
9956 .unwrap()
9957 })
9958 .collect::<Vec<_>>();
9959 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
9960 Err(in_flight) => in_flight,
9961 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
9962 };
9963 let response = handler
9964 .route_open_target_capacity_refusal(
9965 &ctx,
9966 &route_open_frame(397, "busy", unique_project_root("target-cap")),
9967 "busy",
9968 in_flight,
9969 )
9970 .unwrap();
9971 assert_eq!(parse_error(&response)["code"], "target_unavailable");
9972 let event = capture
9973 .events()
9974 .into_iter()
9975 .find(|event| {
9976 event.target == "control"
9977 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
9978 })
9979 .expect("target admission refusal event");
9980 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
9981 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
9982 drop(guards);
9983 }
9984
9985 #[tokio::test(flavor = "current_thread")]
9986 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
9987 let handler = ControlHandler::new(Arc::new(Registry::default()));
9988 let capture = EventCapture::default();
9989 let _subscriber =
9990 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9991 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
9992 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
9993
9994 let _ = handler
9995 .handle_client_control_request(
9996 &ctx,
9997 frame,
9998 ClientControlRequest::SupervisorSetEnabled {
9999 module_id: "broca".to_string(),
10000 enabled: false,
10001 },
10002 )
10003 .await
10004 .unwrap();
10005
10006 let events = capture
10007 .events()
10008 .into_iter()
10009 .filter(|event| {
10010 event.target == "control"
10011 && event.fields.get("message").map(String::as_str)
10012 == Some("supervisor request received")
10013 })
10014 .collect::<Vec<_>>();
10015 assert_eq!(events.len(), 1, "one supervisor request log line");
10016 let event = &events[0];
10017 assert_eq!(event.level, tracing::Level::INFO);
10018 assert_eq!(
10019 event.fields.get("op"),
10020 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
10021 );
10022 assert_eq!(
10023 event.fields.get("module_id"),
10024 Some(&"\"broca\"".to_string())
10025 );
10026 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
10027 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
10028 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
10029 }
10030
10031 #[tokio::test(flavor = "current_thread")]
10032 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
10033 let registry = Arc::new(Registry::default());
10034 let module_connection = ConnectionId::new(303);
10035 registry
10036 .register_with_control_ops(
10037 manifest("aft", PROTOCOL_VERSION),
10038 PROTOCOL_VERSION,
10039 module_connection,
10040 module_baseline_control_ops(),
10041 )
10042 .unwrap();
10043 let handler = ControlHandler::new(registry);
10044 let capture = EventCapture::default();
10045 let _subscriber =
10046 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10047 let (ctx, _rx) = route_ctx(module_connection);
10048 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10049
10050 let _ = handler
10051 .handle_client_control_request(
10052 &ctx,
10053 frame,
10054 ClientControlRequest::SupervisorSetEnabled {
10055 module_id: "broca".to_string(),
10056 enabled: false,
10057 },
10058 )
10059 .await
10060 .unwrap();
10061
10062 let callers = capture
10063 .events()
10064 .into_iter()
10065 .filter(|event| {
10066 event.fields.get("message").map(String::as_str)
10067 == Some("supervisor request received")
10068 })
10069 .map(|event| event.fields.get("caller").cloned())
10070 .collect::<Vec<_>>();
10071 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
10072 }
10073
10074 #[tokio::test(flavor = "current_thread")]
10075 async fn supervisor_list_does_not_log_request_received() {
10076 let handler = ControlHandler::new(Arc::new(Registry::default()));
10077 let capture = EventCapture::default();
10078 let _subscriber =
10079 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10080 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
10081 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
10082
10083 handler
10084 .handle_client_control_request(&ctx, frame, ClientControlRequest::SupervisorList {})
10085 .await
10086 .unwrap();
10087
10088 assert!(capture.events().into_iter().all(|event| {
10089 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10090 }));
10091 }
10092
10093 #[tokio::test(flavor = "current_thread")]
10094 async fn supervisor_rescan_preview_does_not_log_request_received() {
10095 let handler = ControlHandler::new(Arc::new(Registry::default()));
10096 let capture = EventCapture::default();
10097 let _subscriber =
10098 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10099 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
10100 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
10101
10102 handler
10103 .handle_client_control_request(
10104 &ctx,
10105 frame,
10106 ClientControlRequest::SupervisorRescan { preview: true },
10107 )
10108 .await
10109 .unwrap();
10110
10111 assert!(capture.events().into_iter().all(|event| {
10112 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10113 }));
10114 }
10115
10116 #[tokio::test]
10121 async fn route_open_refusal_names_the_check_that_refused() {
10122 let handler = ControlHandler::new(Arc::new(Registry::default()));
10123 let capture = EventCapture::default();
10124 let _subscriber =
10125 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10126 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10127 let response = handler
10128 .handle_control_frame(
10129 &ctx,
10130 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
10131 )
10132 .await
10133 .unwrap();
10134
10135 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10136 let event = capture
10137 .events()
10138 .into_iter()
10139 .find(|event| {
10140 event.target == "control"
10141 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10142 })
10143 .expect("route.open refusal event");
10144 assert_eq!(
10145 event.fields.get("reason"),
10146 Some(&"\"not_registered\"".to_string())
10147 );
10148 }
10149
10150 #[tokio::test]
10151 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
10152 let registry = Arc::new(Registry::default());
10153 let supervisor_handle = SupervisorHandle::new();
10154 let supervisor =
10155 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10156 .with_handle(supervisor_handle.clone())
10157 .with_connection_file_path(std::env::temp_dir().join(format!(
10158 "subc-route-open-refusal-info-{}",
10159 std::process::id()
10160 )));
10161 let module = supervisor
10162 .supervise_configured(
10163 ModuleSpec {
10164 module_id: "warming".to_string(),
10165 program: fake_aft_stub_path(),
10166 args: Vec::new(),
10167 env: Vec::new(),
10168 reserved: false,
10169 reserved_prefixes: Vec::new(),
10170 protocol: ModuleProtocol::Subc,
10171 overlap: Default::default(),
10172 },
10173 true,
10174 )
10175 .unwrap();
10176 assert_eq!(module.state().unwrap(), ModuleState::Running);
10177
10178 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10179 assert!(handler
10180 .counters()
10181 .snapshot()
10182 .get("route_open_refused_by_code")
10183 .is_none());
10184 let capture = EventCapture::default();
10185 let _subscriber =
10186 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10187 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
10188 let response = handler
10189 .handle_control_frame(
10190 &ctx,
10191 route_open_frame(394, "warming", unique_project_root("refusal-info")),
10192 )
10193 .await
10194 .unwrap();
10195 module.stop().await.unwrap();
10196
10197 assert_eq!(parse_error(&response[0])["code"], "module_warming");
10198 let event = capture
10199 .events()
10200 .into_iter()
10201 .find(|event| {
10202 event.target == "control"
10203 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
10204 })
10205 .expect("route.open refusal event");
10206 assert_eq!(
10207 event.fields.get("module_id"),
10208 Some(&"\"warming\"".to_string())
10209 );
10210 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
10211 assert_eq!(
10212 event.fields.get("reason"),
10213 Some(&"\"supervised_not_registered\"".to_string())
10214 );
10215 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
10216 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
10217 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
10218 assert_eq!(
10219 handler.counters().snapshot()["route_open_refused_by_code"],
10220 json!({ "module_warming": 1 })
10221 );
10222 }
10223
10224 const OUTAGE_START: &str = "route.open refusing module: not serving";
10225 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
10226
10227 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
10228 capture
10229 .events()
10230 .into_iter()
10231 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
10232 .collect()
10233 }
10234
10235 fn supervise_stub(
10236 registry: &Arc<Registry>,
10237 module_id: &str,
10238 enabled: bool,
10239 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
10240 let supervisor_handle = SupervisorHandle::new();
10241 let supervisor =
10242 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
10243 .with_handle(supervisor_handle.clone())
10244 .with_connection_file_path(std::env::temp_dir().join(format!(
10245 "subc-route-outage-{module_id}-{}",
10246 std::process::id()
10247 )));
10248 let module = supervisor
10249 .supervise_configured(
10250 ModuleSpec {
10251 module_id: module_id.to_string(),
10252 program: fake_aft_stub_path(),
10253 args: Vec::new(),
10254 env: Vec::new(),
10255 reserved: false,
10256 reserved_prefixes: Vec::new(),
10257 protocol: ModuleProtocol::Subc,
10258 overlap: Default::default(),
10259 },
10260 enabled,
10261 )
10262 .unwrap();
10263 (supervisor_handle, module)
10264 }
10265
10266 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
10267 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
10268 module_id: module_id.to_string(),
10269 drain_timeout_ms: Some(50),
10270 })
10271 .unwrap();
10272 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
10273 }
10274
10275 #[test]
10279 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
10280 let registry = Arc::new(Registry::default());
10281 let forwarding = Arc::new(ForwardingTable::default());
10282 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10283 let second = ControlHandler::with_forwarding(registry, forwarding);
10284 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
10285 }
10286
10287 #[tokio::test(flavor = "current_thread")]
10291 async fn route_open_unknown_module_refusals_add_no_outage_state() {
10292 let handler = ControlHandler::new(Arc::new(Registry::default()));
10293 let capture = EventCapture::default();
10294 let _subscriber =
10295 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10296 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
10297 for corr in 0..8 {
10298 let response = handler
10299 .handle_control_frame(
10300 &ctx,
10301 route_open_frame(
10302 380 + corr,
10303 &format!("nobody-{corr}"),
10304 unique_project_root("outage-unknown"),
10305 ),
10306 )
10307 .await
10308 .unwrap();
10309 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10310 }
10311
10312 assert_eq!(handler.route_outages.tracked_module_count(), 0);
10313 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
10314 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
10315 }
10316
10317 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10322 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
10323 let registry = Arc::new(Registry::default());
10324 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
10325 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10326 let capture = EventCapture::default();
10327 let _subscriber =
10328 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10329 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
10330 handler.route_outages.record_accepted("outage-restart");
10333
10334 let response = handler
10335 .handle_control_frame(
10336 &ctx,
10337 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
10338 )
10339 .await
10340 .unwrap();
10341 assert_eq!(response[0].header.ty, FrameType::Error);
10342 let starts = outage_lines(&capture, OUTAGE_START);
10343 assert_eq!(starts.len(), 1, "{starts:?}");
10344 assert_eq!(starts[0].level, tracing::Level::WARN);
10345 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
10346 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
10347 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
10348 handler.route_outages.record_accepted("outage-restart");
10349 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
10350
10351 let restart = handler
10352 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
10353 .await
10354 .unwrap();
10355 assert_eq!(
10356 restart[0].header.ty,
10357 FrameType::Response,
10358 "{:?}",
10359 parse_error(&restart[0])
10360 );
10361 handler
10362 .handle_control_frame(
10363 &ctx,
10364 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
10365 )
10366 .await
10367 .unwrap();
10368 module.stop().await.unwrap();
10369
10370 let starts = outage_lines(&capture, OUTAGE_START);
10371 assert_eq!(starts.len(), 2, "{starts:?}");
10372 assert_eq!(starts[1].level, tracing::Level::INFO);
10373 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
10374 }
10375
10376 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10380 async fn failed_operator_restart_leaves_no_operator_mark() {
10381 let registry = Arc::new(Registry::default());
10382 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
10383 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10384 let capture = EventCapture::default();
10385 let _subscriber =
10386 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10387 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
10388 handler.route_outages.record_accepted("outage-disabled");
10389
10390 let restart = handler
10391 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
10392 .await
10393 .unwrap();
10394 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
10395 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
10396
10397 handler
10398 .handle_control_frame(
10399 &ctx,
10400 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
10401 )
10402 .await
10403 .unwrap();
10404 let starts = outage_lines(&capture, OUTAGE_START);
10405 assert_eq!(starts.len(), 1, "{starts:?}");
10406 assert_eq!(starts[0].level, tracing::Level::WARN);
10407 }
10408
10409 #[tokio::test(flavor = "current_thread")]
10410 async fn route_open_unknown_module_escapes_target_module_id() {
10411 let handler = ControlHandler::new(Arc::new(Registry::default()));
10412 let capture = EventCapture::default();
10413 let _subscriber =
10414 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10415 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
10416 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10417 let response = handler
10418 .handle_control_frame(
10419 &ctx,
10420 route_open_frame(
10421 395,
10422 hostile_module_id,
10423 unique_project_root("hostile-target-module-id"),
10424 ),
10425 )
10426 .await
10427 .unwrap();
10428
10429 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10430 let event = capture
10431 .events()
10432 .into_iter()
10433 .find(|event| {
10434 event.target == "control"
10435 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10436 })
10437 .expect("route.open unknown-module refusal event");
10438 let logged = event.fields.get("module_id").expect("module_id field");
10439 assert!(!logged.bytes().any(|byte| byte < 0x20));
10440 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10441 }
10442
10443 #[tokio::test(flavor = "current_thread")]
10444 async fn route_open_module_rejection_uses_daemon_counter_key() {
10445 let registry = Arc::new(Registry::default());
10446 let forwarding = Arc::new(ForwardingTable::default());
10447 let handler =
10448 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10449 let module_connection = ConnectionId::new(95);
10450 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10451 hello_via_sink(
10452 &handler,
10453 &module_ctx,
10454 &mut module_rx,
10455 hello_frame("aft", PROTOCOL_VERSION, 395),
10456 )
10457 .await;
10458
10459 let client_connection = ConnectionId::new(96);
10460 let (client_ctx, _client_rx) = route_ctx(client_connection);
10461 let capture = EventCapture::default();
10462 let _subscriber =
10463 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10464 let (route_task, bind) = relay_route_open(
10465 &handler,
10466 client_connection,
10467 &client_ctx.egress,
10468 &mut module_rx,
10469 396,
10470 "aft",
10471 "hostile-module-code",
10472 )
10473 .await;
10474 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
10475 let rejection = Frame::build(
10476 FrameType::Error,
10477 control_flags(),
10478 0,
10479 0,
10480 bind.header.corr,
10481 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
10482 )
10483 .unwrap();
10484 handler
10485 .handle_control_frame(&module_ctx, rejection)
10486 .await
10487 .unwrap();
10488
10489 let response = route_task.await.unwrap();
10490 assert_eq!(parse_error(&response[0])["code"], hostile_code);
10491 let counters = handler.counters().snapshot();
10492 assert_eq!(
10493 counters["route_open_refused_by_code"],
10494 json!({ "module_rejected": 1 })
10495 );
10496 assert!(counters["route_open_refused_by_code"]
10497 .get(hostile_code)
10498 .is_none());
10499
10500 let event = capture
10501 .events()
10502 .into_iter()
10503 .find(|event| {
10504 event.target == "control"
10505 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
10506 })
10507 .expect("route.open module-rejection refusal event");
10508 let logged = event.fields.get("module_code").expect("module_code field");
10509 assert!(!logged.bytes().any(|byte| byte < 0x20));
10510 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10511 }
10512
10513 #[tokio::test]
10514 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
10515 let registry = Arc::new(Registry::default());
10516 let supervisor_handle = SupervisorHandle::new();
10517 let missing_program = std::env::temp_dir().join(format!(
10518 "subc-route-open-missing-program-{}",
10519 std::process::id()
10520 ));
10521 let supervisor =
10522 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10523 .with_handle(supervisor_handle.clone());
10524 let module = supervisor
10525 .supervise_configured(
10526 ModuleSpec {
10527 module_id: "failed".to_string(),
10528 program: missing_program,
10529 args: Vec::new(),
10530 env: Vec::new(),
10531 reserved: false,
10532 reserved_prefixes: Vec::new(),
10533 protocol: ModuleProtocol::Subc,
10534 overlap: Default::default(),
10535 },
10536 true,
10537 )
10538 .unwrap();
10539 assert_eq!(module.state().unwrap(), ModuleState::Failed);
10540
10541 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10542 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
10543 let response = handler
10544 .handle_control_frame(
10545 &ctx,
10546 route_open_frame(305, "failed", unique_project_root("failed")),
10547 )
10548 .await
10549 .unwrap();
10550
10551 assert_eq!(response[0].header.ty, FrameType::Error);
10552 let error = parse_error(&response[0]);
10553 assert_eq!(error["code"], "target_unavailable");
10554 assert!(error["message"]
10555 .as_str()
10556 .unwrap()
10557 .contains("state=failed, enabled=true, live=false"));
10558 }
10559
10560 #[tokio::test]
10561 async fn route_open_role_mismatch_remains_target_unavailable() {
10562 let registry = Arc::new(Registry::default());
10563 let handler = ControlHandler::new(Arc::clone(®istry));
10564 handler
10565 .handle_control(
10566 ConnectionId::new(41),
10567 non_routable_hello_frame_with_control_ops("health-only", 306, None),
10568 )
10569 .unwrap();
10570
10571 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
10572 let response = handler
10573 .handle_control_frame(
10574 &ctx,
10575 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
10576 )
10577 .await
10578 .unwrap();
10579
10580 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10581 assert!(parse_error(&response[0])["message"]
10582 .as_str()
10583 .unwrap()
10584 .contains("does not provide the requested target"));
10585 }
10586
10587 #[tokio::test]
10588 async fn route_open_inactive_registration_remains_target_unavailable() {
10589 let registry = Arc::new(Registry::default());
10590 let handler = ControlHandler::new(Arc::clone(®istry));
10591 handler
10592 .handle_control(
10593 ConnectionId::new(43),
10594 hello_frame("inactive", PROTOCOL_VERSION, 308),
10595 )
10596 .unwrap();
10597 assert!(registry
10598 .set_module_state_for_test("inactive", ChannelState::Closed)
10599 .unwrap());
10600
10601 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
10602 let response = handler
10603 .handle_control_frame(
10604 &ctx,
10605 route_open_frame(309, "inactive", unique_project_root("inactive")),
10606 )
10607 .await
10608 .unwrap();
10609
10610 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10611 assert!(parse_error(&response[0])["message"]
10612 .as_str()
10613 .unwrap()
10614 .contains("is not active"));
10615 }
10616
10617 #[tokio::test]
10618 async fn late_health_reply_is_recorded_through_the_module_response_path() {
10619 let registry = Arc::new(Registry::default());
10620 let forwarding = Arc::new(ForwardingTable::default());
10621 let supervisor_handle = SupervisorHandle::new();
10622 let supervisor =
10623 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
10624 .with_forwarding(Arc::clone(&forwarding))
10625 .with_handle(supervisor_handle.clone());
10626 let module = supervisor
10627 .supervise_configured(
10628 crate::ModuleSpec {
10629 module_id: "late-health-response".to_string(),
10630 program: PathBuf::from("disabled-module"),
10631 args: Vec::new(),
10632 env: Vec::new(),
10633 reserved: false,
10634 reserved_prefixes: Vec::new(),
10635 protocol: ModuleProtocol::Subc,
10636 overlap: Default::default(),
10637 },
10638 false,
10639 )
10640 .unwrap();
10641 let handler =
10642 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10643 .with_supervisor(supervisor_handle);
10644 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
10645 handler
10646 .handle_control_frame(
10647 &module_ctx,
10648 hello_frame_with_control_ops(
10649 "late-health-response",
10650 PROTOCOL_VERSION,
10651 7,
10652 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10653 ),
10654 )
10655 .await
10656 .unwrap();
10657 let probe_started_at = Instant::now() - Duration::from_millis(80);
10658 let pending = forwarding
10659 .begin_health_probe_rpc_for(
10660 "late-health-response",
10661 MODULE_CONTROL_OP_HEALTH_CHECK,
10662 probe_started_at,
10663 Instant::now() - Duration::from_millis(1),
10664 )
10665 .unwrap();
10666 assert!(forwarding
10667 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
10668 .unwrap());
10669
10670 let responses = handler
10671 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
10672 .await
10673 .unwrap();
10674
10675 assert!(responses.is_empty());
10676 let health = module.status().unwrap().health;
10677 assert_eq!(health.late_answer_count, 1);
10678 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
10679 }
10680
10681 #[tokio::test]
10682 async fn health_probe_timeout_and_module_death_are_typed() {
10683 let registry = Arc::new(Registry::default());
10684 let forwarding = Arc::new(ForwardingTable::default());
10685 let handler =
10686 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10687 .with_health_probe_timeout(Duration::from_millis(50));
10688 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10689 hello_via_sink(
10690 &handler,
10691 &module_ctx,
10692 &mut module_rx,
10693 hello_frame_with_control_ops(
10694 "aft",
10695 PROTOCOL_VERSION,
10696 7,
10697 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10698 ),
10699 )
10700 .await;
10701
10702 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10703 let responses = handler
10704 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
10705 .await
10706 .unwrap();
10707 assert_eq!(responses[0].header.ty, FrameType::Error);
10708 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
10709 let _ = module_rx.try_recv();
10710
10711 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
10712 let health_handler = handler.clone();
10713 let death_task = tokio::spawn(async move {
10714 health_handler
10715 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
10716 .await
10717 .unwrap()
10718 });
10719 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10720 .await
10721 .unwrap()
10722 .unwrap();
10723 handler
10724 .cleanup_connection(module_ctx.connection_id)
10725 .unwrap();
10726 let responses = death_task.await.unwrap();
10727 assert_eq!(responses[0].header.ty, FrameType::Error);
10728 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
10729 }
10730
10731 #[test]
10732 fn hello_requires_exact_protocol_version() {
10733 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
10734 let registry = Arc::new(Registry::default());
10735 let handler = ControlHandler::new(Arc::clone(®istry));
10736 let responses = handler
10737 .handle_control(
10738 ConnectionId::new(connection),
10739 hello_frame("aft", offered, 9),
10740 )
10741 .unwrap();
10742
10743 assert_eq!(responses.len(), 1);
10744 assert_eq!(responses[0].header.ty, FrameType::Error);
10745 let error = parse_error(&responses[0]);
10746 assert_eq!(error["code"], "version_unsupported");
10747 assert!(registry.get_module("aft").unwrap().is_none());
10748 assert_eq!(registry.active_registration_count().unwrap(), 0);
10749 }
10750 }
10751
10752 #[test]
10753 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
10754 let registry = Arc::new(Registry::default());
10755 let forwarding = Arc::new(ForwardingTable::default());
10756 let handler =
10757 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10758 let module_connection = ConnectionId::new(301);
10759 let registration = registry
10760 .register_with_control_ops(
10761 manifest("aft-push", PROTOCOL_VERSION),
10762 PROTOCOL_VERSION,
10763 module_connection,
10764 module_baseline_control_ops(),
10765 )
10766 .unwrap();
10767 let (module_tx, _module_rx) = mpsc::channel(8);
10768 let endpoint = forwarding
10769 .register_module_connection(
10770 module_connection,
10771 "aft-push".to_string(),
10772 PROTOCOL_VERSION,
10773 manifest_concurrency(®istration.manifest),
10774 FrameSink::new(module_tx),
10775 )
10776 .unwrap();
10777
10778 let unknown = Frame::build(
10780 FrameType::Push,
10781 control_flags(),
10782 0,
10783 0,
10784 5,
10785 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
10786 )
10787 .unwrap();
10788 let out = handler.handle_status_update(endpoint, unknown).unwrap();
10789 assert!(
10790 out.is_empty(),
10791 "unknown push op must be ignored, got {out:?}"
10792 );
10793
10794 let malformed = Frame::build(
10796 FrameType::Push,
10797 control_flags(),
10798 0,
10799 0,
10800 6,
10801 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
10802 )
10803 .unwrap();
10804 let out = handler.handle_status_update(endpoint, malformed).unwrap();
10805 assert_eq!(out.len(), 1);
10806 assert_eq!(out[0].header.ty, FrameType::Error);
10807 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
10808 }
10809
10810 #[test]
10811 fn hello_rejected_when_connection_already_owns_client_routes() {
10812 let registry = Arc::new(Registry::default());
10813 let forwarding = Arc::new(ForwardingTable::default());
10814 let handler =
10815 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10816 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
10818 let client_connection = ConnectionId::new(202);
10819
10820 let responses = handler
10823 .handle_control(
10824 client_connection,
10825 hello_frame("aft-second", PROTOCOL_VERSION, 9),
10826 )
10827 .unwrap();
10828 assert_eq!(responses[0].header.ty, FrameType::Error);
10829 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
10830 assert!(registry.get_module("aft-second").unwrap().is_none());
10831 }
10832
10833 #[tokio::test]
10834 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
10835 let registry = Arc::new(Registry::default());
10836 let forwarding = Arc::new(ForwardingTable::default());
10837 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
10838 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
10839 hello_via_sink(
10840 &handler,
10841 &module_ctx,
10842 &mut module_rx,
10843 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
10844 )
10845 .await;
10846 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
10847 let pending = forwarding
10848 .begin_route_bind_relay_for_test(
10849 client_ctx.connection_id,
10850 client_ctx.egress.clone(),
10851 2,
10852 "alpha",
10853 )
10854 .unwrap();
10855 forwarding
10856 .complete_pending_relay(
10857 module_ctx.connection_id,
10858 pending.corr,
10859 RouteBindRelayOutcome::Accepted,
10860 )
10861 .unwrap();
10862 client_rx.try_recv().unwrap();
10863 for module_id in ["beta", "alpha"] {
10864 let replies = handler
10865 .handle_control_frame(
10866 &module_ctx,
10867 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
10868 )
10869 .await
10870 .unwrap();
10871 assert_eq!(replies.len(), 1, "second HELLO must be refused");
10872 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
10873 }
10874 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
10875 assert!(registry.get_module("beta").unwrap().is_none());
10876 assert!(matches!(
10877 forwarding
10878 .lookup_data_route(
10879 client_ctx.connection_id,
10880 pending.client_channel,
10881 pending.client_epoch,
10882 )
10883 .unwrap(),
10884 DataRoute::Client(DataRouteState::Bound(_))
10885 ));
10886 assert!(handler
10887 .hello_launch_nonces
10888 .lock()
10889 .unwrap()
10890 .presented(module_ctx.connection_id, Some("alpha-nonce")));
10891 assert!(module_rx.try_recv().is_err());
10892 }
10893
10894 #[test]
10895 fn reserved_module_hello_requires_matching_launch_nonce() {
10896 let registry = Arc::new(Registry::default());
10897 let supervisor = SupervisorHandle::new();
10898 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10901 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10902
10903 let no_nonce = handler
10905 .handle_control(
10906 ConnectionId::new(1),
10907 hello_frame("vault", PROTOCOL_VERSION, 1),
10908 )
10909 .unwrap();
10910 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
10911 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
10912 assert!(registry.get_module("vault").unwrap().is_none());
10913
10914 let wrong = handler
10916 .handle_control(
10917 ConnectionId::new(2),
10918 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
10919 )
10920 .unwrap();
10921 assert_eq!(wrong[0].header.ty, FrameType::Error);
10922 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
10923 assert!(registry.get_module("vault").unwrap().is_none());
10924
10925 let ok = handler
10927 .handle_control(
10928 ConnectionId::new(3),
10929 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
10930 )
10931 .unwrap();
10932 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
10933 assert!(registry.get_module("vault").unwrap().is_some());
10934 }
10935
10936 #[test]
10937 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
10938 let registry = Arc::new(Registry::default());
10939 let supervisor = SupervisorHandle::new();
10940 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10941 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10942 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10943
10944 let squat = handler
10945 .handle_control(
10946 ConnectionId::new(1),
10947 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
10948 )
10949 .unwrap();
10950 assert_eq!(squat[0].header.ty, FrameType::Error);
10951 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
10952 assert!(parse_error(&squat[0])["message"]
10953 .as_str()
10954 .unwrap()
10955 .contains("fed:"));
10956
10957 let accepted_peer = handler
10958 .handle_control(
10959 ConnectionId::new(2),
10960 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
10961 )
10962 .unwrap();
10963 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
10964
10965 let accepted_short = handler
10966 .handle_control(
10967 ConnectionId::new(3),
10968 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
10969 )
10970 .unwrap();
10971 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
10972
10973 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
10974 let response = handler
10975 .handle_control(
10976 ConnectionId::new(conn),
10977 hello_frame(module_id, PROTOCOL_VERSION, conn),
10978 )
10979 .unwrap();
10980 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
10981 }
10982 }
10983
10984 #[test]
10985 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
10986 let registry = Arc::new(Registry::default());
10987 let supervisor = SupervisorHandle::new();
10988 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
10989 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
10990 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
10991 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10992
10993 let owner_nonce = handler
10994 .handle_control(
10995 ConnectionId::new(1),
10996 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
10997 )
10998 .unwrap();
10999 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
11000 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
11001 assert!(registry.get_module("fed:special").unwrap().is_none());
11002
11003 let exact_nonce = handler
11004 .handle_control(
11005 ConnectionId::new(2),
11006 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
11007 )
11008 .unwrap();
11009 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
11010 assert!(registry.get_module("fed:special").unwrap().is_some());
11011 }
11012
11013 #[test]
11014 fn non_reserved_module_ignores_launch_nonce() {
11015 let registry = Arc::new(Registry::default());
11016 let handler = ControlHandler::new(Arc::clone(®istry));
11019 let no_nonce = handler
11020 .handle_control(
11021 ConnectionId::new(1),
11022 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
11023 )
11024 .unwrap();
11025 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
11026 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
11027
11028 let echoed_nonce = handler
11029 .handle_control(
11030 ConnectionId::new(2),
11031 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
11032 )
11033 .unwrap();
11034 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
11035 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
11036 }
11037
11038 #[test]
11039 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
11040 let handler = ControlHandler::default();
11041 let conn = ConnectionId::new(1);
11042 let malformed = Frame::build(
11043 FrameType::Hello,
11044 control_flags(),
11045 0,
11046 0,
11047 3,
11048 b"{not json".to_vec(),
11049 )
11050 .unwrap();
11051
11052 let error = handler.handle_control(conn, malformed).unwrap();
11053 assert_eq!(error[0].header.ty, FrameType::Error);
11054 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
11055
11056 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11057 let pong = handler.handle_control(conn, ping).unwrap();
11058 assert_eq!(pong[0].header.ty, FrameType::Pong);
11059 assert_eq!(pong[0].header.corr, 4);
11060 }
11061
11062 #[test]
11063 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
11064 let registry = Arc::new(Registry::default());
11065 let handler = ControlHandler::new(Arc::clone(®istry));
11066
11067 handler
11068 .handle_control(
11069 ConnectionId::new(1),
11070 hello_frame("aft", PROTOCOL_VERSION, 1),
11071 )
11072 .unwrap();
11073 let duplicate = handler
11074 .handle_control(
11075 ConnectionId::new(2),
11076 hello_frame("aft", PROTOCOL_VERSION, 2),
11077 )
11078 .unwrap();
11079
11080 assert_eq!(duplicate[0].header.ty, FrameType::Error);
11081 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11082 let registration = registry.get_module("aft").unwrap().unwrap();
11083 assert_eq!(registration.connection_id, ConnectionId::new(1));
11084 }
11085
11086 #[test]
11087 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
11088 let registry = Arc::new(Registry::default());
11089 let forwarding = Arc::new(ForwardingTable::default());
11090 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
11091 let handler =
11092 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11093 .with_process_liveness(process_liveness);
11094 let (ctx, route_channel, route_epoch) =
11095 bind_liveness_route(®istry, &forwarding, "aft-dead");
11096 let responses = handler
11097 .handle_route_poll(
11098 &ctx,
11099 route_poll_frame(41, PollKind::Liveness, route_channel),
11100 route_channel,
11101 route_epoch,
11102 PollKind::Liveness,
11103 )
11104 .unwrap();
11105
11106 assert_eq!(responses.len(), 1);
11107 assert_eq!(responses[0].header.ty, FrameType::Response);
11108 assert_route_poll_liveness(&responses[0], false);
11109 }
11110
11111 #[test]
11112 fn liveness_poll_without_process_source_uses_bound_route() {
11113 let registry = Arc::new(Registry::default());
11114 let forwarding = Arc::new(ForwardingTable::default());
11115 let handler =
11116 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11117 let (ctx, route_channel, route_epoch) =
11118 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
11119 let responses = handler
11120 .handle_route_poll(
11121 &ctx,
11122 route_poll_frame(42, PollKind::Liveness, route_channel),
11123 route_channel,
11124 route_epoch,
11125 PollKind::Liveness,
11126 )
11127 .unwrap();
11128
11129 assert_route_poll_liveness(&responses[0], true);
11130 }
11131
11132 #[test]
11133 fn liveness_poll_untracked_process_source_uses_bound_route() {
11134 let registry = Arc::new(Registry::default());
11135 let forwarding = Arc::new(ForwardingTable::default());
11136 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
11137 let handler =
11138 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11139 .with_process_liveness(process_liveness);
11140 let (ctx, route_channel, route_epoch) =
11141 bind_liveness_route(®istry, &forwarding, "aft-untracked");
11142 let responses = handler
11143 .handle_route_poll(
11144 &ctx,
11145 route_poll_frame(43, PollKind::Liveness, route_channel),
11146 route_channel,
11147 route_epoch,
11148 PollKind::Liveness,
11149 )
11150 .unwrap();
11151
11152 assert_route_poll_liveness(&responses[0], true);
11153 }
11154
11155 #[tokio::test]
11156 async fn unknown_op_returns_unknown_control_op() {
11157 let handler = ControlHandler::default();
11158 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
11159 let request = Frame::build(
11160 FrameType::Request,
11161 control_flags(),
11162 0,
11163 0,
11164 55,
11165 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
11166 )
11167 .unwrap();
11168
11169 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11170
11171 assert_eq!(response.len(), 1);
11172 assert_eq!(response[0].header.ty, FrameType::Error);
11173 assert_eq!(response[0].header.corr, 55);
11174 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
11175 }
11176
11177 #[tokio::test]
11178 async fn supervisor_provenance_rejects_unknown_exact_module() {
11179 let handler = ControlHandler::default();
11180 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
11181 let request = Frame::build(
11182 FrameType::Request,
11183 control_flags(),
11184 0,
11185 0,
11186 57,
11187 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
11188 )
11189 .unwrap();
11190
11191 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11192
11193 assert_eq!(response.len(), 1);
11194 assert_eq!(response[0].header.ty, FrameType::Error);
11195 assert_eq!(response[0].header.corr, 57);
11196 let error = parse_error(&response[0]);
11197 assert_eq!(error["code"], "unknown_module");
11198 assert_eq!(error["message"], "module_id 'missing' is not supervised");
11199 }
11200
11201 #[test]
11202 fn provenance_probe_override_keeps_handler_tests_deterministic() {
11203 let expected = subc_control::RunningImageAgreement::Unavailable {
11204 reason: subc_control::RunningImageUnavailableReason::HashFailed,
11205 };
11206 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
11207 assert_eq!(handler.provenance_probe_override, Some(expected));
11208 }
11209
11210 #[test]
11211 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
11212 let verdict = reload_verdict(
11213 std::path::Path::new("/bin/new"),
11214 Some(std::path::Path::new("/bin/old")),
11215 subc_control::RunningImageAgreement::Unavailable {
11216 reason: subc_control::RunningImageUnavailableReason::HashFailed,
11217 },
11218 );
11219 assert!(matches!(
11220 verdict.path,
11221 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
11222 if configured == std::path::Path::new("/bin/new")
11223 && spawned_from == std::path::Path::new("/bin/old")
11224 ));
11225 }
11226
11227 #[test]
11228 fn reload_verdict_detects_replaced_image_at_same_path() {
11229 let image = subc_control::RunningImageAgreement::Mismatch {
11230 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
11231 digest: "old".into(),
11232 },
11233 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
11234 digest: "new".into(),
11235 },
11236 };
11237 let verdict = reload_verdict(
11238 std::path::Path::new("/bin/same"),
11239 Some(std::path::Path::new("/bin/same")),
11240 image.clone(),
11241 );
11242 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11243 assert_eq!(verdict.image, image);
11244 }
11245
11246 #[test]
11247 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
11248 let image = subc_control::RunningImageAgreement::Unavailable {
11249 reason: subc_control::RunningImageUnavailableReason::NotRunning,
11250 };
11251 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
11252 assert_eq!(
11253 verdict.path,
11254 subc_control::ReloadPathAgreement::Unavailable {
11255 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
11256 }
11257 );
11258 assert_eq!(verdict.image, image);
11259
11260 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
11261 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11262 };
11263 let verdict = reload_verdict(
11264 std::path::Path::new("/bin/same"),
11265 Some(std::path::Path::new("/bin/same")),
11266 unconfirmed.clone(),
11267 );
11268 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11269 assert_eq!(verdict.image, unconfirmed);
11270 }
11271
11272 #[test]
11273 fn reload_verdict_preserves_each_image_unavailability_reason() {
11274 use subc_control::RunningImageUnavailableReason as Reason;
11275
11276 for reason in [
11277 Reason::NotRunning,
11278 Reason::UnsupportedPlatform,
11279 Reason::RunningExecutableUnreadable,
11280 Reason::SpawnedPathUnreadable,
11281 Reason::HashFailed,
11282 Reason::ProcessIdentityUnconfirmed,
11283 Reason::Unknown("future_probe_reason".to_string()),
11284 ] {
11285 let image = subc_control::RunningImageAgreement::Unavailable {
11286 reason: reason.clone(),
11287 };
11288 let verdict = reload_verdict(
11289 std::path::Path::new("/bin/same"),
11290 Some(std::path::Path::new("/bin/same")),
11291 image.clone(),
11292 );
11293 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11294 assert_eq!(verdict.image, image, "{reason:?}");
11295 }
11296 }
11297
11298 #[tokio::test]
11299 async fn malformed_control_bodies_return_invalid_control_body() {
11300 let handler = ControlHandler::default();
11301 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
11302
11303 for (corr, body) in [
11304 (56, br#"{"route_channel":1}"#.as_slice()),
11305 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
11306 (
11307 58,
11308 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
11309 ),
11310 ] {
11311 let request = Frame::build(
11312 FrameType::Request,
11313 control_flags(),
11314 0,
11315 0,
11316 corr,
11317 body.to_vec(),
11318 )
11319 .unwrap();
11320 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11321
11322 assert_eq!(response.len(), 1);
11323 assert_eq!(response[0].header.ty, FrameType::Error);
11324 assert_eq!(response[0].header.corr, corr);
11325 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
11326 }
11327 }
11328
11329 #[tokio::test]
11330 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
11331 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11332 let registry = Arc::new(Registry::default());
11333 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11334 let router = Router::with_control_handler(Arc::clone(&control));
11335 let connection = router.begin_connection();
11336 let (ctx, mut rx) = route_ctx(connection.id());
11337
11338 router
11339 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
11340 .await
11341 .unwrap();
11342 let response = rx.recv().await.unwrap();
11343 let ack = parse_ack(&response);
11344 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11345 let channel = 1;
11346
11347 let goodbye =
11348 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
11349 router.route_for_connection(&ctx, goodbye).await.unwrap();
11350 assert!(rx.try_recv().is_err());
11351 assert!(registry.get_module("aft").unwrap().is_none());
11352
11353 router
11354 .route_for_connection(&ctx, channel_request(channel, 13))
11355 .await
11356 .unwrap();
11357 let error_frame = rx.recv().await.unwrap();
11358 assert_eq!(error_frame.header.ty, FrameType::Error);
11359 assert_eq!(error_frame.header.channel, channel);
11360 let captured = crate::router::test_log::captured_logs(&logs);
11361 assert_eq!(
11362 captured
11363 .lines()
11364 .filter(|line| {
11365 line.contains("module_id=aft")
11366 && line.contains("reason=explicit_goodbye")
11367 && line.contains("module registration ended")
11368 })
11369 .count(),
11370 1,
11371 "unexpected GOODBYE registry log: {captured}"
11372 );
11373 }
11374
11375 #[tokio::test]
11376 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
11377 let registry = Arc::new(Registry::default());
11378 let handler = ControlHandler::new(registry).with_capability_config(
11379 [("prov".to_string(), true), ("cons".to_string(), true)],
11380 BTreeMap::new(),
11381 );
11382 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
11383 register_capability_manifest(
11384 &handler,
11385 &provider_ctx,
11386 &mut provider_rx,
11387 capability_manifest("prov", &["thing/v1"], &[]),
11388 1,
11389 )
11390 .await;
11391 let mut consumer = capability_manifest("cons", &[], &[]);
11392 consumer.capabilities.as_mut().unwrap().requires.push(
11393 subc_protocol::manifest::CapabilityRequirement {
11394 capability: "thing/v1".to_string(),
11395 need: subc_protocol::manifest::CapabilityNeed::Required,
11396 },
11397 );
11398 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
11399 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
11400 assert_eq!(
11401 handler.capability_evaluator.verdict("cons", "thing/v1"),
11402 Some(CapabilityVerdict::Provided)
11403 );
11404 let (mut client_rx, _) = open_route_for_capability_test(
11405 &handler,
11406 &provider_ctx,
11407 &mut provider_rx,
11408 703,
11409 3,
11410 "prov",
11411 None,
11412 )
11413 .await;
11414 let goodbye =
11415 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11416 handler
11417 .handle_control_frame(&provider_ctx, goodbye)
11418 .await
11419 .unwrap();
11420 assert_eq!(
11421 handler.capability_evaluator.verdict("cons", "thing/v1"),
11422 Some(CapabilityVerdict::NeverProvided)
11423 );
11424 let closed = client_rx
11425 .try_recv()
11426 .expect("GOODBYE pushes route.closed before route GOODBYE");
11427 assert!(
11428 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
11429 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
11430 );
11431 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
11432 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11433 }
11434
11435 #[tokio::test]
11436 async fn dropping_router_connection_releases_registration() {
11437 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11438 let registry = Arc::new(Registry::default());
11439 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11440 let router = Router::with_control_handler(Arc::clone(&control));
11441 let connection = router.begin_connection();
11442 let connection_id = connection.id();
11443 let (ctx, mut rx) = route_ctx(connection_id);
11444
11445 router
11446 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
11447 .await
11448 .unwrap();
11449 let response = rx.recv().await.unwrap();
11450 let ack = parse_ack(&response);
11451 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11452 assert!(registry.get_module("aft").unwrap().is_some());
11453
11454 drop(connection);
11455
11456 assert!(registry.get_module("aft").unwrap().is_none());
11457 assert_eq!(registry.active_registration_count().unwrap(), 0);
11458
11459 control
11460 .cleanup_connection(ConnectionId::new(u64::MAX))
11461 .unwrap();
11462 let captured = crate::router::test_log::captured_logs(&logs);
11463 println!("captured registry lifecycle logs:\n{captured}");
11464 let events: Vec<_> = captured
11465 .lines()
11466 .filter(|line| {
11467 line.contains("module registered module_id=aft ")
11468 || line.contains("module registration ended")
11469 })
11470 .collect();
11471 assert_eq!(
11472 events.len(),
11473 2,
11474 "unexpected registry lifecycle logs: {captured}"
11475 );
11476 assert!(events[0].contains("module registered module_id=aft "));
11477 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
11478 assert!(events[1].contains(&format!(
11479 "module_id=aft connection_id={}",
11480 connection_id.get()
11481 )));
11482 assert!(events[1].contains("reason=connection_closed"));
11483 assert!(events[1].contains("module registration ended"));
11484 assert_eq!(
11485 captured
11486 .lines()
11487 .filter(|line| line.contains("module registered module_id=aft "))
11488 .count(),
11489 1,
11490 "legacy registration admission line must appear once: {captured}"
11491 );
11492 }
11493
11494 #[tokio::test]
11495 async fn hello_registration_keeps_legacy_module_registered_line_once() {
11496 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11497 let registry = Arc::new(Registry::default());
11498 let control = ControlHandler::new(Arc::clone(®istry));
11499 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
11500 hello_via_sink(
11501 &control,
11502 &ctx,
11503 &mut rx,
11504 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
11505 )
11506 .await;
11507
11508 let captured = crate::router::test_log::captured_logs(&logs);
11509 let admissions: Vec<_> = captured
11510 .lines()
11511 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
11512 .collect();
11513 assert_eq!(
11514 admissions.len(),
11515 1,
11516 "legacy admission line must remain exactly once: {captured}"
11517 );
11518 assert!(admissions[0].contains("routable_provider=true"));
11519 assert!(admissions[0].contains("connection_id=777"));
11520 }
11521
11522 fn capability_manifest(
11523 module_id: &str,
11524 provides: &[&str],
11525 must_never_reach: &[&str],
11526 ) -> ModuleManifest {
11527 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11528 manifest.capabilities = Some(CapabilityDeclarations {
11529 provides: provides
11530 .iter()
11531 .map(|capability| (*capability).to_string())
11532 .collect(),
11533 requires: Vec::new(),
11534 must_never_reach: must_never_reach
11535 .iter()
11536 .map(|capability| (*capability).to_string())
11537 .collect(),
11538 });
11539 manifest
11540 }
11541
11542 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
11543 Frame::build(
11544 FrameType::Hello,
11545 control_flags(),
11546 0,
11547 0,
11548 corr,
11549 serde_json::to_vec(&ModuleHelloBody {
11550 protocol_ver: manifest.protocol_ver,
11551 manifest,
11552 control_ops: None,
11553 launch_nonce: None,
11554 })
11555 .expect("capability test HELLO serializes"),
11556 )
11557 .expect("capability test HELLO frame builds")
11558 }
11559
11560 fn catalog_update_with_capabilities_frame(
11561 corr: u64,
11562 capabilities: CapabilityDeclarations,
11563 ) -> Frame {
11564 Frame::build(
11565 FrameType::Request,
11566 control_flags(),
11567 0,
11568 0,
11569 corr,
11570 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11571 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
11572 capabilities: Some(capabilities),
11573 ready: None,
11574 })
11575 .expect("capability catalog.update serializes"),
11576 )
11577 .expect("capability catalog.update frame builds")
11578 }
11579
11580 async fn register_capability_manifest(
11581 handler: &ControlHandler,
11582 ctx: &RouteCtx,
11583 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11584 manifest: ModuleManifest,
11585 corr: u64,
11586 ) {
11587 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
11588 }
11589
11590 async fn open_route_for_capability_test(
11591 handler: &ControlHandler,
11592 target_ctx: &RouteCtx,
11593 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11594 client_connection_id: u64,
11595 corr: u64,
11596 target_module_id: &str,
11597 consumer_identity: Option<ConsumerIdentity>,
11598 ) -> (
11599 mpsc::Receiver<crate::router::OutboundFrame>,
11600 ModuleControlRequest,
11601 ) {
11602 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
11603 let route_handler = handler.clone();
11604 let target_module_id = target_module_id.to_string();
11605 let route_task = tokio::spawn(async move {
11606 route_handler
11607 .handle_control_frame(
11608 &client_ctx,
11609 route_open_frame_with_admission_facts(
11610 corr,
11611 &target_module_id,
11612 unique_project_root("admission-facts"),
11613 consumer_identity,
11614 None,
11615 ),
11616 )
11617 .await
11618 .expect("capability test route.open succeeds")
11619 });
11620 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
11621 .await
11622 .expect("capability test route.open must reach route.bind")
11623 .expect("target control receiver stays open");
11624 let bind_request: ModuleControlRequest =
11625 serde_json::from_slice(&bind.body).expect("route.bind decodes");
11626 handler
11627 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
11628 .await
11629 .expect("capability test route.bind ACK succeeds");
11630 assert!(route_task.await.expect("route.open task joins").is_empty());
11631 let opened = client_rx
11632 .recv()
11633 .await
11634 .expect("successful route.open publishes a response");
11635 assert!(matches!(
11636 serde_json::from_slice::<ClientControlResponse>(&opened.body),
11637 Ok(ClientControlResponse::RouteOpen { .. })
11638 ));
11639 (client_rx, bind_request)
11640 }
11641
11642 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
11643 assert_eq!(frame.header.ty, FrameType::Push);
11644 assert_eq!(frame.header.channel, 0);
11645 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
11646 .expect("route.closed control push decodes");
11647 let ClientControlPush::RouteClosed { channels, .. } = &push else {
11648 panic!("expected route.closed");
11649 };
11650 assert_eq!(channels.len(), 1, "exactly one violating route closed");
11651 let channels = channels.clone();
11652 assert_eq!(
11653 push,
11654 ClientControlPush::RouteClosed {
11655 module_id: target_module_id.to_string(),
11656 channels,
11657 reason: RouteCloseReason::CapabilityDenied,
11658 drained: false,
11659 abandoned: 0,
11660 excluded_subscriptions: 0,
11661 terminal: Some(false),
11662 }
11663 );
11664 }
11665
11666 #[tokio::test]
11667 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
11668 let registry = Arc::new(Registry::default());
11669 let forwarding = Arc::new(ForwardingTable::default());
11670 let supervisor = SupervisorHandle::new();
11671 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11672 let handler =
11673 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11674 .with_supervisor(supervisor);
11675 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
11676 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
11677 register_capability_manifest(
11678 &handler,
11679 &target_ctx,
11680 &mut target_rx,
11681 capability_manifest("target", &["credentials-provider/v1"], &[]),
11682 1,
11683 )
11684 .await;
11685 register_capability_manifest(
11686 &handler,
11687 &opener_ctx,
11688 &mut opener_rx,
11689 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11690 2,
11691 )
11692 .await;
11693
11694 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
11695 let replies = handler
11696 .handle_control_frame(
11697 &client_ctx,
11698 route_open_frame_with_admission_facts(
11699 3,
11700 "target",
11701 unique_project_root("admission-facts"),
11702 Some(ConsumerIdentity {
11703 module_id: "opener".to_string(),
11704 launch_nonce: "opener-nonce".to_string(),
11705 }),
11706 None,
11707 ),
11708 )
11709 .await
11710 .expect("denied route.open returns a typed frame");
11711 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11712 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11713 assert!(
11714 target_rx.try_recv().is_err(),
11715 "forbidden route.open must not relay route.bind"
11716 );
11717 }
11718
11719 #[tokio::test]
11720 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
11721 let registry = Arc::new(Registry::default());
11722 let forwarding = Arc::new(ForwardingTable::default());
11723 let supervisor = SupervisorHandle::new();
11724 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11725 let handler =
11726 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11727 .with_supervisor(supervisor);
11728 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
11729 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
11730 register_capability_manifest(
11731 &handler,
11732 &target_ctx,
11733 &mut target_rx,
11734 capability_manifest("target", &["credentials-provider/v1"], &[]),
11735 1,
11736 )
11737 .await;
11738 register_capability_manifest(
11739 &handler,
11740 &old_opener_ctx,
11741 &mut old_opener_rx,
11742 capability_manifest("opener", &[], &[]),
11743 2,
11744 )
11745 .await;
11746 let (mut client_rx, _) = open_route_for_capability_test(
11747 &handler,
11748 &target_ctx,
11749 &mut target_rx,
11750 712,
11751 3,
11752 "target",
11753 Some(ConsumerIdentity {
11754 module_id: "opener".to_string(),
11755 launch_nonce: "opener-nonce".to_string(),
11756 }),
11757 )
11758 .await;
11759 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11760
11761 handler
11762 .cleanup_connection(old_opener_ctx.connection_id)
11763 .expect("old opener registration cleans up");
11764 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
11765 register_capability_manifest(
11766 &handler,
11767 &new_opener_ctx,
11768 &mut new_opener_rx,
11769 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11770 4,
11771 )
11772 .await;
11773
11774 assert_capability_denied_push(
11775 client_rx
11776 .try_recv()
11777 .expect("HELLO deny addition must emit route.closed")
11778 .frame,
11779 "target",
11780 );
11781 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11782 assert!(matches!(
11783 target_rx.try_recv(),
11784 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
11785 ));
11786 }
11787
11788 #[tokio::test]
11789 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
11790 let registry = Arc::new(Registry::default());
11791 let forwarding = Arc::new(ForwardingTable::default());
11792 let supervisor = SupervisorHandle::new();
11793 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11794 let handler =
11795 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11796 .with_supervisor(supervisor);
11797 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
11798 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
11799 register_capability_manifest(
11800 &handler,
11801 &target_ctx,
11802 &mut target_rx,
11803 capability_manifest("target", &[], &[]),
11804 1,
11805 )
11806 .await;
11807 register_capability_manifest(
11808 &handler,
11809 &opener_ctx,
11810 &mut opener_rx,
11811 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11812 2,
11813 )
11814 .await;
11815 let (mut client_rx, _) = open_route_for_capability_test(
11816 &handler,
11817 &target_ctx,
11818 &mut target_rx,
11819 722,
11820 3,
11821 "target",
11822 Some(ConsumerIdentity {
11823 module_id: "opener".to_string(),
11824 launch_nonce: "opener-nonce".to_string(),
11825 }),
11826 )
11827 .await;
11828 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11829
11830 let replies = handler
11831 .handle_control_frame(
11832 &target_ctx,
11833 catalog_update_with_capabilities_frame(
11834 4,
11835 CapabilityDeclarations {
11836 provides: vec!["credentials-provider/v1".to_string()],
11837 requires: Vec::new(),
11838 must_never_reach: Vec::new(),
11839 },
11840 ),
11841 )
11842 .await
11843 .expect("claim catalog.update succeeds");
11844 assert!(matches!(
11845 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
11846 Ok(ModuleControlResponseToModule::CatalogUpdate {})
11847 ));
11848 assert_capability_denied_push(
11849 client_rx
11850 .try_recv()
11851 .expect("claim addition must emit route.closed")
11852 .frame,
11853 "target",
11854 );
11855 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11856 assert!(matches!(
11857 target_rx.try_recv(),
11858 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
11859 ));
11860 }
11861
11862 #[tokio::test]
11863 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
11864 let registry = Arc::new(Registry::default());
11865 let forwarding = Arc::new(ForwardingTable::default());
11866 let supervisor = SupervisorHandle::new();
11867 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11868 let handler =
11869 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11870 .with_supervisor(supervisor);
11871 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
11872 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
11873 register_capability_manifest(
11874 &handler,
11875 &target_ctx,
11876 &mut target_rx,
11877 capability_manifest("target", &["credentials-provider/v1"], &[]),
11878 1,
11879 )
11880 .await;
11881 register_capability_manifest(
11882 &handler,
11883 &opener_ctx,
11884 &mut opener_rx,
11885 capability_manifest("opener", &[], &[]),
11886 2,
11887 )
11888 .await;
11889 let (mut client_rx, _) = open_route_for_capability_test(
11890 &handler,
11891 &target_ctx,
11892 &mut target_rx,
11893 732,
11894 3,
11895 "target",
11896 Some(ConsumerIdentity {
11897 module_id: "opener".to_string(),
11898 launch_nonce: "opener-nonce".to_string(),
11899 }),
11900 )
11901 .await;
11902 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11903
11904 handler
11905 .handle_control_frame(
11906 &target_ctx,
11907 catalog_update_with_capabilities_frame(
11908 4,
11909 CapabilityDeclarations {
11910 provides: Vec::new(),
11911 requires: Vec::new(),
11912 must_never_reach: Vec::new(),
11913 },
11914 ),
11915 )
11916 .await
11917 .expect("claim removal catalog.update succeeds");
11918 assert_eq!(
11919 forwarding.active_binding_count().unwrap(),
11920 1,
11921 "removing an attested target claim must leave the route census unchanged"
11922 );
11923 assert!(
11924 client_rx.try_recv().is_err(),
11925 "claim removal must not emit route.closed capability_denied"
11926 );
11927 assert!(
11928 target_rx.try_recv().is_err(),
11929 "claim removal must not send the target a route GOODBYE"
11930 );
11931 }
11932
11933 #[tokio::test]
11936 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
11937 let registry = Arc::new(Registry::default());
11938 let forwarding = Arc::new(ForwardingTable::default());
11939 let supervisor = SupervisorHandle::new();
11940 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11941 let handler =
11942 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11943 .with_supervisor(supervisor);
11944 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
11945 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
11946 register_capability_manifest(
11947 &handler,
11948 &target_ctx,
11949 &mut target_rx,
11950 capability_manifest("target", &["credentials-provider/v1"], &[]),
11951 1,
11952 )
11953 .await;
11954 register_capability_manifest(
11955 &handler,
11956 &opener_ctx,
11957 &mut opener_rx,
11958 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11959 2,
11960 )
11961 .await;
11962
11963 let (_client_rx, bind) = open_route_for_capability_test(
11964 &handler,
11965 &target_ctx,
11966 &mut target_rx,
11967 742,
11968 3,
11969 "target",
11970 None,
11971 )
11972 .await;
11973 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11974 panic!("direct scope-honesty route must bind");
11975 };
11976 assert_eq!(principal, Some(Principal::Direct));
11977 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11978 }
11979
11980 #[tokio::test]
11983 async fn must_never_reach_self_route_is_capability_forbidden() {
11984 let registry = Arc::new(Registry::default());
11985 let forwarding = Arc::new(ForwardingTable::default());
11986 let supervisor = SupervisorHandle::new();
11987 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
11988 let handler =
11989 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11990 .with_supervisor(supervisor);
11991 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
11992 register_capability_manifest(
11993 &handler,
11994 &self_ctx,
11995 &mut self_rx,
11996 capability_manifest(
11997 "self-provider",
11998 &["credentials-provider/v1"],
11999 &["credentials-provider/v1"],
12000 ),
12001 1,
12002 )
12003 .await;
12004
12005 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
12006 let replies = handler
12007 .handle_control_frame(
12008 &client_ctx,
12009 route_open_frame_with_admission_facts(
12010 2,
12011 "self-provider",
12012 unique_project_root("admission-facts"),
12013 Some(ConsumerIdentity {
12014 module_id: "self-provider".to_string(),
12015 launch_nonce: "self-nonce".to_string(),
12016 }),
12017 None,
12018 ),
12019 )
12020 .await
12021 .expect("self-route refusal returns a typed frame");
12022 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
12023 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12024 assert!(
12025 self_rx.try_recv().is_err(),
12026 "self denial must not relay route.bind"
12027 );
12028 }
12029
12030 #[test]
12031 fn unsupported_channel_zero_frame_returns_error() {
12032 let handler = ControlHandler::default();
12033 let request = Frame::build(
12034 FrameType::Request,
12035 control_flags(),
12036 0,
12037 0,
12038 21,
12039 b"opaque".to_vec(),
12040 )
12041 .unwrap();
12042
12043 let response = handler
12044 .handle_control(ConnectionId::new(1), request)
12045 .unwrap();
12046
12047 assert_eq!(response[0].header.ty, FrameType::Error);
12048 assert_eq!(
12049 parse_error(&response[0])["code"],
12050 "unsupported_control_frame"
12051 );
12052 }
12053
12054 mod swap {
12059 use super::*;
12060
12061 const INCUMBENT: ConnectionId = ConnectionId::new(30);
12062 const CANDIDATE: ConnectionId = ConnectionId::new(40);
12063
12064 struct Swap {
12065 registry: Arc<Registry>,
12066 forwarding: Arc<ForwardingTable>,
12067 handler: ControlHandler,
12068 incumbent_ctx: RouteCtx,
12069 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12070 candidate_ctx: RouteCtx,
12071 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12072 }
12073
12074 async fn swap_with_incumbent() -> Swap {
12075 let registry = Arc::new(Registry::default());
12076 let forwarding = Arc::new(ForwardingTable::default());
12077 let handler =
12078 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12079 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
12080 hello_via_sink(
12081 &handler,
12082 &incumbent_ctx,
12083 &mut incumbent_rx,
12084 hello_frame("aft", PROTOCOL_VERSION, 7),
12085 )
12086 .await;
12087 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
12088 Swap {
12089 registry,
12090 forwarding,
12091 handler,
12092 incumbent_ctx,
12093 incumbent_rx,
12094 candidate_ctx,
12095 candidate_rx,
12096 }
12097 }
12098
12099 fn register_candidate(swap: &Swap, ready: Option<bool>) {
12100 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
12101 candidate_manifest.ready = ready;
12102 let registration = swap
12103 .registry
12104 .register_candidate_with_control_ops(
12105 candidate_manifest,
12106 PROTOCOL_VERSION,
12107 CANDIDATE,
12108 module_baseline_control_ops(),
12109 )
12110 .unwrap();
12111 swap.forwarding
12112 .register_candidate_module_connection(
12113 CANDIDATE,
12114 "aft".to_string(),
12115 PROTOCOL_VERSION,
12116 manifest_concurrency(®istration.manifest),
12117 swap.candidate_ctx.egress.clone(),
12118 )
12119 .unwrap();
12120 }
12121
12122 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
12123 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
12124 swap.registry.promote_candidate("aft").unwrap().unwrap();
12125 cutover.incumbent.unwrap()
12126 }
12127
12128 fn keyed_total(counters: &Value, key: &str) -> u64 {
12129 counters[key]
12130 .as_object()
12131 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
12132 .unwrap_or(0)
12133 }
12134
12135 #[tokio::test]
12136 async fn replacement_logs_old_end_and_new_admission_once() {
12137 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12138 let mut swap = swap_with_incumbent().await;
12139 swap.handler
12140 .supervisor
12141 .open_swap("aft", "candidate-nonce".to_string());
12142 hello_via_sink(
12143 &swap.handler,
12144 &swap.candidate_ctx,
12145 &mut swap.candidate_rx,
12146 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
12147 )
12148 .await;
12149 cutover(&swap);
12150 swap.handler.cleanup_connection(INCUMBENT).unwrap();
12151
12152 let captured = crate::router::test_log::captured_logs(&logs);
12153 println!("captured replacement registry logs:\n{captured}");
12154 let new_admissions: Vec<_> = captured
12155 .lines()
12156 .filter(|line| {
12157 line.contains("connection_id=40")
12158 && line.contains("swap candidate registered; not routable until cutover")
12159 })
12160 .collect();
12161 assert_eq!(
12162 new_admissions.len(),
12163 1,
12164 "unexpected admission logs: {captured}"
12165 );
12166 assert!(new_admissions[0].contains("module_id=aft"));
12167 assert!(new_admissions[0].contains("connection_id=40"));
12168 assert!(
12169 new_admissions[0].contains("swap candidate registered; not routable until cutover")
12170 );
12171 assert!(new_admissions[0].contains("ready=true"));
12172
12173 let old_admissions: Vec<_> = captured
12174 .lines()
12175 .filter(|line| {
12176 line.contains("module registered module_id=aft ")
12177 && line.contains("connection_id=30")
12178 })
12179 .collect();
12180 assert_eq!(
12181 old_admissions.len(),
12182 1,
12183 "unexpected incumbent admission: {captured}"
12184 );
12185
12186 let old_ends: Vec<_> = captured
12187 .lines()
12188 .filter(|line| {
12189 line.contains("connection_id=30") && line.contains("module registration ended")
12190 })
12191 .collect();
12192 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
12193 assert!(old_ends[0].contains("module_id=aft"));
12194 assert!(old_ends[0].contains("connection_id=30"));
12195 assert!(old_ends[0].contains("reason=replaced"));
12196 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
12197 assert_eq!(
12198 captured
12199 .lines()
12200 .filter(|line| line.contains("module registration promoted"))
12201 .count(),
12202 1,
12203 "unexpected promotion logs: {captured}"
12204 );
12205 }
12206
12207 #[tokio::test]
12213 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
12214 let mut swap = swap_with_incumbent().await;
12215 let handler = swap.handler.clone();
12216
12217 let cotenant = ConnectionId::new(31);
12219 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
12220 let (cotenant_task, cotenant_bind) = relay_route_open(
12221 &handler,
12222 cotenant,
12223 &cotenant_ctx.egress,
12224 &mut swap.incumbent_rx,
12225 100,
12226 "aft",
12227 "swap-cotenant",
12228 )
12229 .await;
12230 handler
12231 .handle_control_frame(
12232 &swap.incumbent_ctx,
12233 route_bind_ack(cotenant_bind.header.corr),
12234 )
12235 .await
12236 .unwrap();
12237 assert!(cotenant_task.await.unwrap().is_empty());
12238 let (cotenant_channel, cotenant_epoch) =
12239 published_route(&cotenant_rx.recv().await.unwrap());
12240
12241 let caller = ConnectionId::new(32);
12243 let (caller_ctx, mut caller_rx) = route_ctx(caller);
12244 let (caller_task, caller_bind) = relay_route_open(
12245 &handler,
12246 caller,
12247 &caller_ctx.egress,
12248 &mut swap.incumbent_rx,
12249 101,
12250 "aft",
12251 "swap-caller",
12252 )
12253 .await;
12254 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
12255
12256 register_candidate(&swap, None);
12257 cutover(&swap);
12258
12259 let ack = handler
12261 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
12262 .await;
12263 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
12264 if module_loop_error.is_some() {
12265 handler.cleanup_connection(INCUMBENT).unwrap();
12268 }
12269
12270 assert!(
12272 cotenant_rx.try_recv().is_err(),
12273 "the co-tenant route on the incumbent was torn down by one late ack: \
12274 {module_loop_error:?}"
12275 );
12276 assert!(matches!(
12277 swap.forwarding
12278 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
12279 .unwrap(),
12280 DataRoute::Client(DataRouteState::Bound(_))
12281 ));
12282 assert_eq!(module_loop_error, None);
12283 assert!(swap
12284 .registry
12285 .get_module_by_connection(INCUMBENT)
12286 .unwrap()
12287 .is_some());
12288
12289 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
12291 .await
12292 .expect("the incumbent is told to drop the abandoned binding")
12293 .unwrap()
12294 .frame;
12295 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12296 assert_eq!(goodbye.header.channel, abandoned_channel);
12297 assert_eq!(goodbye.header.epoch, abandoned_epoch);
12298 assert!(swap.incumbent_rx.try_recv().is_err());
12299
12300 let response = caller_task.await.unwrap();
12302 assert_eq!(response.len(), 1);
12303 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
12304 assert!(caller_rx.try_recv().is_err());
12305
12306 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
12310 let counters = handler.counters().snapshot();
12311 assert_eq!(
12312 keyed_total(&counters, "route_open_accepted_by_principal"),
12313 1
12314 );
12315 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
12316 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
12317 }
12318
12319 #[tokio::test]
12323 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
12324 let mut swap = swap_with_incumbent().await;
12325 register_candidate(&swap, None);
12326 let incumbent = cutover(&swap);
12327 swap.forwarding
12328 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
12329 .unwrap()
12330 .expect("the incumbent is still registered");
12331
12332 let client = ConnectionId::new(33);
12333 let (client_ctx, mut client_rx) = route_ctx(client);
12334 let route_handler = swap.handler.clone();
12335 let open_ctx = RouteCtx {
12336 connection_id: client,
12337 egress: client_ctx.egress.clone(),
12338 };
12339 let mut route_task = tokio::spawn(async move {
12340 route_handler
12341 .handle_control_frame(
12342 &open_ctx,
12343 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
12344 )
12345 .await
12346 .unwrap()
12347 });
12348 let bind = tokio::select! {
12349 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
12350 response = &mut route_task => {
12351 let response = response.unwrap();
12352 panic!(
12353 "post-cutover route.open was refused instead of relayed to the candidate: {}",
12354 parse_error(&response[0])["code"]
12355 );
12356 }
12357 };
12358 swap.handler
12359 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
12360 .await
12361 .unwrap();
12362 assert!(route_task.await.unwrap().is_empty());
12363 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
12364 match swap
12365 .forwarding
12366 .lookup_data_route(client, channel, epoch)
12367 .unwrap()
12368 {
12369 DataRoute::Client(DataRouteState::Bound(route)) => {
12370 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
12371 }
12372 other => panic!("expected a bound route on the candidate, got {other:?}"),
12373 }
12374 assert!(swap.incumbent_rx.try_recv().is_err());
12375 }
12376
12377 #[tokio::test]
12382 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
12383 let swap = swap_with_incumbent().await;
12384 register_candidate(&swap, Some(false));
12385 let update = Frame::build(
12386 FrameType::Request,
12387 control_flags(),
12388 0,
12389 0,
12390 55,
12391 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
12392 provides: manifest("aft", PROTOCOL_VERSION).provides,
12393 capabilities: None,
12394 ready: Some(true),
12395 })
12396 .unwrap(),
12397 )
12398 .unwrap();
12399
12400 let replies = swap
12401 .handler
12402 .handle_control_frame(&swap.candidate_ctx, update)
12403 .await
12404 .unwrap();
12405
12406 assert_eq!(replies.len(), 1);
12407 assert_eq!(
12408 replies[0].header.ty,
12409 FrameType::Response,
12410 "candidate catalog.update was refused: {:?}",
12411 serde_json::from_slice::<Value>(&replies[0].body).ok()
12412 );
12413 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
12414 assert_eq!(
12415 swap.registry
12416 .get_module("aft")
12417 .unwrap()
12418 .unwrap()
12419 .connection_id,
12420 INCUMBENT
12421 );
12422 }
12423 }
12424
12425 mod swap_admission {
12429 use super::*;
12430
12431 const INCUMBENT_NONCE: &str = "incumbent-nonce";
12432 const CANDIDATE_NONCE: &str = "candidate-nonce";
12433
12434 fn handler_with_incumbent(
12435 module_id: &str,
12436 reserved: bool,
12437 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
12438 let registry = Arc::new(Registry::default());
12439 let supervisor = SupervisorHandle::new();
12440 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
12441 if reserved {
12442 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
12443 }
12444 let handler =
12445 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
12446 let incumbent = handler
12447 .handle_control(
12448 ConnectionId::new(1),
12449 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
12450 )
12451 .unwrap();
12452 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
12453 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
12454 (registry, supervisor, handler)
12455 }
12456
12457 #[test]
12463 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
12464 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
12465
12466 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
12467 let replies = handler
12468 .handle_control(
12469 ConnectionId::new(connection),
12470 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
12471 )
12472 .unwrap();
12473 assert_eq!(replies[0].header.ty, FrameType::Error);
12474 assert_eq!(
12475 parse_error(&replies[0])["code"],
12476 "swap_token_invalid",
12477 "nonce {nonce:?}"
12478 );
12479 }
12480 assert!(registry.get_candidate("aft").unwrap().is_none());
12481 assert_eq!(
12482 registry.get_module("aft").unwrap().unwrap().connection_id,
12483 ConnectionId::new(1)
12484 );
12485
12486 let admitted = handler
12489 .handle_control(
12490 ConnectionId::new(4),
12491 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
12492 )
12493 .unwrap();
12494 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
12495 assert_eq!(
12496 registry
12497 .get_candidate("aft")
12498 .unwrap()
12499 .unwrap()
12500 .connection_id,
12501 ConnectionId::new(4)
12502 );
12503 assert_eq!(
12504 registry.get_module("aft").unwrap().unwrap().connection_id,
12505 ConnectionId::new(1),
12506 "the candidate must not take the active slot"
12507 );
12508 let replayed = handler
12509 .handle_control(
12510 ConnectionId::new(5),
12511 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
12512 )
12513 .unwrap();
12514 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
12515
12516 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
12520 let squatter = handler
12521 .handle_control(
12522 ConnectionId::new(6),
12523 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
12524 )
12525 .unwrap();
12526 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
12527 assert!(
12528 registry.get_module("aft").unwrap().is_none(),
12529 "a squatter took the active slot of an id being swapped"
12530 );
12531 }
12532
12533 #[test]
12538 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
12539 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
12540
12541 let replies = handler
12542 .handle_control(
12543 ConnectionId::new(2),
12544 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12545 )
12546 .unwrap();
12547
12548 assert_eq!(
12549 replies[0].header.ty,
12550 FrameType::HelloAck,
12551 "reserved candidate refused: {:?}",
12552 serde_json::from_slice::<Value>(&replies[0].body).ok()
12553 );
12554 assert_eq!(
12555 registry
12556 .get_candidate("vault")
12557 .unwrap()
12558 .unwrap()
12559 .connection_id,
12560 ConnectionId::new(2)
12561 );
12562 }
12563
12564 #[test]
12567 fn without_an_open_swap_the_ordinary_gates_decide() {
12568 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
12569 supervisor.close_swap("vault");
12570
12571 let candidate = handler
12572 .handle_control(
12573 ConnectionId::new(2),
12574 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12575 )
12576 .unwrap();
12577 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
12578 let duplicate = handler
12579 .handle_control(
12580 ConnectionId::new(3),
12581 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
12582 )
12583 .unwrap();
12584 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12585 assert!(registry.get_candidate("vault").unwrap().is_none());
12586 }
12587 }
12588
12589 mod scopes {
12593 use subc_protocol::scope::{
12594 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
12595 ScopeStatus,
12596 };
12597
12598 use super::*;
12599
12600 const OWNER: &str = "prefrontal-core";
12601
12602 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
12603 ScopeRecord {
12604 scope_ref: scope_ref.to_string(),
12605 scope_epoch,
12606 kind: ScopeKind::Head,
12607 parent: None,
12608 child_owners: Vec::new(),
12609 carriers: Vec::new(),
12610 attributes: Default::default(),
12611 }
12612 }
12613
12614 async fn call(
12615 handler: &ControlHandler,
12616 ctx: &RouteCtx,
12617 request: &ModuleControlRequestFromModule,
12618 ) -> Frame {
12619 let body = serde_json::to_vec(request).unwrap();
12620 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
12621 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
12622 assert_eq!(replies.len(), 1, "{replies:?}");
12623 replies.pop().unwrap()
12624 }
12625
12626 async fn sync(
12627 handler: &ControlHandler,
12628 ctx: &RouteCtx,
12629 generation: u64,
12630 scopes: Vec<ScopeRecord>,
12631 ) -> Result<ModuleControlResponseToModule, String> {
12632 let reply = call(
12633 handler,
12634 ctx,
12635 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
12636 )
12637 .await;
12638 match reply.header.ty {
12639 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
12640 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
12641 }
12642 }
12643
12644 async fn describe(
12645 handler: &ControlHandler,
12646 ctx: &RouteCtx,
12647 owner: &str,
12648 scope_ref: &str,
12649 ) -> ModuleControlResponseToModule {
12650 let reply = call(
12651 handler,
12652 ctx,
12653 &ModuleControlRequestFromModule::ScopeDescribe {
12654 owner: Principal::Reserved {
12655 module_id: owner.to_string(),
12656 },
12657 scope_ref: scope_ref.to_string(),
12658 },
12659 )
12660 .await;
12661 assert_eq!(
12662 reply.header.ty,
12663 FrameType::Response,
12664 "{:?}",
12665 parse_error(&reply)
12666 );
12667 serde_json::from_slice(&reply.body).unwrap()
12668 }
12669
12670 async fn module(
12672 handler: &ControlHandler,
12673 connection: u64,
12674 module_id: &str,
12675 nonce: Option<&str>,
12676 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12677 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
12678 hello_via_sink(
12679 handler,
12680 &ctx,
12681 &mut rx,
12682 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
12683 )
12684 .await;
12685 (ctx, rx)
12686 }
12687
12688 #[tokio::test]
12691 async fn a_client_connection_cannot_sync_or_describe() {
12692 let handler = ControlHandler::new(Arc::new(Registry::default()));
12693 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
12694 for request in [
12695 ModuleControlRequestFromModule::ScopeSync {
12696 generation: 1,
12697 scopes: vec![head("s", 1)],
12698 },
12699 ModuleControlRequestFromModule::ScopeDescribe {
12700 owner: Principal::Direct,
12701 scope_ref: "s".to_string(),
12702 },
12703 ] {
12704 let reply = call(&handler, &ctx, &request).await;
12705 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
12706 }
12707 assert!(
12708 !handler
12709 .scopes
12710 .read()
12711 .unwrap()
12712 .describe(
12713 &Principal::Reserved {
12714 module_id: OWNER.to_string()
12715 },
12716 "s"
12717 )
12718 .owner_synced
12719 );
12720 }
12721
12722 #[tokio::test]
12725 async fn a_module_without_a_supervised_launch_cannot_sync() {
12726 let handler = ControlHandler::new(Arc::new(Registry::default()));
12727 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
12728 assert_eq!(
12729 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
12730 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12731 );
12732 }
12733
12734 #[tokio::test]
12735 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
12736 let supervisor = SupervisorHandle::new();
12737 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12738 let handler = ControlHandler::new(Arc::new(Registry::default()))
12739 .with_supervisor(supervisor.clone());
12740 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12741 sync(&handler, &incumbent, 1, vec![head("s", 1)])
12742 .await
12743 .expect("the current launch syncs");
12744
12745 supervisor.open_swap(OWNER, "n2".to_string());
12748 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
12749 assert_eq!(
12750 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12751 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12752 );
12753 sync(&handler, &incumbent, 2, vec![head("s", 1)])
12754 .await
12755 .expect("the serving owner syncs during the swap");
12756
12757 supervisor.close_swap(OWNER);
12760 assert_eq!(
12761 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12762 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12763 );
12764 sync(&handler, &incumbent, 3, vec![head("s", 1)])
12765 .await
12766 .expect("the serving owner syncs after the rollback");
12767 handler.cleanup_connection(candidate.connection_id).unwrap();
12768
12769 supervisor.open_swap(OWNER, "n3".to_string());
12774 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
12775 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
12776 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
12777 .await
12778 .expect("the promoted launch takes authority");
12779 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
12780 panic!("unexpected reply {reply:?}");
12781 };
12782 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
12783 assert_eq!(
12784 sync(&handler, &incumbent, 4, Vec::new()).await,
12785 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12786 );
12787 }
12788
12789 #[tokio::test]
12792 async fn closing_the_authority_connection_frees_sync_authority() {
12793 let supervisor = SupervisorHandle::new();
12794 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12795 let handler = ControlHandler::new(Arc::new(Registry::default()))
12796 .with_supervisor(supervisor.clone());
12797 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12798 sync(&handler, &first, 10, vec![head("s", 1)])
12799 .await
12800 .unwrap();
12801 handler.cleanup_connection(first.connection_id).unwrap();
12802
12803 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
12804 sync(&handler, &second, 1, vec![head("s", 1)])
12805 .await
12806 .expect("the next connection takes the released authority");
12807 }
12808
12809 #[tokio::test]
12810 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
12811 let supervisor = SupervisorHandle::new();
12812 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12813 let handler =
12814 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
12815 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
12816 sync(&handler, &first, 10, vec![head("s", 1)])
12817 .await
12818 .unwrap();
12819 handler
12820 .handle_control_frame(
12821 &first,
12822 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
12823 )
12824 .await
12825 .unwrap();
12826 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
12827 sync(&handler, &second, 1, vec![head("s", 1)])
12828 .await
12829 .expect("GOODBYE releases authority even if the old socket remains open");
12830 }
12831
12832 #[tokio::test]
12833 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
12834 let registry = Arc::new(Registry::default());
12835 let supervisor_handle = SupervisorHandle::new();
12836 let supervisor =
12837 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
12838 .with_handle(supervisor_handle.clone())
12839 .with_daemon_incarnation("incarnation-7".to_string());
12840 supervisor
12843 .supervise_configured(
12844 ModuleSpec {
12845 module_id: OWNER.to_string(),
12846 program: PathBuf::from("/nonexistent/prefrontal-core"),
12847 args: Vec::new(),
12848 env: Vec::new(),
12849 reserved: false,
12850 reserved_prefixes: Vec::new(),
12851 protocol: ModuleProtocol::Subc,
12852 overlap: Default::default(),
12853 },
12854 false,
12855 )
12856 .unwrap();
12857 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
12858 let handler =
12859 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12860 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
12861
12862 let ModuleControlResponseToModule::ScopeDescribe {
12864 status,
12865 daemon_incarnation,
12866 owner_synced,
12867 owner_configured,
12868 scope,
12869 ..
12870 } = describe(&handler, &reader, OWNER, "s").await
12871 else {
12872 panic!("not a describe reply");
12873 };
12874 assert_eq!(status, ScopeStatus::NotLive);
12875 assert_eq!(daemon_incarnation, "incarnation-7");
12876 assert!(!owner_synced);
12877 assert!(owner_configured);
12878 assert!(scope.is_none());
12879
12880 let ModuleControlResponseToModule::ScopeDescribe {
12883 status,
12884 owner_configured,
12885 ..
12886 } = describe(&handler, &reader, "ghost", "s").await
12887 else {
12888 panic!("not a describe reply");
12889 };
12890 assert_eq!(status, ScopeStatus::NotLive);
12891 assert!(!owner_configured);
12892
12893 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
12895 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
12896 let ModuleControlResponseToModule::ScopeDescribe {
12897 status,
12898 scope_epoch,
12899 owner_synced,
12900 scope,
12901 ..
12902 } = describe(&handler, &reader, OWNER, "s").await
12903 else {
12904 panic!("not a describe reply");
12905 };
12906 assert_eq!(status, ScopeStatus::Live);
12907 assert_eq!(scope_epoch, Some(4));
12908 assert!(owner_synced);
12909 let stamp = scope.expect("a live scope carries its stamp");
12910 assert!(
12911 stamp.owner_authorized,
12912 "prefrontal-core is the default authority"
12913 );
12914 assert_eq!(stamp.kind, ScopeKind::Head);
12915 }
12916
12917 #[tokio::test]
12918 async fn scope_authority_owners_decides_owner_authorized() {
12919 let supervisor = SupervisorHandle::new();
12920 supervisor.set_spawn_nonce("broca", "b1".to_string());
12921 let handler = ControlHandler::new(Arc::new(Registry::default()))
12922 .with_supervisor(supervisor)
12923 .with_scope_authority_owners(vec!["broca".to_string()]);
12924 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
12925 let mut gated = head("s", 1);
12926 gated.attributes.agent_id = Some("agent".to_string());
12927 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
12928 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
12929 describe(&handler, &broca, "broca", "s").await
12930 else {
12931 panic!("not a describe reply");
12932 };
12933 assert!(scope.unwrap().owner_authorized);
12934 }
12935
12936 #[tokio::test]
12940 async fn scope_ops_and_the_scopes_capability_are_advertised() {
12941 let handler = ControlHandler::new(Arc::new(Registry::default()));
12942 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
12943 let ack = hello_via_sink(
12944 &handler,
12945 &ctx,
12946 &mut rx,
12947 hello_frame("m", PROTOCOL_VERSION, 1),
12948 )
12949 .await;
12950 let ack = parse_ack(&ack);
12951 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
12952 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
12953 }
12954 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
12955
12956 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
12957 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12958 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12959 let reply = handler
12960 .handle_control_frame(&client, frame)
12961 .await
12962 .unwrap()
12963 .pop()
12964 .unwrap();
12965 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12966 serde_json::from_slice(&reply.body).unwrap()
12967 else {
12968 panic!("not a server.describe reply");
12969 };
12970 assert!(
12971 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
12972 "{capabilities:?}"
12973 );
12974 }
12975
12976 const PLEXUS: &str = "plexus";
12979 const OTHER: &str = "other";
12980 const AFT: &str = "aft";
12981 const BROCA: &str = "broca";
12982 const MAGIC: &str = "magic-context";
12983
12984 fn nonce(module_id: &str) -> String {
12985 format!("nonce-{module_id}")
12986 }
12987
12988 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12989 let (tx, rx) = mpsc::channel(64);
12990 (
12991 RouteCtx {
12992 connection_id: ConnectionId::new(connection),
12993 egress: FrameSink::new(tx),
12994 },
12995 rx,
12996 )
12997 }
12998
12999 struct Rig {
13003 handler: ControlHandler,
13004 forwarding: Arc<ForwardingTable>,
13005 owner: RouteCtx,
13006 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13007 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
13008 generation: u64,
13009 next_connection: u64,
13010 _supervisor: Supervisor,
13011 }
13012
13013 async fn rig() -> Rig {
13014 rig_with_flow_support(true).await
13015 }
13016
13017 async fn rig_with_flow_support(flow_support: bool) -> Rig {
13018 let registry = Arc::new(Registry::default());
13019 let forwarding = Arc::new(ForwardingTable::default());
13020 let supervisor_handle = SupervisorHandle::new();
13021 let supervisor =
13022 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
13023 .with_handle(supervisor_handle.clone());
13024 supervisor
13025 .supervise_configured(
13026 ModuleSpec {
13027 module_id: OWNER.to_string(),
13028 program: PathBuf::from("/nonexistent/prefrontal-core"),
13029 args: Vec::new(),
13030 env: Vec::new(),
13031 reserved: false,
13032 reserved_prefixes: Vec::new(),
13033 protocol: ModuleProtocol::Subc,
13034 overlap: Default::default(),
13035 },
13036 false,
13037 )
13038 .unwrap();
13039 for module_id in [OWNER, AFT, BROCA, MAGIC] {
13040 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
13041 }
13042 let handler =
13043 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13044 .with_supervisor(supervisor_handle);
13045 let (owner, mut owner_rx) = wide_ctx(1);
13046 hello_via_sink(
13047 &handler,
13048 &owner,
13049 &mut owner_rx,
13050 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
13051 )
13052 .await;
13053 let mut modules = BTreeMap::new();
13054 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
13055 let (ctx, mut rx) = wide_ctx(connection);
13056 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
13057 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
13058 body["manifest"]["provenance"] =
13062 serde_json::json!({"wire_crate_version": "0.29.0"});
13063 if flow_support {
13064 body["manifest"]["capabilities"] =
13065 serde_json::json!({"provides": ["flow-scopes/v1"]});
13066 }
13067 let hello = Frame::build(
13068 FrameType::Hello,
13069 control_flags(),
13070 0,
13071 0,
13072 connection,
13073 serde_json::to_vec(&body).unwrap(),
13074 )
13075 .unwrap();
13076 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
13077 modules.insert(module_id.to_string(), (ctx, rx));
13078 }
13079 Rig {
13080 handler,
13081 forwarding,
13082 owner,
13083 _owner_rx: owner_rx,
13084 modules,
13085 generation: 0,
13086 next_connection: 100,
13087 _supervisor: supervisor,
13088 }
13089 }
13090
13091 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
13092 ScopeCarrier {
13093 principal: Principal::Reserved {
13094 module_id: module_id.to_string(),
13095 },
13096 targets: targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()),
13097 }
13098 }
13099
13100 fn session(scope_epoch: u64) -> ScopeRecord {
13103 let mut record = head("s", scope_epoch);
13104 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
13105 record.attributes.agent_id = Some("agent-1".to_string());
13106 record.attributes.delegates = true;
13107 record
13108 }
13109
13110 impl Rig {
13111 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
13112 self.generation += 1;
13113 sync(&self.handler, &self.owner, self.generation, scopes)
13114 .await
13115 .expect("the owner's sync is accepted");
13116 }
13117
13118 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
13119 ScopeSelector {
13120 owner: Principal::Reserved {
13121 module_id: OWNER.to_string(),
13122 },
13123 scope_ref: scope_ref.to_string(),
13124 scope_epoch,
13125 }
13126 }
13127
13128 fn open_frame(
13129 &mut self,
13130 opener: Option<&str>,
13131 target: &str,
13132 scope: Option<ScopeSelector>,
13133 ) -> (
13134 RouteCtx,
13135 mpsc::Receiver<crate::router::OutboundFrame>,
13136 Frame,
13137 ) {
13138 self.next_connection += 1;
13139 let (ctx, rx) = wide_ctx(self.next_connection);
13140 let root = unique_project_root("scoped-open");
13141 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
13142 target: RouteTarget::ToolProvider {
13143 module_id: target.to_string(),
13144 },
13145 identity: BindIdentity::new(
13146 root.path().to_path_buf(),
13147 "unit".to_string(),
13148 "session".to_string(),
13149 ),
13150 consumer_identity: opener.map(|module_id| ConsumerIdentity {
13151 module_id: module_id.to_string(),
13152 launch_nonce: nonce(module_id),
13153 }),
13154 consumer_capabilities: None,
13155 role_versions: None,
13156 admission_facts: None,
13157 scope,
13158 })
13159 .unwrap();
13160 let frame = Frame::build(
13161 FrameType::Request,
13162 control_flags(),
13163 0,
13164 0,
13165 self.next_connection,
13166 body,
13167 )
13168 .unwrap();
13169 (ctx, rx, frame)
13170 }
13171
13172 async fn refused(
13174 &mut self,
13175 opener: Option<&str>,
13176 target: &str,
13177 scope: Option<ScopeSelector>,
13178 ) -> String {
13179 self.refusal_body(opener, target, scope).await["code"]
13180 .as_str()
13181 .unwrap()
13182 .to_string()
13183 }
13184
13185 async fn refusal_body(
13186 &mut self,
13187 opener: Option<&str>,
13188 target: &str,
13189 scope: Option<ScopeSelector>,
13190 ) -> Value {
13191 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
13192 let replies = tokio::time::timeout(
13193 Duration::from_secs(2),
13194 self.handler.handle_control_frame(&ctx, frame),
13195 )
13196 .await
13197 .expect("the open must be refused before waiting for a bind ack")
13198 .unwrap();
13199 assert_eq!(replies.len(), 1, "{replies:?}");
13200 assert_eq!(replies[0].header.ty, FrameType::Error);
13201 let (_, module_rx) = self.modules.get_mut(target).unwrap();
13202 assert!(
13203 module_rx.try_recv().is_err(),
13204 "a refused open relays nothing"
13205 );
13206 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
13207 parse_error(&replies[0])
13208 }
13209
13210 async fn relayed(
13212 &mut self,
13213 opener: Option<&str>,
13214 target: &str,
13215 scope: Option<ScopeSelector>,
13216 ) -> Relayed {
13217 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
13218 let handler = self.handler.clone();
13219 let task_ctx = ctx.clone();
13220 let task = tokio::spawn(async move {
13221 handler
13222 .handle_control_frame(&task_ctx, frame)
13223 .await
13224 .unwrap()
13225 });
13226 let (_, module_rx) = self.modules.get_mut(target).unwrap();
13227 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
13228 .await
13229 .expect("the target receives the relayed route.bind")
13230 .unwrap()
13231 .frame;
13232 Relayed {
13233 target: target.to_string(),
13234 client: ctx,
13235 client_rx: rx,
13236 task,
13237 bind,
13238 }
13239 }
13240
13241 async fn ack(&self, relayed: &Relayed) {
13242 let (module, _) = &self.modules[&relayed.target];
13243 self.handler
13244 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
13245 .await
13246 .unwrap();
13247 }
13248
13249 async fn bound(
13251 &mut self,
13252 opener: Option<&str>,
13253 target: &str,
13254 scope: Option<ScopeSelector>,
13255 ) -> Bound {
13256 let relayed = self.relayed(opener, target, scope).await;
13257 self.ack(&relayed).await;
13258 let Relayed {
13259 target,
13260 client,
13261 mut client_rx,
13262 task,
13263 bind,
13264 } = relayed;
13265 assert!(
13266 task.await.unwrap().is_empty(),
13267 "the open is answered by commit"
13268 );
13269 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
13270 Bound {
13271 target,
13272 client,
13273 client_rx,
13274 channel,
13275 epoch,
13276 bind,
13277 }
13278 }
13279
13280 fn live(&self, route: &Bound) -> bool {
13281 matches!(
13282 self.forwarding
13283 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
13284 .unwrap(),
13285 DataRoute::Client(DataRouteState::Bound(_))
13286 )
13287 }
13288 }
13289
13290 struct Relayed {
13291 target: String,
13292 client: RouteCtx,
13293 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13294 task: tokio::task::JoinHandle<Vec<Frame>>,
13295 bind: Frame,
13296 }
13297
13298 struct Bound {
13299 target: String,
13300 client: RouteCtx,
13301 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13302 channel: u16,
13303 epoch: u32,
13304 bind: Frame,
13305 }
13306
13307 impl Bound {
13308 fn closed_reason(&mut self) -> RouteCloseReason {
13311 let mut reason = None;
13312 let mut goodbye = false;
13313 while let Ok(outbound) = self.client_rx.try_recv() {
13314 let frame = outbound.frame;
13315 match frame.header.ty {
13316 FrameType::Goodbye => {
13317 assert_eq!(
13318 (frame.header.channel, frame.header.epoch),
13319 (self.channel, self.epoch)
13320 );
13321 goodbye = true;
13322 }
13323 FrameType::Push => {
13324 let ClientControlPush::RouteClosed {
13325 reason: r,
13326 module_id,
13327 ..
13328 } = serde_json::from_slice(&frame.body).unwrap()
13329 else {
13330 panic!("unexpected push");
13331 };
13332 assert_eq!(module_id, self.target);
13333 reason = Some(r);
13334 }
13335 other => panic!("unexpected frame {other:?}"),
13336 }
13337 }
13338 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
13339 reason.expect("the client is told why the route closed")
13340 }
13341
13342 fn untouched(&mut self) -> bool {
13343 self.client_rx.try_recv().is_err()
13344 }
13345
13346 fn stamp(&self) -> Option<ScopeStamp> {
13347 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
13348 ModuleControlRequest::RouteBind { scope, .. } => scope,
13349 other => panic!("expected a route.bind, got {other:?}"),
13350 }
13351 }
13352 }
13353
13354 #[tokio::test]
13355 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
13356 ) {
13357 let mut rig = rig().await;
13358 let mut record = session(1);
13359 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
13360 record.child_owners = vec![Principal::Reserved {
13361 module_id: MAGIC.to_string(),
13362 }];
13363 rig.sync(vec![record]).await;
13364 let scope = || Some(rig_selector("s", Some(1)));
13365
13366 rig.bound(Some(OWNER), PLEXUS, scope()).await;
13369 rig.bound(Some(AFT), OTHER, scope()).await;
13370 rig.bound(Some(BROCA), PLEXUS, scope()).await;
13371
13372 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
13376 assert_eq!(
13377 rig.refused(opener, target, scope()).await,
13378 error_codes::SCOPE_NOT_CARRIER,
13379 "{opener:?} -> {target}"
13380 );
13381 }
13382 }
13383
13384 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
13385 ScopeSelector {
13386 owner: Principal::Reserved {
13387 module_id: OWNER.to_string(),
13388 },
13389 scope_ref: scope_ref.to_string(),
13390 scope_epoch,
13391 }
13392 }
13393
13394 #[tokio::test]
13395 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
13396 let mut rig = rig().await;
13397 rig.sync(vec![session(1)]).await;
13398 for opener in [OWNER, AFT] {
13399 assert_eq!(
13400 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
13401 .await,
13402 error_codes::SCOPE_EPOCH_REQUIRED,
13403 "{opener}"
13404 );
13405 }
13406 }
13407
13408 #[tokio::test]
13409 async fn admission_separates_not_synced_not_live_and_ended() {
13410 let mut rig = rig().await;
13411 let code = rig
13413 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13414 .await;
13415 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
13416 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
13417
13418 let ghost = ScopeSelector {
13420 owner: Principal::Reserved {
13421 module_id: "ghost".to_string(),
13422 },
13423 scope_ref: "s".to_string(),
13424 scope_epoch: Some(1),
13425 };
13426 assert_eq!(
13427 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
13428 error_codes::SCOPE_NOT_LIVE
13429 );
13430
13431 rig.sync(vec![session(2)]).await;
13432 assert_eq!(
13433 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
13434 .await,
13435 error_codes::SCOPE_NOT_LIVE
13436 );
13437 for epoch in [1, 3] {
13438 assert_eq!(
13439 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
13440 .await,
13441 error_codes::SCOPE_ENDED,
13442 "epoch {epoch}"
13443 );
13444 }
13445 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
13447 .await;
13448 }
13449
13450 #[tokio::test]
13451 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
13452 let mut rig = rig().await;
13453 rig.sync(vec![session(1)]).await;
13454 let route = rig
13455 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13456 .await;
13457 let stamp = route.stamp().expect("a scoped bind carries the stamp");
13458 assert_eq!(stamp.scope_ref, "s");
13459 assert_eq!(stamp.scope_epoch, 1);
13460 assert_eq!(stamp.kind, ScopeKind::Head);
13461 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
13462 assert!(stamp.attributes.delegates);
13463 assert!(stamp.owner_authorized);
13464 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13465 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
13466
13467 let (broca, mut broca_rx) = wide_ctx(50);
13470 hello_via_sink(
13471 &rig.handler,
13472 &broca,
13473 &mut broca_rx,
13474 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
13475 )
13476 .await;
13477 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
13478 .await
13479 .unwrap();
13480 let own = ScopeSelector {
13481 owner: Principal::Reserved {
13482 module_id: BROCA.to_string(),
13483 },
13484 scope_ref: "b".to_string(),
13485 scope_epoch: Some(1),
13486 };
13487 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
13488 assert!(!route.stamp().unwrap().owner_authorized);
13489 }
13490
13491 #[tokio::test]
13492 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
13493 let mut rig = rig().await;
13494 let mut record = head("s", 1);
13495 record.carriers = vec![carrier(AFT, None)];
13496 let flow_id = "Flow:run-7/step_2!~";
13497 record.attributes.flow_id = Some(flow_id.to_string());
13498 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
13499 .await
13500 .unwrap();
13501 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
13502 panic!("not a sync reply");
13503 };
13504 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
13505 let route = rig
13506 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13507 .await;
13508 assert!(rig.live(&route), "the stamped bind committed");
13509 let stamp = route.stamp().expect("a flow scope carries a stamp");
13510 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
13511 assert_eq!(stamp.attributes.agent_id, None);
13512 assert!(!stamp.attributes.delegates);
13513 assert!(stamp.owner_authorized);
13514 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13515 assert_eq!(unscoped.stamp(), None);
13516 }
13517
13518 #[tokio::test]
13519 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
13520 let mut rig = rig_with_flow_support(false).await;
13521 let mut record = session(1);
13522 record.attributes.flow_id = Some("flow:7".to_string());
13523 rig.sync(vec![record]).await;
13524 for opener in [OWNER, AFT] {
13525 let body = rig
13526 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13527 .await;
13528 assert_eq!(body["code"], "target_flow_unsupported");
13529 let message = body["message"].as_str().unwrap();
13530 for required in [PLEXUS, "flow-scopes/v1"] {
13531 assert!(message.contains(required), "{message}");
13532 }
13533 }
13534 }
13535
13536 #[tokio::test]
13537 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
13538 let mut rig = rig_with_flow_support(true).await;
13539 let mut record = session(1);
13540 record.attributes.flow_id = Some("flow:7".to_string());
13541 rig.sync(vec![record]).await;
13542 for opener in [OWNER, AFT] {
13543 let route = rig
13544 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13545 .await;
13546 assert!(rig.live(&route));
13547 assert_eq!(
13548 route.stamp().unwrap().attributes.flow_id.as_deref(),
13549 Some("flow:7")
13550 );
13551 }
13552 }
13553
13554 #[tokio::test]
13555 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
13556 use std::future::Future;
13557
13558 let mut rig = rig_with_flow_support(true).await;
13559 let mut record = session(1);
13560 record.attributes.flow_id = Some("flow:7".to_string());
13561 rig.sync(vec![record]).await;
13562 let (client, mut client_rx, frame) =
13563 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
13564 for _ in 0..64 {
13567 client.egress.try_send(route_bind_ack(1)).unwrap();
13568 }
13569 let handler = rig.handler.clone();
13570 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
13571 std::future::poll_fn(|cx| {
13572 assert!(open.as_mut().poll(cx).is_pending());
13573 std::task::Poll::Ready(())
13574 })
13575 .await;
13576 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13577
13578 let old_connection = rig.modules[PLEXUS].0.connection_id;
13579 rig.handler.cleanup_connection(old_connection).unwrap();
13580 let (replacement, mut replacement_rx) = wide_ctx(200);
13581 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
13582 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
13583 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
13584 let hello = Frame::build(
13585 FrameType::Hello,
13586 control_flags(),
13587 0,
13588 0,
13589 200,
13590 serde_json::to_vec(&body).unwrap(),
13591 )
13592 .unwrap();
13593 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
13594
13595 client_rx.try_recv().unwrap();
13596 let replies = tokio::time::timeout(Duration::from_secs(2), open)
13597 .await
13598 .expect("the replacement is refused without waiting for a bind ack")
13599 .unwrap();
13600 assert_eq!(replies.len(), 1);
13601 let body = parse_error(&replies[0]);
13602 assert_eq!(body["code"], "target_flow_unsupported");
13603 for required in [PLEXUS, "flow-scopes/v1"] {
13604 assert!(body["message"].as_str().unwrap().contains(required));
13605 }
13606 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
13607 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
13608 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13609 assert!(rig
13610 .handler
13611 .registry
13612 .get_module_by_connection(replacement.connection_id)
13613 .unwrap()
13614 .is_some());
13615 }
13616
13617 #[tokio::test]
13618 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
13619 {
13620 let mut rig = rig_with_flow_support(false).await;
13621 rig.sync(vec![session(1)]).await;
13622 let route = rig
13623 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13624 .await;
13625 assert!(rig.live(&route));
13626 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
13627 let mut record = session(1);
13628 record.attributes.flow_id = Some("flow:7".to_string());
13629 rig.sync(vec![record]).await;
13630 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
13631 assert!(rig.live(&unscoped));
13632 assert_eq!(unscoped.stamp(), None);
13633 }
13634
13635 #[tokio::test]
13636 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
13637 let mut rig = rig().await;
13638 let mut record = session(1);
13639 record.attributes.flow_id = Some("flow:7".to_string());
13640 rig.sync(vec![record.clone()]).await;
13641 let before = rig.forwarding.published_scope_tag(OWNER, "s");
13642 let mut owner_route = rig
13643 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
13644 .await;
13645 let mut carrier_route = rig
13646 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
13647 .await;
13648 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13649 rig.sync(vec![record.clone()]).await;
13650 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
13651 assert!(rig.live(&owner_route) && owner_route.untouched());
13652 assert!(rig.live(&carrier_route) && carrier_route.untouched());
13653
13654 record.attributes.flow_id = Some("flow:8".to_string());
13655 rig.sync(vec![record]).await;
13656 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
13657 let before = before.unwrap();
13658 assert_eq!(after.scope_epoch, before.scope_epoch);
13659 assert!(after.version > before.version);
13660 for route in [&mut owner_route, &mut carrier_route] {
13661 assert!(!rig.live(route));
13662 assert_eq!(
13663 route.closed_reason(),
13664 RouteCloseReason::ScopeDelegationChanged
13665 );
13666 }
13667 assert!(rig.live(&unscoped) && unscoped.untouched());
13668 for target in [PLEXUS, OTHER] {
13671 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
13672 let goodbye = module_rx
13673 .try_recv()
13674 .expect("the provider sees the drain")
13675 .frame;
13676 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13677 assert!(module_rx.try_recv().is_err());
13678 }
13679 let rebound = rig
13680 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13681 .await;
13682 assert_eq!(
13683 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
13684 Some("flow:8")
13685 );
13686 }
13687
13688 #[tokio::test]
13693 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
13694 let mut rig = rig().await;
13695 rig.sync(vec![session(1)]).await;
13696 let mut cotenant = rig
13697 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
13698 .await;
13699
13700 let mut changed = session(1);
13701 changed.child_owners.push(Principal::Reserved {
13702 module_id: MAGIC.to_string(),
13703 });
13704 let mut ended = None;
13705 for (code, next) in [
13706 (error_codes::SCOPE_CHANGED, vec![changed]),
13707 (error_codes::SCOPE_ENDED, Vec::new()),
13708 ] {
13709 let relayed = rig
13710 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13711 .await;
13712 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
13713 ended = Some(next.is_empty());
13714 rig.sync(next).await;
13715 rig.ack(&relayed).await;
13716 let replies = relayed.task.await.unwrap();
13717 assert_eq!(replies.len(), 1, "{replies:?}");
13718 assert_eq!(parse_error(&replies[0])["code"], code);
13719 assert_eq!(
13720 subc_protocol::error_codes::is_retryable_route_open(code),
13721 code == error_codes::SCOPE_CHANGED
13722 );
13723 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13724 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13726 let mut goodbyes = Vec::new();
13729 while let Ok(outbound) = plexus_rx.try_recv() {
13730 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
13731 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
13732 }
13733 assert!(
13734 goodbyes.contains(&(bind_channel, bind_epoch)),
13735 "{goodbyes:?}"
13736 );
13737 assert!(rig
13738 .handler
13739 .registry
13740 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
13741 .unwrap()
13742 .is_some());
13743 }
13744 assert_eq!(ended, Some(true));
13745 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
13748 }
13749
13750 #[tokio::test]
13753 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
13754 struct Case {
13755 name: &'static str,
13756 change: fn(&mut ScopeRecord),
13757 closed: [Option<RouteCloseReason>; 4],
13760 }
13761 use RouteCloseReason::*;
13762 let cases = [
13763 Case {
13764 name: "a carrier entry removed",
13765 change: |r| {
13766 r.carriers.retain(|c| {
13767 c.principal
13768 != Principal::Reserved {
13769 module_id: AFT.to_string(),
13770 }
13771 })
13772 },
13773 closed: [None, Some(ScopeCarrierRemoved), None, None],
13774 },
13775 Case {
13776 name: "a target removed from a carrier",
13777 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
13778 closed: [None, None, None, Some(ScopeCarrierRemoved)],
13779 },
13780 Case {
13781 name: "a bare carrier narrowed to targets",
13782 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
13783 closed: [None, Some(ScopeCarrierRemoved), None, None],
13784 },
13785 Case {
13786 name: "delegates turned off",
13787 change: |r| r.attributes.delegates = false,
13788 closed: [Some(ScopeDelegationChanged); 4],
13789 },
13790 Case {
13791 name: "agent_id changed",
13792 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
13793 closed: [Some(ScopeDelegationChanged); 4],
13794 },
13795 Case {
13796 name: "a carrier added, child owners changed, the record re-sent",
13797 change: |r| {
13798 r.carriers.push(carrier(MAGIC, None));
13799 r.child_owners.push(Principal::Reserved {
13800 module_id: MAGIC.to_string(),
13801 });
13802 },
13803 closed: [None; 4],
13804 },
13805 Case {
13806 name: "a target added",
13807 change: |r| {
13808 r.carriers[1]
13809 .targets
13810 .as_mut()
13811 .unwrap()
13812 .push("third".to_string())
13813 },
13814 closed: [None; 4],
13815 },
13816 Case {
13817 name: "delegates turned on",
13818 change: |r| r.attributes.delegates = true,
13819 closed: [None; 4],
13820 },
13821 ];
13822 for case in cases {
13823 let mut rig = rig().await;
13824 rig.sync(vec![session(1)]).await;
13825 let scope = || Some(rig_selector("s", Some(1)));
13826 let mut routes = [
13827 rig.bound(Some(OWNER), PLEXUS, scope()).await,
13828 rig.bound(Some(AFT), PLEXUS, scope()).await,
13829 rig.bound(Some(BROCA), PLEXUS, scope()).await,
13830 rig.bound(Some(BROCA), OTHER, scope()).await,
13831 ];
13832 let mut record = session(1);
13833 (case.change)(&mut record);
13834 rig.sync(vec![record]).await;
13835 for (index, expected) in case.closed.iter().enumerate() {
13836 let route = &mut routes[index];
13837 match expected {
13838 Some(reason) => {
13839 assert!(!rig.live(route), "{}: route {index} still live", case.name);
13840 assert_eq!(
13841 route.closed_reason(),
13842 *reason,
13843 "{}: route {index}",
13844 case.name
13845 );
13846 }
13847 None => {
13848 assert!(rig.live(route), "{}: route {index} closed", case.name);
13849 assert!(
13850 route.untouched(),
13851 "{}: route {index} was told something",
13852 case.name
13853 );
13854 }
13855 }
13856 }
13857 }
13858 }
13859
13860 #[tokio::test]
13861 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
13862 for next in [Vec::new(), vec![session(2)]] {
13864 let mut rig = rig().await;
13865 rig.sync(vec![session(1)]).await;
13866 let mut route = rig
13867 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13868 .await;
13869 rig.sync(next).await;
13870 assert!(!rig.live(&route));
13871 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
13872 }
13873
13874 let mut rig = rig().await;
13877 let mut child = session(1);
13878 child.scope_ref = "child".to_string();
13879 child.kind = ScopeKind::Worker;
13880 child.parent = Some(ScopeParent {
13881 owner: Principal::Reserved {
13882 module_id: OWNER.to_string(),
13883 },
13884 scope_ref: "s".to_string(),
13885 scope_epoch: 1,
13886 });
13887 rig.sync(vec![session(1), child.clone()]).await;
13888 let mut child_route = rig
13889 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
13890 .await;
13891 assert_eq!(
13892 child_route.stamp().unwrap().parent_state,
13893 Some(ParentState::Linked)
13894 );
13895 rig.sync(vec![child]).await;
13896 assert!(!rig.live(&child_route));
13897 assert_eq!(
13898 child_route.closed_reason(),
13899 RouteCloseReason::ScopeParentEnded
13900 );
13901 }
13902
13903 #[tokio::test]
13904 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
13905 ) {
13906 let mut rig = rig().await;
13907 rig.sync(vec![session(1)]).await;
13908 let mut route = rig
13909 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13910 .await;
13911 let before = rig.forwarding.published_scope_tag(OWNER, "s");
13912 rig.sync(vec![session(1)]).await;
13913 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
13914 assert!(rig.live(&route) && route.untouched());
13915
13916 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
13920 .forwarding
13921 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
13922 .unwrap()
13923 else {
13924 panic!("the route is bound");
13925 };
13926 binding.flow.acquire_tagged(9, false).await.unwrap();
13927 let mut widened = session(1);
13928 widened.carriers.push(carrier(MAGIC, None));
13929 rig.sync(vec![widened]).await;
13930 assert!(rig.live(&route) && route.untouched());
13931 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13932 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
13933 assert_eq!(binding.flow.in_flight(), 1);
13936 binding
13937 .flow
13938 .acquire_tagged(10, false)
13939 .await
13940 .expect("the flow is still open");
13941 }
13942
13943 #[tokio::test]
13946 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
13947 let mut rig = rig().await;
13948 rig.sync(vec![session(1)]).await;
13949 let mut on_incumbent = rig
13950 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13951 .await;
13952
13953 let (candidate, _candidate_rx) = wide_ctx(9);
13956 let registration = rig
13957 .handler
13958 .registry
13959 .register_candidate_with_control_ops(
13960 manifest(PLEXUS, PROTOCOL_VERSION),
13961 PROTOCOL_VERSION,
13962 candidate.connection_id,
13963 module_baseline_control_ops(),
13964 )
13965 .unwrap();
13966 rig.forwarding
13967 .register_candidate_module_connection(
13968 candidate.connection_id,
13969 PLEXUS.to_string(),
13970 PROTOCOL_VERSION,
13971 manifest_concurrency(®istration.manifest),
13972 candidate.egress.clone(),
13973 )
13974 .unwrap();
13975 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
13976 rig.handler
13977 .registry
13978 .promote_candidate(PLEXUS)
13979 .unwrap()
13980 .unwrap();
13981 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
13982
13983 rig.sync(Vec::new()).await;
13984 assert!(!rig.live(&on_incumbent));
13985 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
13986 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13987 let goodbye = incumbent_rx
13988 .try_recv()
13989 .expect("the superseded endpoint is told")
13990 .frame;
13991 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13992 }
13993 }
13994}
13995
13996#[cfg(test)]
13997mod concurrency_default_exposure_tests {
13998 use super::*;
13999
14000 fn hello_body(role_json: &str) -> Vec<u8> {
14001 format!(
14002 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
14003 )
14004 .into_bytes()
14005 }
14006
14007 fn manifest_from(body: &[u8]) -> ModuleManifest {
14008 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
14009 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
14010 .expect("manifest parses")
14011 }
14012
14013 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
14014
14015 #[test]
14016 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
14017 let body = hello_body(&format!(
14018 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
14019 ));
14020 let manifest = manifest_from(&body);
14021 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
14024 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
14025 }
14026
14027 #[test]
14028 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
14029 let body = hello_body(&format!(
14030 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
14031 ));
14032 let manifest = manifest_from(&body);
14033 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
14034 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
14035 }
14036
14037 #[test]
14038 fn non_management_roles_are_never_reported() {
14039 let body = hello_body(
14040 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
14041 );
14042 let manifest = manifest_from(&body);
14043 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
14044 }
14045}