1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_event_declarations,
23 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
24 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
25 },
26 scope::{
27 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
29 SCOPE_SYNC_OP,
30 },
31 session::{
32 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
33 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
34 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
35 ROLE_VERSIONS_FIELD,
36 },
37 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
38 Priority, RouteTarget, PROTOCOL_VERSION,
39};
40use tokio::time::{timeout_at, Instant};
41use tracing::{debug, info, warn};
42
43use crate::{
44 capability_requirements::{
45 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
46 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
47 RuntimeModule,
48 },
49 daemon_config::RestartRequiredSection,
50 forwarding::{
51 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
52 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
53 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
54 },
55 observability::{
56 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
57 },
58 provenance::{
59 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
60 },
61 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
62 router::{RouteCtx, RouterError},
63 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
64 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
65 stderr_tail::{CaptureState, TailEntry},
66 supervise::{
67 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
68 SupervisorHandle, SwapHelloAdmission,
69 },
70 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
71};
72
73pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
79
80const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
81const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
82const CAP_PING_PONG: &str = "ping_pong_v1";
83const CAP_SESSION_ATTACH: &str = "session_attach_v1";
84const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
85
86const SUBC_CONTROL_OPS: &[&str] = &[
87 ops::SERVER_DESCRIBE,
88 ops::CATALOG_LIST,
89 ops::ROUTE_OPEN,
90 ops::ROUTE_POLL,
91 ops::ROUTE_CLOSING,
92 ops::ROUTE_CLOSED,
93 ops::SUPERVISOR_LIST,
94 ops::SUPERVISOR_RESTART,
95 ops::SUPERVISOR_SWAP,
96 ops::SUPERVISOR_RELOAD,
97 ops::SUPERVISOR_RESCAN,
98 ops::SUPERVISOR_RELEASE_RESERVED,
99 ops::SUPERVISOR_SET_ENABLED,
100 ops::SUPERVISOR_HEALTH_PROBE,
101 ops::SUPERVISOR_HEALTH,
102 ops::SUPERVISOR_STDERR_TAIL,
103 ops::SUPERVISOR_TERMINALS,
104 ops::SUPERVISOR_ROUTES,
105 ops::SUPERVISOR_PROVENANCE,
106 ops::SUPERVISOR_SPAWN_SNAPSHOT,
107 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
108];
109
110const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
111 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
112 "supervisor.live_roots",
113 SCOPE_SYNC_OP,
114 SCOPE_APPLY_OP,
115 SCOPE_DESCRIBE_OP,
116 "operator.confirm",
117];
118
119const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
123
124const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
125
126pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
134
135pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
150
151pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
169
170const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
171const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
172
173fn reload_verdict(
174 configured: &Path,
175 spawned_from: Option<&Path>,
176 image: subc_control::RunningImageAgreement,
177) -> PendingReloadVerdict {
178 let path = match spawned_from {
179 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
180 Some(spawned_from) => ReloadPathAgreement::Mismatch {
181 configured: configured.to_path_buf(),
182 spawned_from: spawned_from.to_path_buf(),
183 },
184 None => ReloadPathAgreement::Unavailable {
185 reason: if matches!(
186 image,
187 subc_control::RunningImageAgreement::Unavailable {
188 reason: subc_control::RunningImageUnavailableReason::NotRunning
189 }
190 ) {
191 ReloadPathUnavailableReason::NotRunning
192 } else {
193 ReloadPathUnavailableReason::SpawnedPathUnavailable
194 },
195 },
196 };
197 PendingReloadVerdict { path, image }
198}
199
200#[derive(Clone)]
201struct DaemonProvenanceFacts {
202 build: DaemonBuildProvenance,
203 pid: Option<u32>,
204 started_at_ms: Option<u64>,
205 start_clock: Option<crate::clock::StartClock>,
206 executable_path: Option<PathBuf>,
207 executable_identity: Option<SpawnedFileIdentity>,
208 process_start_time: Option<u64>,
209 probe: ExecutableIdentityProbe,
210}
211
212impl Default for DaemonProvenanceFacts {
213 fn default() -> Self {
214 Self {
215 build: DaemonBuildProvenance {
216 build_git_sha: None,
217 build_lock_digest: None,
218 },
219 pid: None,
220 started_at_ms: None,
221 start_clock: None,
222 executable_path: None,
223 executable_identity: None,
224 process_start_time: None,
225 probe: ExecutableIdentityProbe::default(),
226 }
227 }
228}
229
230#[derive(Debug, Clone)]
231struct SupervisorRescanContext {
232 supervisor: Supervisor,
233 config_path: PathBuf,
234 configured_port: Option<u16>,
235 storage_config: Option<crate::daemon_config::StorageConfig>,
236 admission_facts_carrier_module_id: Option<String>,
237 admission_facts_targets: Option<Vec<String>>,
238 scope_authority_owners: Vec<String>,
239}
240
241const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
262 "reloading",
263 "supervisor_not_live",
264 "registration_not_active",
265 "no_forwarding_connection",
266 "relay_send_failed",
267];
268
269#[derive(Clone)]
271pub struct ControlHandler {
272 registry: Arc<Registry>,
273 forwarding: Arc<ForwardingTable>,
274 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
275 supervisor: SupervisorHandle,
276 subc_capabilities: Arc<[String]>,
277 route_bind_relay_timeout: Duration,
281 route_bind_relay_timeouts: BTreeMap<String, Duration>,
285 route_bind_breakers: RouteBindBreakers,
288 route_bind_concurrency: RouteBindConcurrency,
291 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
296 route_bind_breaker_threshold: u32,
298 route_bind_breaker_cooldown: Duration,
300 health_probe_timeout: Duration,
301 storage_config: Option<crate::daemon_config::StorageConfig>,
304 machine_id: Option<crate::machine_id::MachineId>,
308 admission_facts_carrier_module_id: Option<String>,
309 admission_facts_targets: Option<Vec<String>>,
310 scopes: Arc<RwLock<ScopeTable>>,
314 scope_deadline_added: Arc<tokio::sync::Notify>,
318 scope_authority_owners: Vec<String>,
321 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
324 rescan: Option<SupervisorRescanContext>,
325 connected_clients: ConnectedClients,
326 counters: DaemonCounters,
327 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
328 daemon_provenance: DaemonProvenanceFacts,
329 #[cfg(test)]
330 control_dispatch_delay: Option<Duration>,
331 #[cfg(test)]
332 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
333}
334
335impl fmt::Debug for ControlHandler {
336 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
337 f.debug_struct("ControlHandler")
338 .field("registry", &self.registry)
339 .field("forwarding", &self.forwarding)
340 .field("process_liveness", &self.process_liveness.is_some())
341 .field("supervisor", &self.supervisor)
342 .field("subc_capabilities", &self.subc_capabilities)
343 .finish()
344 }
345}
346
347struct RouteOpenRequest {
348 target: RouteTarget,
349 identity: BindIdentity,
350 consumer_identity: Option<ConsumerIdentity>,
351 consumer_capabilities: Option<Vec<String>>,
352 role_versions: Option<BTreeMap<String, String>>,
353 admission_facts: Option<serde_json::Value>,
354 scope: Option<ScopeSelector>,
355}
356
357struct RouteBindReservationGuard {
358 forwarding: Arc<ForwardingTable>,
359 endpoint: ModuleEndpointId,
360 relay_corr: u64,
361 armed: bool,
362}
363
364struct ModuleControlRpcGuard {
365 forwarding: Arc<ForwardingTable>,
366 endpoint: ModuleEndpointId,
367 corr: u64,
368 armed: bool,
369}
370
371impl ModuleControlRpcGuard {
372 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
373 Self {
374 forwarding,
375 endpoint,
376 corr,
377 armed: true,
378 }
379 }
380
381 fn disarm(&mut self) {
382 self.armed = false;
383 }
384}
385
386impl Drop for ModuleControlRpcGuard {
387 fn drop(&mut self) {
388 if self.armed {
389 let _ = self
390 .forwarding
391 .cancel_module_control_rpc(self.endpoint, self.corr);
392 }
393 }
394}
395
396impl RouteBindReservationGuard {
397 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
398 Self {
399 forwarding,
400 endpoint,
401 relay_corr,
402 armed: true,
403 }
404 }
405
406 fn release_and_disarm(&mut self) {
407 if !self.armed {
408 return;
409 }
410 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
411 self.endpoint,
412 self.relay_corr,
413 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
414 ) {
415 send_goodbye_target_best_effort(
416 &self.forwarding.counters(),
417 &target,
418 "canceled route.bind",
419 );
420 }
421 self.armed = false;
422 }
423
424 fn disarm(&mut self) {
425 self.armed = false;
426 }
427}
428
429impl Drop for RouteBindReservationGuard {
430 fn drop(&mut self) {
431 self.release_and_disarm();
432 }
433}
434
435#[derive(Debug, Clone, Default)]
465pub(crate) struct RouteBindBreakers {
466 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
467}
468
469#[derive(Debug, Clone, Default)]
470pub(crate) struct RouteBindConcurrency {
471 modules: Arc<Mutex<HashMap<String, usize>>>,
472}
473
474struct RouteBindConcurrencyGuard {
475 concurrency: RouteBindConcurrency,
476 module_id: String,
477}
478
479impl RouteBindConcurrency {
480 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
483 let mut modules = self
484 .modules
485 .lock()
486 .expect("route.bind concurrency mutex poisoned");
487 let in_flight = modules.entry(module_id.to_string()).or_default();
488 if *in_flight >= limit {
489 return Err(*in_flight);
490 }
491 *in_flight += 1;
492 Ok(RouteBindConcurrencyGuard {
493 concurrency: self.clone(),
494 module_id: module_id.to_string(),
495 })
496 }
497}
498
499impl Drop for RouteBindConcurrencyGuard {
500 fn drop(&mut self) {
501 let mut modules = self
502 .concurrency
503 .modules
504 .lock()
505 .expect("route.bind concurrency mutex poisoned");
506 let remove = {
507 let in_flight = modules
508 .get_mut(&self.module_id)
509 .expect("admitted route.bind has a concurrency entry");
510 *in_flight -= 1;
511 *in_flight == 0
512 };
513 if remove {
514 modules.remove(&self.module_id);
515 }
516 }
517}
518
519#[derive(Debug, Default)]
520struct ModuleBreakerState {
521 consecutive_timeouts: u32,
523 cooldown_until: Option<Instant>,
526 probe_in_flight: Option<Arc<()>>,
531}
532
533enum RouteBindAdmission<'a> {
535 Admitted {
536 guard: RouteBindBreakerGuard<'a>,
537 probe: bool,
540 },
541 Refused {
542 consecutive_timeouts: u32,
543 retry_in: Duration,
547 probe_in_flight: bool,
548 },
549}
550
551struct RouteBindBreakerGuard<'a> {
558 breakers: RouteBindBreakers,
559 module_id: &'a str,
560 probe_token: Option<Arc<()>>,
561 settled: bool,
562}
563
564impl RouteBindBreakerGuard<'_> {
565 fn record_accepted(&mut self) -> bool {
569 self.settled = true;
570 self.breakers.record_accepted(self.module_id)
571 }
572
573 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
576 self.settled = true;
577 self.breakers.record_timeout(
578 self.module_id,
579 self.probe_token.as_ref(),
580 threshold,
581 cooldown,
582 )
583 }
584
585 fn record_inconclusive(&mut self) {
594 self.settled = true;
595 self.breakers
596 .record_inconclusive(self.module_id, self.probe_token.as_ref());
597 }
598}
599
600impl Drop for RouteBindBreakerGuard<'_> {
601 fn drop(&mut self) {
602 if !self.settled {
603 self.breakers
604 .record_inconclusive(self.module_id, self.probe_token.as_ref());
605 }
606 }
607}
608
609struct BreakerOpened {
613 consecutive_timeouts: u32,
614 reopened_after_probe: bool,
617}
618
619impl RouteBindBreakers {
620 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
621 self.modules
622 .lock()
623 .expect("route.bind breaker mutex poisoned")
624 }
625
626 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
629 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
630 probe: probe_token.is_some(),
631 guard: RouteBindBreakerGuard {
632 breakers: self.clone(),
633 module_id,
634 probe_token,
635 settled: false,
636 },
637 };
638
639 let mut modules = self.lock();
640 let Some(state) = modules.get_mut(module_id) else {
641 return admitted(None);
642 };
643 let Some(cooldown_until) = state.cooldown_until else {
644 return admitted(None);
645 };
646 if state.probe_in_flight.is_some() {
647 return RouteBindAdmission::Refused {
648 consecutive_timeouts: state.consecutive_timeouts,
649 retry_in: Duration::ZERO,
650 probe_in_flight: true,
651 };
652 }
653 let now = Instant::now();
654 if now < cooldown_until {
655 return RouteBindAdmission::Refused {
656 consecutive_timeouts: state.consecutive_timeouts,
657 retry_in: cooldown_until - now,
658 probe_in_flight: false,
659 };
660 }
661 let token = Arc::new(());
662 state.probe_in_flight = Some(Arc::clone(&token));
663 admitted(Some(token))
664 }
665
666 fn record_accepted(&self, module_id: &str) -> bool {
667 self.lock()
668 .remove(module_id)
669 .is_some_and(|state| state.cooldown_until.is_some())
670 }
671
672 fn record_timeout(
673 &self,
674 module_id: &str,
675 probe_token: Option<&Arc<()>>,
676 threshold: u32,
677 cooldown: Duration,
678 ) -> Option<BreakerOpened> {
679 let mut modules = self.lock();
680 let state = modules.entry(module_id.to_string()).or_default();
681 let was_open = state.cooldown_until.is_some();
682 let was_probe = Self::owns_probe(state, probe_token);
683 if was_probe {
684 state.probe_in_flight = None;
685 }
686 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
687 if state.consecutive_timeouts < threshold {
688 return None;
689 }
690 state.cooldown_until = Some(Instant::now() + cooldown);
691 Some(BreakerOpened {
692 consecutive_timeouts: state.consecutive_timeouts,
693 reopened_after_probe: was_open && was_probe,
694 })
695 }
696
697 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
698 state
702 .probe_in_flight
703 .as_ref()
704 .zip(token)
705 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
706 }
707
708 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
709 if let Some(state) = self.lock().get_mut(module_id) {
710 if Self::owns_probe(state, probe_token) {
711 state.probe_in_flight = None;
712 }
713 }
714 }
715
716 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
731 self.lock()
732 .remove(module_id)
733 .map(|state| state.consecutive_timeouts)
734 .filter(|discarded| *discarded > 0)
735 }
736
737 fn open_snapshot(&self) -> Option<serde_json::Value> {
744 let now = Instant::now();
745 let modules = self.lock();
746 let open = modules
747 .iter()
748 .filter_map(|(module_id, state)| {
749 let cooldown_until = state.cooldown_until?;
750 Some((
751 module_id.clone(),
752 serde_json::json!({
753 "consecutive_timeouts": state.consecutive_timeouts,
754 "cooldown_remaining_ms":
755 cooldown_until.saturating_duration_since(now).as_millis() as u64,
756 "probe_in_flight": state.probe_in_flight.is_some(),
757 }),
758 ))
759 })
760 .collect::<serde_json::Map<String, serde_json::Value>>();
761 (!open.is_empty()).then_some(serde_json::Value::Object(open))
762 }
763}
764
765impl ControlHandler {
766 pub fn new(registry: Arc<Registry>) -> Self {
767 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
768 }
769
770 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
771 let counters = forwarding.counters();
772 let route_bind_breakers = forwarding.route_bind_breakers();
776 let route_bind_concurrency = forwarding.route_bind_concurrency();
777 let route_outages = forwarding.route_outages();
778 Self {
779 registry,
780 forwarding,
781 process_liveness: None,
782 supervisor: SupervisorHandle::new(),
783 subc_capabilities: Arc::from([
784 CAP_MANIFEST_REGISTRATION.to_string(),
785 CAP_CHANNEL_LIFECYCLE.to_string(),
786 CAP_PING_PONG.to_string(),
787 CAP_SESSION_ATTACH.to_string(),
788 CAP_ADMISSION_FACTS_RELAY.to_string(),
789 CAP_SCOPES_V1.to_string(),
790 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
791 ]),
792 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
793 route_bind_relay_timeouts: BTreeMap::new(),
794 route_bind_breakers,
795 route_bind_concurrency,
796 route_outages,
797 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
798 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
799 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
800 storage_config: None,
801 machine_id: None,
802 admission_facts_carrier_module_id: None,
803 admission_facts_targets: None,
804 scopes: Arc::new(RwLock::new(ScopeTable::new(
805 crate::daemon_config::default_scope_authority_owners(),
806 ))),
807 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
808 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
809 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
810 rescan: None,
811 connected_clients: ConnectedClients::new(),
812 counters,
813 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
814 daemon_provenance: DaemonProvenanceFacts::default(),
815 #[cfg(test)]
816 control_dispatch_delay: None,
817 #[cfg(test)]
818 provenance_probe_override: None,
819 }
820 }
821
822 pub fn with_storage_config(
825 mut self,
826 storage_config: Option<crate::daemon_config::StorageConfig>,
827 ) -> Self {
828 self.storage_config = storage_config;
829 self
830 }
831
832 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
835 self.machine_id = machine_id;
836 self
837 }
838
839 pub fn with_admission_facts_config(
843 mut self,
844 carrier_module_id: Option<String>,
845 targets: Option<Vec<String>>,
846 ) -> Self {
847 self.admission_facts_carrier_module_id = carrier_module_id;
848 self.admission_facts_targets = targets;
849 self
850 }
851
852 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
856 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
857 self.scope_authority_owners = owners;
858 self
859 }
860
861 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
864 self.route_bind_relay_timeout = timeout;
865 self
866 }
867
868 pub fn with_route_bind_relay_timeouts(
874 mut self,
875 timeouts: impl IntoIterator<Item = (String, Duration)>,
876 ) -> Self {
877 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
878 self
879 }
880
881 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
887 self.route_bind_relay_timeouts
888 .get(module_id)
889 .copied()
890 .unwrap_or(self.route_bind_relay_timeout)
891 }
892
893 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
901 self.route_bind_breaker_threshold = threshold.max(1);
902 self.route_bind_breaker_cooldown = cooldown;
903 self
904 }
905
906 #[cfg(test)]
907 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
908 self.health_probe_timeout = timeout;
909 self
910 }
911
912 #[cfg(test)]
913 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
914 self.control_dispatch_delay = Some(delay);
915 self
916 }
917
918 pub fn with_process_liveness(
919 mut self,
920 process_liveness: Arc<dyn ModuleProcessLiveness>,
921 ) -> Self {
922 self.process_liveness = Some(process_liveness);
923 self
924 }
925
926 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
927 self.supervisor = supervisor;
928 self
929 }
930
931 pub fn with_daemon_provenance(
932 mut self,
933 pid: u32,
934 started_at_ms: u64,
935 executable_path: Option<PathBuf>,
936 build_git_sha: Option<String>,
937 build_lock_digest: Option<String>,
938 ) -> Self {
939 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
940 let process_start_time = process_start_time(pid);
941 self.daemon_provenance = DaemonProvenanceFacts {
942 build: DaemonBuildProvenance {
943 build_git_sha,
944 build_lock_digest,
945 },
946 pid: Some(pid),
947 started_at_ms: Some(started_at_ms),
948 start_clock: None,
949 executable_path,
950 executable_identity,
951 process_start_time,
952 probe: ExecutableIdentityProbe::default(),
953 };
954 self
955 }
956
957 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
958 self.daemon_provenance.start_clock = Some(clock);
959 self
960 }
961
962 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
964 self.scopes
965 .write()
966 .unwrap_or_else(|poisoned| poisoned.into_inner())
967 .set_wall_clock(Arc::new(clock));
968 self
969 }
970
971 #[cfg(test)]
972 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
973 self.provenance_probe_override = Some(result);
974 self
975 }
976
977 pub fn with_capability_config(
981 self,
982 modules: impl IntoIterator<Item = (String, bool)>,
983 reserved_capabilities: BTreeMap<String, String>,
984 ) -> Self {
985 self.capability_evaluator
986 .configure(modules, reserved_capabilities);
987 self
988 }
989
990 pub fn with_supervisor_rescan(
991 mut self,
992 supervisor: Supervisor,
993 config_path: impl Into<PathBuf>,
994 configured_port: Option<u16>,
995 ) -> Self {
996 self.rescan = Some(SupervisorRescanContext {
997 supervisor,
998 config_path: config_path.into(),
999 configured_port,
1000 storage_config: self.storage_config.clone(),
1001 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1002 admission_facts_targets: self.admission_facts_targets.clone(),
1003 scope_authority_owners: self.scope_authority_owners.clone(),
1004 });
1005 self
1006 }
1007
1008 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1009 self.connected_clients = connected_clients;
1010 self
1011 }
1012
1013 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1014 Arc::clone(&self.forwarding)
1015 }
1016
1017 pub(crate) fn counters(&self) -> DaemonCounters {
1018 self.counters.clone()
1019 }
1020
1021 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1024 tokio::spawn(async move {
1025 loop {
1026 self.capability_evaluator
1027 .wait_for_change_or_deadline()
1028 .await;
1029 self.refresh_capability_requirements();
1030 }
1031 });
1032 }
1033
1034 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1040 tokio::spawn(async move {
1041 loop {
1042 let has_deadlines = self
1043 .scopes
1044 .read()
1045 .unwrap_or_else(|poisoned| poisoned.into_inner())
1046 .has_deadlines();
1047 if !has_deadlines {
1048 self.scope_deadline_added.notified().await;
1049 continue;
1050 }
1051 tokio::time::sleep(Duration::from_secs(1)).await;
1052 if let Err(error) = self.sweep_expired_scopes() {
1053 warn!(%error, "scope expiry sweep failed");
1054 }
1055 }
1056 });
1057 }
1058
1059 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1060 let mut table = self
1061 .scopes
1062 .write()
1063 .unwrap_or_else(|poisoned| poisoned.into_inner());
1064 let swept = table.sweep_expired();
1065 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1066 drop(table);
1067 self.close_scope_drained_routes(drained);
1068 Ok(())
1069 }
1070
1071 fn publish_scope_changes(
1072 &self,
1073 changes: &[crate::scopes::ScopeTagChange],
1074 expired: &[crate::scopes::ScopeExpired],
1075 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1076 if changes.is_empty() {
1077 return Ok(Vec::new());
1078 }
1079 let drained = self
1080 .forwarding
1081 .publish_scope_changes(changes)
1082 .map_err(RouterError::Forwarding)?;
1083 let mut counts = HashMap::new();
1084 for route in &drained {
1085 *counts
1086 .entry((
1087 route.scope.owner.as_str(),
1088 route.scope.scope_ref.as_str(),
1089 route.scope.tag.scope_epoch,
1090 ))
1091 .or_insert(0usize) += 1;
1092 }
1093 for scope in expired {
1094 let routes_closed = counts
1095 .get(&(
1096 scope.owner.as_str(),
1097 scope.scope_ref.as_str(),
1098 scope.scope_epoch,
1099 ))
1100 .copied()
1101 .unwrap_or(0);
1102 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1103 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1104 routes_closed, "scope expired");
1105 }
1106 Ok(drained)
1107 }
1108
1109 fn runtime_capability_snapshot(
1110 &self,
1111 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1112 let runtime = self
1113 .supervisor
1114 .list()
1115 .into_iter()
1116 .map(|module| {
1117 let status = module.status().map_err(|err| {
1118 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1119 })?;
1120 Ok(RuntimeModule {
1121 module_id: status.module_id,
1122 state: status.state,
1123 enabled: status.enabled,
1124 })
1125 })
1126 .collect::<Result<Vec<_>, RouterError>>()?;
1127 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1128 RouterError::backend(
1129 0,
1130 0,
1131 format!("failed to list capability registrations: {err}"),
1132 )
1133 })?;
1134 let registrations = registrations
1135 .into_iter()
1136 .map(|registration| RegisteredModule {
1137 module_id: registration.manifest.module_id,
1138 module_version: registration.manifest.module_version,
1139 capabilities: registration.manifest.capabilities,
1140 })
1141 .collect();
1142 Ok((runtime, registrations))
1143 }
1144
1145 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1152 let cached_registration = RegisteredModule {
1153 module_id: registration.manifest.module_id.clone(),
1154 module_version: registration.manifest.module_version.clone(),
1155 capabilities: registration.manifest.capabilities.clone(),
1156 };
1157 if self.capability_evaluator.record_hello(&cached_registration) {
1158 warn!(
1159 module_id = %cached_registration.module_id,
1160 "capability claims drifted from the cached manifest"
1161 );
1162 }
1163 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1164 self.enforce_capability_denies();
1165 }
1166 self.refresh_capability_requirements();
1167 }
1168
1169 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1173 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1174 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1175 self.supervisor.set_swap_promotion_observer(observer);
1176 }
1177
1178 pub fn refresh_capability_requirements(&self) {
1179 match self.runtime_capability_snapshot() {
1180 Ok((runtime, registrations)) => {
1181 log_requirement_events(
1182 self.capability_evaluator
1183 .evaluate_now(&runtime, ®istrations),
1184 );
1185 }
1186 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1187 }
1188 }
1189
1190 fn enforce_capability_denies(&self) {
1194 let (_, registrations) = match self.registry.list_modules() {
1195 Ok(snapshot) => snapshot,
1196 Err(err) => {
1197 warn!(error = %err, "failed to read registrations for capability deny census");
1198 return;
1199 }
1200 };
1201 let manifests = registrations
1202 .into_iter()
1203 .map(|registration| {
1204 (
1205 registration.manifest.module_id.clone(),
1206 registration.manifest,
1207 )
1208 })
1209 .collect::<BTreeMap<_, _>>();
1210 let census = match self.forwarding.route_census(None) {
1211 Ok(census) => census,
1212 Err(err) => {
1213 warn!(error = %err, "failed to read route census for capability deny enforcement");
1214 return;
1215 }
1216 };
1217
1218 for (target_module_id, routes) in census {
1219 let Some(target_manifest) = manifests.get(&target_module_id) else {
1220 continue;
1221 };
1222 let mut closed_routes = Vec::new();
1223 let mut module_goodbyes = Vec::new();
1224 for route in routes {
1225 let Principal::Reserved {
1226 module_id: opening_module_id,
1227 } = &route.principal
1228 else {
1229 continue;
1230 };
1231 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1232 continue;
1233 };
1234 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1235 continue;
1236 };
1237
1238 match self.forwarding.release_client_route(
1239 route.goodbye_target.connection_id,
1240 route.goodbye_target.channel,
1241 route.goodbye_target.epoch,
1242 ) {
1243 Ok(RouteRelease::Removed(module_goodbye)) => {
1244 warn!(
1245 opening_module_id,
1246 target_module_id,
1247 capability,
1248 "force-closing route because an attested capability deny edge now matches"
1249 );
1250 closed_routes.push(route);
1251 module_goodbyes.push(module_goodbye);
1252 }
1253 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1254 Err(err) => warn!(
1255 opening_module_id,
1256 target_module_id,
1257 capability,
1258 error = %err,
1259 "failed to force-close capability-denied route"
1260 ),
1261 }
1262 }
1263
1264 if closed_routes.is_empty() {
1265 continue;
1266 }
1267 send_route_control_pushes(
1268 &self.forwarding,
1269 closed_routes,
1270 ClientControlPush::RouteClosed {
1271 module_id: target_module_id,
1272 channels: Vec::new(),
1273 reason: RouteCloseReason::CapabilityDenied,
1274 drained: false,
1275 abandoned: 0,
1276 excluded_subscriptions: 0,
1277 terminal: Some(false),
1278 },
1279 );
1280 self.emit_route_goodbyes(module_goodbyes);
1281 }
1282 }
1283
1284 fn not_ready_reason(
1290 &self,
1291 registration: &crate::registry::ModuleRegistration,
1292 ) -> Option<NotReadyReason> {
1293 if !registration.ready {
1294 return Some(NotReadyReason {
1295 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1296 capability: None,
1297 });
1298 }
1299 self.first_unprovided_required_capability(registration)
1300 .map(|capability| NotReadyReason {
1301 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1302 capability: Some(capability),
1303 })
1304 }
1305
1306 fn first_unprovided_required_capability(
1324 &self,
1325 registration: &crate::registry::ModuleRegistration,
1326 ) -> Option<String> {
1327 let required = registration
1328 .manifest
1329 .capabilities
1330 .iter()
1331 .flat_map(|declarations| declarations.requires.iter())
1332 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1333 .map(|requirement| requirement.capability.as_str())
1334 .collect::<BTreeSet<_>>();
1335 if required.is_empty() {
1336 return None;
1337 }
1338 let module_id = registration.manifest.module_id.as_str();
1339 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1340 if required
1341 .iter()
1342 .any(|capability| verdict(capability).is_none())
1343 {
1344 self.refresh_capability_requirements();
1345 }
1346 required
1347 .into_iter()
1348 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1349 .map(str::to_string)
1350 }
1351
1352 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1353 self.capability_evaluator
1354 .statuses()
1355 .into_iter()
1356 .map(capability_requirement_status)
1357 .collect()
1358 }
1359
1360 fn deregister_connection(
1367 &self,
1368 connection_id: ConnectionId,
1369 reason: RegistrationEndReason,
1370 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1371 self.registry
1372 .deregister_connection_with_reason(connection_id, reason)
1373 }
1374
1375 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1376 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1377 return None;
1378 }
1379 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1380 parse_client_control_request(&frame.body)
1381 else {
1382 return None;
1383 };
1384 Some(target_module_id(&target).to_string())
1385 }
1386
1387 pub(crate) fn route_open_capacity_refusal(
1388 &self,
1389 ctx: &RouteCtx,
1390 frame: &Frame,
1391 target_module_id: &str,
1392 in_flight: usize,
1393 limit: usize,
1394 ) -> Result<Frame, RouterError> {
1395 self.route_open_admission_refusal_frame(
1396 ctx,
1397 frame,
1398 target_module_id,
1399 "open_admission_full",
1400 (in_flight, limit),
1401 format!(
1402 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1403 ),
1404 )
1405 }
1406
1407 fn route_open_target_capacity_refusal(
1408 &self,
1409 ctx: &RouteCtx,
1410 frame: &Frame,
1411 target_module_id: &str,
1412 in_flight: usize,
1413 ) -> Result<Frame, RouterError> {
1414 self.route_open_admission_refusal_frame(
1415 ctx,
1416 frame,
1417 target_module_id,
1418 "target_binds_full",
1419 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1420 format!(
1421 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1422 ),
1423 )
1424 }
1425
1426 fn route_open_admission_refusal_frame(
1434 &self,
1435 ctx: &RouteCtx,
1436 frame: &Frame,
1437 target_module_id: &str,
1438 reason: &'static str,
1439 (in_flight, limit): (usize, usize),
1440 message: impl Into<String>,
1441 ) -> Result<Frame, RouterError> {
1442 let code = error_codes::TARGET_UNAVAILABLE;
1443 self.counters.increment_route_open_refused(code);
1444 info!(
1445 target: "control",
1446 code,
1447 reason,
1448 module_id = ?target_module_id,
1449 connection_id = ctx.connection_id.get(),
1450 in_flight,
1451 limit,
1452 "route.open refused"
1453 );
1454 control_error_frame(frame, code, message.into())
1455 }
1456
1457 #[cfg(test)]
1463 pub fn handle_control(
1464 &self,
1465 connection_id: ConnectionId,
1466 frame: Frame,
1467 ) -> Result<Vec<Frame>, RouterError> {
1468 match frame.header.ty {
1469 FrameType::Ping => Ok(vec![pong(&frame)?]),
1470 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1471 FrameType::Goodbye => self.handle_goodbye(connection_id),
1472 ty => Ok(vec![control_error_frame(
1473 &frame,
1474 "unsupported_control_frame",
1475 format!("unsupported channel-0 frame {ty:?}"),
1476 )?]),
1477 }
1478 }
1479
1480 pub async fn handle_control_frame(
1481 &self,
1482 ctx: &RouteCtx,
1483 frame: Frame,
1484 ) -> Result<Vec<Frame>, RouterError> {
1485 self.handle_control_frame_timed(ctx, frame, None).await
1486 }
1487
1488 pub(crate) async fn handle_control_frame_timed(
1489 &self,
1490 ctx: &RouteCtx,
1491 frame: Frame,
1492 dispatch_started_at: Option<StdInstant>,
1493 ) -> Result<Vec<Frame>, RouterError> {
1494 match frame.header.ty {
1495 FrameType::Ping => Ok(vec![pong(&frame)?]),
1496 FrameType::Hello => {
1497 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1498 }
1499 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1500 FrameType::Cancel => {
1501 if self
1504 .forwarding
1505 .operator_confirms()
1506 .cancel(ctx.connection_id, frame.header.corr)
1507 || self
1508 .supervisor
1509 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1510 {
1511 Ok(Vec::new())
1512 } else {
1513 Ok(vec![control_error_frame(
1514 &frame,
1515 "unknown_subscription",
1516 "no supervisor spawn subscription has this correlation id",
1517 )?])
1518 }
1519 }
1520 FrameType::Request => {
1521 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1524 if op
1525 .as_ref()
1526 .is_some_and(|probe| probe.op == "operator.confirm")
1527 {
1528 return self.handle_operator_confirm(ctx, frame);
1529 }
1530 if self
1531 .forwarding
1532 .module_endpoint_for_connection(ctx.connection_id)
1533 .map_err(RouterError::Forwarding)?
1534 .is_some()
1535 {
1536 if !is_known_module_request_op(&frame.body) {
1537 return Ok(vec![control_error_frame(
1538 &frame,
1539 "unsupported_control_frame",
1540 "module-originated channel-0 REQUEST is not supported",
1541 )?]);
1542 }
1543 let request = match parse_module_control_request_from_module(&frame.body) {
1544 Ok(request) => request,
1545 Err((err, ControlRequestBodyError::UnknownOp)) => {
1546 return Ok(vec![control_error_frame(
1547 &frame,
1548 "unsupported_control_frame",
1549 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1550 )?])
1551 }
1552 Err((err, ControlRequestBodyError::InvalidBody)) => {
1553 return Ok(vec![control_error_frame(
1554 &frame,
1555 "invalid_control_body",
1556 format!("malformed module control body: {err}"),
1557 )?])
1558 }
1559 };
1560 let op = module_control_request_op(&request);
1561 let corr = frame.header.corr;
1562 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1563 let result =
1564 self.handle_module_control_request(ctx.connection_id, frame, request);
1565 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1566 return result;
1567 }
1568
1569 if is_known_module_request_op(&frame.body) {
1570 return Ok(vec![control_error_frame(
1571 &frame,
1572 "not_registered",
1573 "catalog.update requires an active module registration owned by this connection",
1574 )?]);
1575 }
1576
1577 let request = match parse_client_control_request(&frame.body) {
1578 Ok(request) => request,
1579 Err((err, ControlRequestBodyError::UnknownOp)) => {
1580 return Ok(vec![control_error_frame(
1581 &frame,
1582 "unknown_control_op",
1583 format!("unknown client control op: {err}"),
1584 )?])
1585 }
1586 Err((err, ControlRequestBodyError::InvalidBody)) => {
1587 return Ok(vec![control_error_frame(
1588 &frame,
1589 "invalid_control_body",
1590 format!("malformed client control body: {err}"),
1591 )?])
1592 }
1593 };
1594 let op = client_control_request_op(&request);
1595 let corr = frame.header.corr;
1596 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1597 #[cfg(test)]
1598 if let Some(delay) = self.control_dispatch_delay {
1599 tokio::time::sleep(delay).await;
1600 }
1601 let result = self
1602 .handle_client_control_request(ctx, frame, request)
1603 .await;
1604 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1605 result
1606 }
1607 FrameType::Push => {
1608 let Some(endpoint) = self
1609 .forwarding
1610 .module_endpoint_for_connection(ctx.connection_id)
1611 .map_err(RouterError::Forwarding)?
1612 else {
1613 return Ok(vec![control_error_frame(
1614 &frame,
1615 "unsupported_control_frame",
1616 "client-originated channel-0 PUSH is not supported",
1617 )?]);
1618 };
1619 self.handle_status_update(endpoint, frame)
1620 }
1621 FrameType::Response | FrameType::Error
1622 if self
1623 .forwarding
1624 .module_endpoint_for_connection(ctx.connection_id)
1625 .map_err(RouterError::Forwarding)?
1626 .is_some() =>
1627 {
1628 self.handle_module_relay_response(ctx.connection_id, frame)
1629 }
1630 ty => Ok(vec![control_error_frame(
1631 &frame,
1632 "unsupported_control_frame",
1633 format!("unsupported channel-0 frame {ty:?}"),
1634 )?]),
1635 }
1636 }
1637
1638 pub fn cleanup_connection(
1639 &self,
1640 connection_id: ConnectionId,
1641 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1642 self.cleanup_connection_with_end_reason(
1643 connection_id,
1644 RegistrationEndReason::ConnectionClosed,
1645 )
1646 }
1647
1648 fn cleanup_connection_with_end_reason(
1649 &self,
1650 connection_id: ConnectionId,
1651 requested_reason: RegistrationEndReason,
1652 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1653 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1654 self.registry
1655 .get_module_by_connection(connection_id)?
1656 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1657 .and_then(|module| module.registration_end_reason().ok().flatten())
1658 .unwrap_or(requested_reason)
1659 } else {
1660 requested_reason
1661 };
1662 let crash_closed = self
1663 .registry
1664 .get_module_by_connection(connection_id)?
1665 .and_then(|registration| {
1666 self.forwarding
1667 .module_endpoint_for_connection(connection_id)
1668 .ok()
1669 .flatten()
1670 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1671 .map(|routes| (registration.manifest.module_id, routes))
1672 });
1673 let crash_closed = crash_closed.map(|(module_id, routes)| {
1674 let terminal = match self.supervisor.get(&module_id) {
1675 None => false,
1676 Some(module) => match module.will_recover_after_connection_loss() {
1677 Ok(will_recover) => !will_recover,
1678 Err(err) => {
1679 warn!(
1680 %module_id,
1681 error = %err,
1682 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1683 );
1684 false
1685 }
1686 },
1687 };
1688 let reason = match self.forwarding.is_daemon_draining() {
1692 Ok(true) => RouteCloseReason::Restart,
1693 Ok(false) => RouteCloseReason::Crash,
1694 Err(err) => {
1695 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1696 RouteCloseReason::Crash
1697 }
1698 };
1699 (module_id, routes, reason, terminal)
1700 });
1701 let registrations = self.deregister_connection(connection_id, end_reason);
1702 let cleanup = if crash_closed.is_some() {
1703 self.forwarding.cleanup_connection_counted(connection_id)
1704 } else {
1705 self.forwarding
1711 .cleanup_connection(connection_id)
1712 .map(|released| crate::forwarding::ConnectionCleanup {
1713 released,
1714 abandoned_relays: 0,
1715 })
1716 };
1717 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1722 let abandoned = cleanup
1723 .as_ref()
1724 .map_or(0, |cleanup| cleanup.abandoned_relays);
1725 send_route_control_pushes(
1726 &self.forwarding,
1727 routes,
1728 ClientControlPush::RouteClosed {
1729 module_id,
1730 channels: Vec::new(),
1731 reason,
1732 drained: false,
1733 abandoned,
1734 excluded_subscriptions: 0,
1735 terminal: Some(terminal),
1736 },
1737 );
1738 }
1739 if let Ok(cleanup) = cleanup {
1740 self.emit_route_goodbyes(cleanup.released);
1741 }
1742 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1746 crate::supervise::notify_registration_release();
1747 self.capability_evaluator.wake_deadline_loop();
1748 self.refresh_capability_requirements();
1749 }
1750 self.supervisor.remove_spawn_subscribers(connection_id);
1751 self.hello_launch_nonces
1754 .lock()
1755 .unwrap_or_else(|poisoned| poisoned.into_inner())
1756 .forget(connection_id);
1757 self.scopes
1758 .write()
1759 .unwrap_or_else(|poisoned| poisoned.into_inner())
1760 .release_connection(connection_id);
1761 registrations
1762 }
1763
1764 pub(crate) fn handle_route_goodbye(
1765 &self,
1766 connection_id: ConnectionId,
1767 route_channel: u16,
1768 route_epoch: u32,
1769 ) -> Result<bool, RouterError> {
1770 debug!(
1771 connection_id = connection_id.get(),
1772 route_channel, route_epoch, "handling route GOODBYE"
1773 );
1774 let RouteRelease::Removed(released_route) = self
1775 .forwarding
1776 .release_client_route(connection_id, route_channel, route_epoch)
1777 .map_err(RouterError::Forwarding)?
1778 else {
1779 return Ok(false);
1780 };
1781 self.emit_route_goodbyes(vec![released_route]);
1782 Ok(true)
1783 }
1784
1785 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1786 for released in released_routes {
1787 let frame = match Frame::build_with_version(
1788 released.negotiated_ver,
1789 FrameType::Goodbye,
1790 control_flags(),
1791 released.channel,
1792 released.epoch,
1793 0,
1794 Vec::new(),
1795 ) {
1796 Ok(frame) => frame,
1797 Err(err) => {
1798 warn!(
1799 route_channel = released.channel,
1800 error = %err,
1801 "failed to build route GOODBYE frame"
1802 );
1803 continue;
1804 }
1805 };
1806 if !released.close_on_delivery_failure() {
1807 crate::forwarding::send_module_route_goodbye(
1808 &self.counters,
1809 &released.sink,
1810 frame,
1811 released.module_id.as_deref(),
1812 "client route released",
1813 );
1814 continue;
1815 }
1816 if let Err(err) = released.sink.try_send(frame) {
1817 warn!(
1818 target_connection_id = released.connection_id.get(),
1819 route_channel = released.channel,
1820 error = %err,
1821 "route GOODBYE was not delivered to client; closing target connection"
1822 );
1823 if self
1824 .forwarding
1825 .escalate_client_delivery_failure(
1826 released.connection_id,
1827 released.channel,
1828 released.epoch,
1829 CloseReason::new(
1830 "route_goodbye_delivery_failed",
1831 format!(
1832 "failed to enqueue route GOODBYE for channel {}: {err}",
1833 released.channel
1834 ),
1835 ),
1836 crate::forwarding::UndeliveredFrame {
1837 module_id: released.module_id.as_deref(),
1838 sink: &released.sink,
1839 },
1840 )
1841 .unwrap_or(false)
1842 {
1843 self.counters.increment_goodbye_relay_client_failed();
1844 }
1845 }
1846 }
1847 }
1848
1849 fn send_abandoned_route_bind_goodbye(
1861 &self,
1862 module_sink: &crate::FrameSink,
1863 negotiated_ver: u8,
1864 module_channel: u16,
1865 module_epoch: u32,
1866 ) {
1867 let frame = match Frame::build_with_version(
1868 negotiated_ver,
1869 FrameType::Goodbye,
1870 control_flags(),
1871 module_channel,
1872 module_epoch,
1873 0,
1874 Vec::new(),
1875 ) {
1876 Ok(frame) => frame,
1877 Err(err) => {
1878 warn!(
1879 route_channel = module_channel,
1880 error = %err,
1881 "failed to build GOODBYE for abandoned route.bind"
1882 );
1883 return;
1884 }
1885 };
1886 crate::forwarding::send_module_route_goodbye(
1887 &self.counters,
1888 module_sink,
1889 frame,
1890 None,
1891 "abandoned route.bind",
1892 );
1893 }
1894
1895 fn handle_hello(
1896 &self,
1897 connection_id: ConnectionId,
1898 sink: Option<crate::FrameSink>,
1899 frame: Frame,
1900 ) -> Result<Vec<Frame>, RouterError> {
1901 debug!(
1902 connection_id = connection_id.get(),
1903 corr = frame.header.corr,
1904 "handling HELLO"
1905 );
1906 if self
1910 .registry
1911 .get_module_by_connection(connection_id)
1912 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
1913 .is_some()
1914 {
1915 return Ok(vec![control_error_frame(
1916 &frame,
1917 "invalid_hello",
1918 "connection is already registered as a module",
1919 )?]);
1920 }
1921 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1922 Ok(value) => value,
1923 Err(err) => {
1924 return Ok(vec![control_error_frame(
1925 &frame,
1926 "invalid_hello",
1927 format!("malformed HELLO body: {err}"),
1928 )?])
1929 }
1930 };
1931 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1932 return Ok(vec![control_error_frame(
1933 &frame,
1934 "invalid_capability_grammar",
1935 err.to_string(),
1936 )?]);
1937 }
1938 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1939 return Ok(vec![control_error_frame(
1940 &frame,
1941 "invalid_manifest",
1942 err.to_string(),
1943 )?]);
1944 }
1945 if let Err(err) = validate_hello_event_declarations(&hello_value) {
1946 return Ok(vec![control_error_frame(
1947 &frame,
1948 "invalid_event_declaration",
1949 err.to_string(),
1950 )?]);
1951 }
1952 if let Some(provenance) = hello_value
1953 .get("manifest")
1954 .and_then(|manifest| manifest.get("provenance"))
1955 {
1956 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1957 return Ok(vec![control_error_frame(
1958 &frame,
1959 "invalid_manifest",
1960 format!("malformed manifest provenance: {err}"),
1961 )?]);
1962 }
1963 }
1964 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1965 Ok(hello) => hello,
1966 Err(err) => {
1967 return Ok(vec![control_error_frame(
1968 &frame,
1969 "invalid_hello",
1970 format!("malformed HELLO body: {err}"),
1971 )?])
1972 }
1973 };
1974
1975 if hello.protocol_ver != hello.manifest.protocol_ver {
1976 return Ok(vec![control_error_frame(
1977 &frame,
1978 "invalid_manifest",
1979 format!(
1980 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
1981 hello.protocol_ver, hello.manifest.protocol_ver
1982 ),
1983 )?]);
1984 }
1985
1986 if hello.manifest.module_id.trim().is_empty() {
1987 return Ok(vec![control_error_frame(
1988 &frame,
1989 "invalid_manifest",
1990 "manifest module_id must not be empty",
1991 )?]);
1992 }
1993
1994 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
1995 Ok(negotiated_ver) => negotiated_ver,
1996 Err(message) => {
1997 return Ok(vec![control_error_frame(
1998 &frame,
1999 "version_unsupported",
2000 message,
2001 )?])
2002 }
2003 };
2004
2005 let swap_admission = self
2013 .supervisor
2014 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2015 if swap_admission == SwapHelloAdmission::Refused {
2016 warn!(
2017 module_id = %hello.manifest.module_id,
2018 connection_id = connection_id.get(),
2019 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2020 );
2021 return Ok(vec![control_error_frame(
2022 &frame,
2023 "swap_token_invalid",
2024 format!(
2025 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2026 hello.manifest.module_id
2027 ),
2028 )?]);
2029 }
2030 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2031
2032 if let Some(rejection) = (!swap_candidate)
2040 .then(|| {
2041 self.supervisor.reserved_hello_rejection(
2042 &hello.manifest.module_id,
2043 hello.launch_nonce.as_deref(),
2044 )
2045 })
2046 .flatten()
2047 {
2048 let message = match rejection {
2049 ReservedHelloRejection::Exact { module_id } => format!(
2050 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2051 ),
2052 ReservedHelloRejection::Prefix {
2053 prefix,
2054 owner_module_id,
2055 } => format!(
2056 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2057 hello.manifest.module_id
2058 ),
2059 };
2060 return Ok(vec![control_error_frame(
2061 &frame,
2062 "reserved_module",
2063 message,
2064 )?]);
2065 }
2066
2067 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2068 &hello.manifest.module_id,
2069 hello.manifest.capabilities.as_ref(),
2070 );
2071 if let Some(refusal) = reserved_capability_refusals.first() {
2072 let capability = refusal.capability.clone();
2073 let bound_module = refusal.claimants[0].clone();
2074 log_duplicate_claim_events(reserved_capability_refusals);
2075 return Ok(vec![control_error_frame(
2076 &frame,
2077 "reserved_capability",
2078 format!(
2079 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2080 capability, bound_module, hello.manifest.module_id
2081 ),
2082 )?]);
2083 }
2084
2085 if self
2088 .forwarding
2089 .connection_has_client_routes(connection_id)
2090 .map_err(RouterError::Forwarding)?
2091 {
2092 return Ok(vec![control_error_frame(
2093 &frame,
2094 "invalid_hello",
2095 "connection has open client routes and cannot also register as a module",
2096 )?]);
2097 }
2098
2099 self.hello_launch_nonces
2104 .lock()
2105 .unwrap_or_else(|poisoned| poisoned.into_inner())
2106 .record(connection_id, hello.launch_nonce.as_deref());
2107 let control_ops = effective_module_control_ops(hello.control_ops);
2108 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2111 if swap_candidate {
2112 return self.register_swap_candidate(
2113 connection_id,
2114 sink,
2115 &frame,
2116 hello.manifest,
2117 negotiated_ver,
2118 control_ops,
2119 hello_ack,
2120 );
2121 }
2122 let registration = match self.registry.register_with_control_ops(
2123 hello.manifest,
2124 negotiated_ver,
2125 connection_id,
2126 control_ops,
2127 ) {
2128 Ok(registration) => registration,
2129 Err(RegistryError::DuplicateModuleId { module_id }) => {
2130 return Ok(vec![control_error_frame(
2131 &frame,
2132 "duplicate_module_id",
2133 format!(
2134 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2135 ),
2136 )?])
2137 }
2138 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2139 return Ok(vec![control_error_frame(
2140 &frame,
2141 "invalid_module_id",
2142 err.to_string(),
2143 )?])
2144 }
2145 Err(err) => {
2146 return Ok(vec![control_error_frame(
2147 &frame,
2148 "registry_error",
2149 err.to_string(),
2150 )?])
2151 }
2152 };
2153
2154 let reply = if let Some(sink) = sink {
2155 let concurrency = manifest_concurrency(®istration.manifest);
2171 if let Err(err) = self.forwarding.register_module_connection_acked(
2172 connection_id,
2173 registration.manifest.module_id.clone(),
2174 negotiated_ver,
2175 concurrency,
2176 sink,
2177 hello_ack,
2178 ) {
2179 if matches!(
2183 self.deregister_connection(
2184 connection_id,
2185 RegistrationEndReason::RegistrationFailed,
2186 ),
2187 Ok(r) if !r.is_empty()
2188 ) {
2189 crate::supervise::notify_registration_release();
2190 }
2191 return Ok(vec![control_error_frame(
2192 &frame,
2193 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2194 "invalid_hello"
2195 } else {
2196 forwarding_error_code(&err)
2197 },
2198 err.to_string(),
2199 )?]);
2200 }
2201 Vec::new()
2202 } else {
2203 vec![hello_ack]
2206 };
2207
2208 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2217 info!(
2218 module_id = %registration.manifest.module_id,
2219 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2220 );
2221 }
2222
2223 self.apply_registration_capabilities(®istration);
2224
2225 info!(
2226 module_id = %registration.manifest.module_id,
2227 module_version = %registration.manifest.module_version,
2228 negotiated_ver,
2229 routable_provider = manifest_provides_routable_role(®istration.manifest),
2230 connection_id = connection_id.get(),
2231 "module registered"
2232 );
2233
2234 Ok(reply)
2235 }
2236
2237 #[allow(clippy::too_many_arguments)]
2247 fn register_swap_candidate(
2248 &self,
2249 connection_id: ConnectionId,
2250 sink: Option<crate::FrameSink>,
2251 frame: &Frame,
2252 manifest: ModuleManifest,
2253 negotiated_ver: u8,
2254 control_ops: Vec<String>,
2255 hello_ack: Frame,
2256 ) -> Result<Vec<Frame>, RouterError> {
2257 let module_id = manifest.module_id.clone();
2258 let registration = match self.registry.register_candidate_with_control_ops(
2259 manifest,
2260 negotiated_ver,
2261 connection_id,
2262 control_ops,
2263 ) {
2264 Ok(registration) => registration,
2265 Err(RegistryError::DuplicateModuleId { module_id }) => {
2266 return Ok(vec![control_error_frame(
2267 frame,
2268 "duplicate_module_id",
2269 format!(
2270 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2271 ),
2272 )?])
2273 }
2274 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2275 return Ok(vec![control_error_frame(
2276 frame,
2277 "invalid_module_id",
2278 err.to_string(),
2279 )?])
2280 }
2281 Err(err) => {
2282 return Ok(vec![control_error_frame(
2283 frame,
2284 "registry_error",
2285 err.to_string(),
2286 )?])
2287 }
2288 };
2289 let reply = if let Some(sink) = sink {
2290 let concurrency = manifest_concurrency(®istration.manifest);
2294 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2295 connection_id,
2296 module_id.clone(),
2297 negotiated_ver,
2298 concurrency,
2299 sink,
2300 hello_ack,
2301 ) {
2302 if matches!(
2303 self.deregister_connection(
2304 connection_id,
2305 RegistrationEndReason::RegistrationFailed,
2306 ),
2307 Ok(r) if !r.is_empty()
2308 ) {
2309 crate::supervise::notify_registration_release();
2310 }
2311 return Ok(vec![control_error_frame(
2312 frame,
2313 forwarding_error_code(&err),
2314 err.to_string(),
2315 )?]);
2316 }
2317 Vec::new()
2318 } else {
2319 vec![hello_ack]
2320 };
2321 self.supervisor.mark_swap_candidate_admitted(&module_id);
2322 info!(
2323 module_id = %module_id,
2324 module_version = %registration.manifest.module_version,
2325 negotiated_ver,
2326 ready = registration.ready,
2327 connection_id = connection_id.get(),
2328 "swap candidate registered; not routable until cutover"
2329 );
2330 Ok(reply)
2331 }
2332
2333 fn build_hello_ack(
2334 &self,
2335 frame: &Frame,
2336 negotiated_ver: u8,
2337 module_id: &str,
2338 ) -> Result<Frame, RouterError> {
2339 let ack = ModuleHelloAckBody {
2340 negotiated_ver,
2341 subc_ops: module_subc_ops(),
2342 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2343 storage: self
2344 .storage_config
2345 .as_ref()
2346 .map(|cfg| cfg.descriptor_for(module_id)),
2347 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2348 };
2349 let body = serde_json::to_vec(&ack).map_err(|err| {
2350 RouterError::backend(
2351 0,
2352 frame.header.corr,
2353 format!("failed to encode HELLO_ACK: {err}"),
2354 )
2355 })?;
2356
2357 Frame::build_with_version(
2358 negotiated_ver,
2359 FrameType::HelloAck,
2360 control_flags(),
2361 0,
2362 0,
2363 frame.header.corr,
2364 body,
2365 )
2366 .map_err(RouterError::FrameBuild)
2367 }
2368
2369 async fn handle_client_control_request(
2370 &self,
2371 ctx: &RouteCtx,
2372 frame: Frame,
2373 request: ClientControlRequest,
2374 ) -> Result<Vec<Frame>, RouterError> {
2375 match request {
2376 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2377 ClientControlRequest::CatalogList { module_id } => {
2378 self.handle_catalog_list(frame, module_id)
2379 }
2380 ClientControlRequest::RouteOpen {
2381 target,
2382 identity,
2383 consumer_identity,
2384 consumer_capabilities,
2385 role_versions,
2386 admission_facts,
2387 scope,
2388 } => {
2389 self.handle_route_open(
2390 ctx,
2391 frame,
2392 RouteOpenRequest {
2393 target,
2394 identity,
2395 consumer_identity,
2396 consumer_capabilities,
2397 role_versions,
2398 admission_facts,
2399 scope,
2400 },
2401 )
2402 .await
2403 }
2404 ClientControlRequest::RoutePoll {
2405 route_channel,
2406 route_epoch,
2407 kind,
2408 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2409 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2410 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2411 self.handle_supervisor_spawn_snapshot(frame)
2412 }
2413 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2414 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2415 }
2416 ClientControlRequest::SupervisorRestart {
2417 module_id,
2418 drain_timeout_ms,
2419 } => {
2420 self.log_supervisor_request_received(
2421 ctx,
2422 frame.header.corr,
2423 ops::SUPERVISOR_RESTART,
2424 Some(&module_id),
2425 None,
2426 )?;
2427 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2428 .await
2429 }
2430 ClientControlRequest::SupervisorSwap {
2431 module_id,
2432 ready_timeout_ms,
2433 } => {
2434 self.log_supervisor_request_received(
2435 ctx,
2436 frame.header.corr,
2437 ops::SUPERVISOR_SWAP,
2438 Some(&module_id),
2439 None,
2440 )?;
2441 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2442 .await
2443 }
2444 ClientControlRequest::SupervisorReload { module_id } => {
2445 self.log_supervisor_request_received(
2446 ctx,
2447 frame.header.corr,
2448 ops::SUPERVISOR_RELOAD,
2449 Some(&module_id),
2450 None,
2451 )?;
2452 self.handle_supervisor_reload(frame, module_id).await
2453 }
2454 ClientControlRequest::SupervisorRescan { preview } => {
2455 if !preview {
2456 self.log_supervisor_request_received(
2457 ctx,
2458 frame.header.corr,
2459 ops::SUPERVISOR_RESCAN,
2460 None,
2461 None,
2462 )?;
2463 }
2464 self.handle_supervisor_rescan(frame, preview).await
2465 }
2466 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2467 self.log_supervisor_request_received(
2468 ctx,
2469 frame.header.corr,
2470 ops::SUPERVISOR_RELEASE_RESERVED,
2471 Some(&module_id),
2472 None,
2473 )?;
2474 self.handle_supervisor_release_reserved(frame, module_id)
2475 .await
2476 }
2477 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2478 self.log_supervisor_request_received(
2479 ctx,
2480 frame.header.corr,
2481 ops::SUPERVISOR_SET_ENABLED,
2482 Some(&module_id),
2483 Some(enabled),
2484 )?;
2485 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2486 .await
2487 }
2488 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2489 self.handle_supervisor_health_probe(frame, module_id).await
2490 }
2491 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2492 ClientControlRequest::SupervisorRoutes { module_id } => {
2493 self.handle_supervisor_routes(frame, module_id)
2494 }
2495 ClientControlRequest::SupervisorProvenance { module_id } => {
2496 self.handle_supervisor_provenance(frame, module_id).await
2497 }
2498 ClientControlRequest::SupervisorStderrTail {
2499 module_id,
2500 max_lines,
2501 max_bytes,
2502 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2503 ClientControlRequest::SupervisorTerminals { module_id } => {
2504 self.handle_supervisor_terminals(frame, module_id).await
2505 }
2506 }
2507 }
2508
2509 fn log_supervisor_request_received(
2510 &self,
2511 ctx: &RouteCtx,
2512 corr: u64,
2513 op: &'static str,
2514 module_id: Option<&str>,
2515 enabled: Option<bool>,
2516 ) -> Result<(), RouterError> {
2517 let caller = self
2518 .registry
2519 .get_module_by_connection(ctx.connection_id)
2520 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2521 .map(|registration| Principal::Reserved {
2522 module_id: registration.manifest.module_id,
2523 })
2524 .unwrap_or(Principal::Direct);
2525 let caller = principal_label(&caller);
2526 info!(
2529 target: "control",
2530 op,
2531 module_id,
2532 enabled,
2533 connection_id = ctx.connection_id.get(),
2534 caller = %caller,
2535 "supervisor request received"
2536 );
2537 Ok(())
2538 }
2539
2540 fn handle_module_control_request(
2541 &self,
2542 connection_id: ConnectionId,
2543 frame: Frame,
2544 request: ModuleControlRequestFromModule,
2545 ) -> Result<Vec<Frame>, RouterError> {
2546 match request {
2547 ModuleControlRequestFromModule::CatalogUpdate {
2548 provides,
2549 capabilities,
2550 ready,
2551 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2552 ModuleControlRequestFromModule::LiveRoots {} => {
2553 let registered = self
2554 .registry
2555 .get_module_by_connection(connection_id)
2556 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2557 let Some(registration) = registered else {
2558 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2559 };
2560 let response = self
2561 .forwarding
2562 .live_roots(®istration.manifest.module_id)
2563 .map_err(RouterError::Forwarding)?;
2564 Ok(vec![control_response_body_frame(
2565 &frame,
2566 &response,
2567 "ModuleControlResponseToModule::LiveRoots",
2568 )?])
2569 }
2570 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2571 self.handle_scope_sync(connection_id, frame, generation, scopes)
2572 }
2573 ModuleControlRequestFromModule::ScopeApply {
2574 generation,
2575 upsert,
2576 end,
2577 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2578 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2579 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2580 }
2581 }
2582 }
2583
2584 fn handle_operator_confirm(
2585 &self,
2586 ctx: &RouteCtx,
2587 frame: Frame,
2588 ) -> Result<Vec<Frame>, RouterError> {
2589 use crate::operator_confirm::{audit, Outcome};
2590 let registration = self
2591 .registry
2592 .get_module_by_connection(ctx.connection_id)
2593 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2594 let Some(registration) = registration else {
2595 let outcome = Outcome::refusal("not_registered");
2596 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2597 return Ok(vec![outcome.frame(&frame)]);
2598 };
2599 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2600 Ok(request) => request,
2601 Err(_) => {
2602 let outcome = Outcome::refusal("invalid_control_body");
2603 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2604 return Ok(vec![outcome.frame(&frame)]);
2605 }
2606 };
2607 let module_id = registration.manifest.module_id;
2608 let nonce = self
2612 .hello_launch_nonces
2613 .lock()
2614 .unwrap_or_else(|p| p.into_inner())
2615 .nonce(ctx.connection_id)
2616 .map(str::to_owned);
2617 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2618 self.supervisor
2619 .spawned_consumer_authorized(&module_id, nonce)
2620 });
2621 let confirms = self.forwarding.operator_confirms();
2622 self.forwarding
2623 .with_operator_route(
2624 ctx.connection_id,
2625 request.route_channel,
2626 request.route_epoch,
2627 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
2628 )
2629 .map_err(RouterError::Forwarding)
2630 }
2631
2632 fn handle_scope_sync(
2636 &self,
2637 connection_id: ConnectionId,
2638 frame: Frame,
2639 generation: u64,
2640 scopes: Vec<ScopeRecord>,
2641 ) -> Result<Vec<Frame>, RouterError> {
2642 self.handle_scope_change(connection_id, frame, generation, scopes, None)
2643 }
2644
2645 fn handle_scope_change(
2646 &self,
2647 connection_id: ConnectionId,
2648 frame: Frame,
2649 generation: u64,
2650 scopes: Vec<ScopeRecord>,
2651 end: Option<Vec<ScopeEnd>>,
2652 ) -> Result<Vec<Frame>, RouterError> {
2653 let is_apply = end.is_some();
2654 let Some(registration) = self
2655 .registry
2656 .get_module_by_connection(connection_id)
2657 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2658 else {
2659 return Ok(vec![control_error_frame(
2660 &frame,
2661 "not_registered",
2662 "scope.sync and scope.apply require an active module registration owned by this connection",
2663 )?]);
2664 };
2665 let owner = registration.manifest.module_id;
2666 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2667 let is_current_launch = |connection: ConnectionId| {
2668 self.hello_launch_nonces
2669 .lock()
2670 .unwrap_or_else(|poisoned| poisoned.into_inner())
2671 .presented(connection, current_nonce.as_deref())
2672 };
2673 let mut table = self
2678 .scopes
2679 .write()
2680 .unwrap_or_else(|poisoned| poisoned.into_inner());
2681 let outcome = match end {
2682 Some(end) => table.apply(
2683 &owner,
2684 connection_id,
2685 is_current_launch,
2686 generation,
2687 scopes,
2688 end,
2689 ),
2690 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
2691 };
2692 let drained = match &outcome {
2693 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
2694 Err(_) => Vec::new(),
2695 };
2696 let has_deadlines = outcome.is_ok() && table.has_deadlines();
2697 drop(table);
2698 if has_deadlines {
2699 self.scope_deadline_added.notify_one();
2700 }
2701 match outcome {
2702 Ok(applied) => {
2703 let counts = ScopeOutcomeCounts::of(&applied.results);
2704 info!(
2705 owner = %owner,
2706 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2707 generation,
2708 records = applied.results.len(),
2709 created = counts.created,
2710 replaced = counts.replaced,
2711 updated = counts.updated,
2712 unchanged = counts.unchanged,
2713 refused = counts.refused,
2714 ended = applied.ended.len(),
2715 tag_changes = applied.tag_changes.len(),
2716 routes_closed = drained.len(),
2717 "scope change accepted"
2718 );
2719 for refused in applied
2725 .results
2726 .iter()
2727 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2728 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2729 {
2730 warn!(
2731 owner = %owner,
2732 generation,
2733 scope_ref = %refused.scope_ref,
2734 scope_epoch = refused.scope_epoch,
2735 code = refused.code.as_deref().unwrap_or(""),
2736 "scope record refused"
2737 );
2738 }
2739 self.close_scope_drained_routes(drained);
2740 let response = if is_apply {
2741 ModuleControlResponseToModule::ScopeApply {
2742 generation,
2743 results: applied.results,
2744 end_results: applied.end_results,
2745 ended: applied.ended,
2746 }
2747 } else {
2748 ModuleControlResponseToModule::ScopeSync {
2749 generation,
2750 results: applied.results,
2751 ended: applied.ended,
2752 }
2753 };
2754 Ok(vec![control_response_body_frame(
2755 &frame,
2756 &response,
2757 "scope change response",
2758 )?])
2759 }
2760 Err(refusal) => {
2761 info!(
2762 owner = %owner,
2763 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2764 generation,
2765 code = refusal.code,
2766 "scope change refused"
2767 );
2768 Ok(vec![control_error_frame(
2769 &frame,
2770 refusal.code,
2771 refusal.message,
2772 )?])
2773 }
2774 }
2775 }
2776
2777 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2783 if drained.is_empty() {
2784 return;
2785 }
2786 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2787 BTreeMap::new();
2788 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2789 for route in drained {
2790 warn!(
2791 module_id = %route.module_id,
2792 reason = ?route.reason,
2793 client_connection_id = route.client.connection_id.get(),
2794 route_channel = route.client.channel,
2795 "closing route because its scope changed"
2796 );
2797 pushes
2798 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2799 .or_insert_with(|| (route.reason, Vec::new()))
2800 .1
2801 .push(EndpointRoute {
2802 goodbye_target: route.client.clone(),
2803 principal: Principal::Unverified,
2804 bound_at: Instant::now(),
2805 draining: false,
2806 drain_reason: None,
2807 });
2808 goodbyes.push(route.module);
2809 goodbyes.push(route.client);
2810 }
2811 for ((module_id, _), (reason, routes)) in pushes {
2812 send_route_control_pushes(
2813 &self.forwarding,
2814 routes,
2815 ClientControlPush::RouteClosed {
2816 module_id,
2817 channels: Vec::new(),
2818 reason,
2819 drained: false,
2820 abandoned: 0,
2821 excluded_subscriptions: 0,
2822 terminal: Some(false),
2823 },
2824 );
2825 }
2826 self.emit_route_goodbyes(goodbyes);
2827 }
2828
2829 fn handle_scope_describe(
2832 &self,
2833 connection_id: ConnectionId,
2834 frame: Frame,
2835 owner: Principal,
2836 scope_ref: String,
2837 ) -> Result<Vec<Frame>, RouterError> {
2838 let registered = self
2839 .registry
2840 .get_module_by_connection(connection_id)
2841 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2842 if registered.is_none() {
2843 return Ok(vec![control_error_frame(
2844 &frame,
2845 "not_registered",
2846 "scope.describe requires an active module registration owned by this connection",
2847 )?]);
2848 }
2849 let description = self
2850 .scopes
2851 .read()
2852 .unwrap_or_else(|poisoned| poisoned.into_inner())
2853 .describe(&owner, &scope_ref);
2854 let owner_configured = match &owner {
2855 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
2860 _ => false,
2861 };
2862 let response = ModuleControlResponseToModule::ScopeDescribe {
2863 status: description.status,
2864 scope_epoch: description.scope_epoch,
2865 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2866 owner_synced: description.owner_synced,
2867 owner_configured,
2868 scope: description.stamp,
2869 };
2870 Ok(vec![control_response_body_frame(
2871 &frame,
2872 &response,
2873 "ModuleControlResponseToModule::ScopeDescribe",
2874 )?])
2875 }
2876
2877 fn handle_catalog_update(
2878 &self,
2879 connection_id: ConnectionId,
2880 frame: Frame,
2881 provides: Vec<ProviderRole>,
2882 capabilities: Option<CapabilityDeclarations>,
2883 ready: Option<bool>,
2884 ) -> Result<Vec<Frame>, RouterError> {
2885 self.refresh_capability_requirements();
2886 let Some(registration) = self
2887 .registry
2888 .get_module_by_connection(connection_id)
2889 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2890 else {
2891 return Ok(vec![control_error_frame(
2892 &frame,
2893 "not_registered",
2894 "catalog.update requires an active module registration owned by this connection",
2895 )?]);
2896 };
2897
2898 if let Some(message) =
2899 catalog_update_frozen_field_message(®istration.manifest, &provides)
2900 {
2901 return Ok(vec![control_error_frame(
2902 &frame,
2903 "catalog_update_frozen_field",
2904 message,
2905 )?]);
2906 }
2907
2908 let mut candidate = registration.manifest.clone();
2909 candidate.provides = provides.clone();
2910 candidate.capabilities = capabilities
2911 .clone()
2912 .or_else(|| registration.manifest.capabilities.clone());
2913 if let Err(err) = candidate.validate_capability_grammar() {
2914 return Ok(vec![control_error_frame(
2915 &frame,
2916 "invalid_capability_grammar",
2917 err.to_string(),
2918 )?]);
2919 }
2920
2921 let mut conflicts = self
2924 .capability_evaluator
2925 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
2926 if let Some(conflict) = conflicts.first() {
2927 let message = format!(
2928 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2929 conflict.capability, conflict.claimants[0], candidate.module_id
2930 );
2931 for conflict in &mut conflicts {
2932 conflict.source = DuplicateClaimSource::CatalogUpdate;
2933 }
2934 log_duplicate_claim_events(conflicts);
2935 return Ok(vec![control_error_frame(
2936 &frame,
2937 "reserved_capability",
2938 message,
2939 )?]);
2940 }
2941
2942 let updated = self
2943 .registry
2944 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2945 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2946 if updated.is_none() {
2947 return Ok(vec![control_error_frame(
2948 &frame,
2949 "not_registered",
2950 "catalog.update requires an active module registration owned by this connection",
2951 )?]);
2952 }
2953 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2954 log_duplicate_claim_events(
2955 self.capability_evaluator
2956 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2957 );
2958 }
2959 if capability_census_trigger(
2960 registration.manifest.capabilities.as_ref(),
2961 updated
2962 .as_ref()
2963 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2964 ) {
2965 self.enforce_capability_denies();
2966 }
2967 self.refresh_capability_requirements();
2968
2969 let response = ModuleControlResponseToModule::CatalogUpdate {};
2970 control_response_body_frame(
2971 &frame,
2972 &response,
2973 "ModuleControlResponseToModule::CatalogUpdate",
2974 )
2975 .map(|frame| vec![frame])
2976 }
2977
2978 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2979 self.refresh_capability_requirements();
2980 let mut counters = self.counters.snapshot();
2986 if let (Ok((connections_with_routes, max)), Some(obj)) = (
2987 self.forwarding.client_route_concentration(),
2988 counters.as_object_mut(),
2989 ) {
2990 obj.insert(
2991 "client_connections_with_routes".into(),
2992 connections_with_routes.into(),
2993 );
2994 obj.insert("max_routes_on_one_connection".into(), max.into());
2995 }
2996 if let (Some(open_breakers), Some(obj)) = (
3002 self.route_bind_breakers.open_snapshot(),
3003 counters.as_object_mut(),
3004 ) {
3005 obj.insert("route_bind_breakers_open".into(), open_breakers);
3006 }
3007 let response = ClientControlResponse::ServerDescribe {
3008 protocol_ver: PROTOCOL_VERSION,
3009 subc_ops: subc_ops(),
3010 capabilities: self.subc_capabilities.as_ref().to_vec(),
3011 connected_clients: self.connected_clients.count(),
3012 counters: Some(counters),
3013 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3014 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3015 capability_requirements: self.capability_requirement_statuses(),
3016 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3017 };
3018 Ok(vec![control_response_body_frame(
3019 &frame,
3020 &response,
3021 "ClientControlResponse::ServerDescribe",
3022 )?])
3023 }
3024
3025 fn handle_catalog_list(
3026 &self,
3027 frame: Frame,
3028 module_id: Option<String>,
3029 ) -> Result<Vec<Frame>, RouterError> {
3030 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3031 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3032 })?;
3033 let entries = modules
3034 .into_iter()
3035 .filter(|registration| {
3036 module_id
3037 .as_deref()
3038 .map(|wanted| registration.manifest.module_id == wanted)
3039 .unwrap_or(true)
3040 })
3041 .map(|registration| {
3042 let not_ready = self.not_ready_reason(®istration);
3043 let roles = registration.manifest.provides;
3044 CatalogEntry::new(
3045 registration.manifest.module_id,
3046 roles,
3047 registration.control_ops,
3048 )
3049 .with_ready(not_ready.is_none())
3050 .with_not_ready(not_ready)
3051 .with_module_version(Some(registration.manifest.module_version))
3052 .with_capabilities(registration.manifest.capabilities)
3053 .with_self_signals(registration.manifest.self_signals)
3054 })
3055 .collect();
3056 let response = ClientControlResponse::CatalogList {
3057 generation,
3058 modules: entries,
3059 subc_ops: subc_ops(),
3060 };
3061 Ok(vec![control_response_body_frame(
3062 &frame,
3063 &response,
3064 "ClientControlResponse::CatalogList",
3065 )?])
3066 }
3067
3068 fn route_open_principal(
3069 &self,
3070 frame: &Frame,
3071 consumer_identity: Option<ConsumerIdentity>,
3072 ) -> Result<Result<Principal, Frame>, RouterError> {
3073 let Some(consumer_identity) = consumer_identity else {
3074 return Ok(Ok(Principal::Direct));
3075 };
3076
3077 if self.supervisor.spawned_consumer_authorized(
3078 &consumer_identity.module_id,
3079 &consumer_identity.launch_nonce,
3080 ) {
3081 return Ok(Ok(Principal::Reserved {
3082 module_id: consumer_identity.module_id,
3083 }));
3084 }
3085
3086 Ok(Err(control_error_frame(
3087 frame,
3088 "bad_consumer_identity",
3089 format!(
3090 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3091 consumer_identity.module_id
3092 ),
3093 )?))
3094 }
3095
3096 fn route_open_refusal_frame(
3103 &self,
3104 ctx: &RouteCtx,
3105 frame: &Frame,
3106 module_id: &str,
3107 reason: &'static str,
3108 code: &'static str,
3109 message: impl Into<String>,
3110 ) -> Result<Frame, RouterError> {
3111 self.observe_route_open_refusal(ctx, module_id, reason, code);
3112 control_error_frame(frame, code, message.into())
3113 }
3114
3115 fn route_open_breaker_refusal_frame(
3133 &self,
3134 ctx: &RouteCtx,
3135 frame: &Frame,
3136 module_id: &str,
3137 consecutive_timeouts: u32,
3138 retry_in: Duration,
3139 probe_in_flight: bool,
3140 ) -> Result<Frame, RouterError> {
3141 self.counters
3142 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3143 debug!(
3144 target: "control",
3145 code = "module_timeout",
3146 module_id = ?module_id,
3147 connection_id = ctx.connection_id.get(),
3148 consecutive_timeouts,
3149 retry_in_ms = retry_in.as_millis() as u64,
3150 probe_in_flight,
3151 "route.open refused by open bind-relay breaker"
3152 );
3153 let detail = if probe_in_flight {
3160 "one test route is already being tried; retry once it settles".to_string()
3161 } else {
3162 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3163 };
3164 control_error_frame(
3165 frame,
3166 "module_timeout",
3167 format!(
3168 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3169 timed out in a row); {detail}; its established routes are unaffected"
3170 ),
3171 )
3172 }
3173
3174 fn observe_route_open_refusal(
3187 &self,
3188 ctx: &RouteCtx,
3189 module_id: &str,
3190 reason: &'static str,
3191 code: &'static str,
3192 ) {
3193 self.counters.increment_route_open_refused(code);
3194 info!(
3195 target: "control",
3196 code,
3197 reason,
3198 module_id = ?module_id,
3199 connection_id = ctx.connection_id.get(),
3200 "route.open refused"
3201 );
3202 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3203 self.route_outages.record_not_serving(module_id, reason);
3204 }
3205 }
3206
3207 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3252 self.route_outages.record_accepted(module_id);
3253 self.counters.increment_route_open_accepted(principal);
3254 info!(
3255 target: "control",
3256 principal,
3257 module_id,
3258 connection_id = ctx.connection_id.get(),
3259 "route.open accepted"
3260 );
3261 }
3262
3263 fn supervised_absent_route_open_refusal_frame(
3264 &self,
3265 ctx: &RouteCtx,
3266 frame: &Frame,
3267 module_id: &str,
3268 code: &'static str,
3269 status: &crate::supervise::ModuleStatus,
3270 ) -> Result<Frame, RouterError> {
3271 self.counters.increment_route_open_refused(code);
3272 info!(
3273 target: "control",
3274 code,
3275 reason = "supervised_not_registered",
3276 module_id = ?module_id,
3277 connection_id = ctx.connection_id.get(),
3278 state = %status.state,
3279 enabled = status.enabled,
3280 live = status.live,
3281 "route.open refused"
3282 );
3283 self.route_outages
3287 .record_not_serving(module_id, "supervised_not_registered");
3288 control_error_frame(
3289 frame,
3290 code,
3291 format!(
3292 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3293 status.state, status.enabled, status.live
3294 ),
3295 )
3296 }
3297
3298 async fn handle_route_open(
3299 &self,
3300 ctx: &RouteCtx,
3301 frame: Frame,
3302 request: RouteOpenRequest,
3303 ) -> Result<Vec<Frame>, RouterError> {
3304 let RouteOpenRequest {
3305 target,
3306 mut identity,
3307 consumer_identity,
3308 consumer_capabilities,
3309 role_versions,
3310 admission_facts,
3311 scope,
3312 } = request;
3313 let target_module_id = target_module_id(&target).to_string();
3314 if self
3315 .registry
3316 .get_module_by_connection(ctx.connection_id)
3317 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3318 .is_some()
3319 {
3320 return Ok(vec![control_error_frame(
3321 &frame,
3322 "invalid_request",
3323 "module connections cannot open client routes",
3324 )?]);
3325 }
3326 debug!(
3327 connection_id = ctx.connection_id.get(),
3328 corr = frame.header.corr,
3329 module_id = %target_module_id,
3330 "handling route.open"
3331 );
3332
3333 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3339 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3340 self.observe_route_open_refusal(
3341 ctx,
3342 &target_module_id,
3343 "invalid_role_versions",
3344 error_codes::INVALID_REQUEST,
3345 );
3346 return Ok(vec![control_error_body_frame(
3347 &frame,
3348 ErrorBody {
3349 code: error_codes::INVALID_REQUEST.to_string(),
3350 message: error.to_string(),
3351 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3352 },
3353 )?]);
3354 }
3355
3356 let Some(registration) = self
3375 .registry
3376 .get_module(&target_module_id)
3377 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3378 else {
3379 if let Some((status, warming)) =
3380 self.supervisor_status(&target_module_id, frame.header.corr)?
3381 {
3382 if status.protocol == ModuleProtocol::None {
3389 return Ok(vec![self.route_open_refusal_frame(
3390 ctx,
3391 &frame,
3392 &target_module_id,
3393 "protocol_none",
3394 error_codes::MODULE_NO_PROTOCOL,
3395 format!(
3396 "module_id '{target_module_id}' is declared protocol: none; \
3397 it speaks no subc wire and serves no routes"
3398 ),
3399 )?]);
3400 }
3401 let code = if warming {
3402 "module_warming"
3403 } else {
3404 "target_unavailable"
3405 };
3406 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3407 ctx,
3408 &frame,
3409 &target_module_id,
3410 code,
3411 &status,
3412 )?]);
3413 }
3414 if let Some(removed_ago_ms) =
3415 self.supervisor.removal_tombstone_age_ms(&target_module_id)
3416 {
3417 return Ok(vec![self.route_open_refusal_frame(
3418 ctx,
3419 &frame,
3420 &target_module_id,
3421 "removed",
3422 error_codes::MODULE_REMOVED,
3423 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3424 )?]);
3425 }
3426 return Ok(vec![self.route_open_refusal_frame(
3427 ctx,
3428 &frame,
3429 &target_module_id,
3430 "not_registered",
3431 error_codes::UNKNOWN_MODULE,
3432 format!("module_id '{target_module_id}' is not registered"),
3433 )?]);
3434 };
3435
3436 if !registration.ready {
3440 self.counters
3441 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3442 info!(
3443 target: "control",
3444 code = error_codes::MODULE_WARMING,
3445 module_id = ?target_module_id,
3446 connection_id = ctx.connection_id.get(),
3447 reason = "declared_not_ready",
3448 "route.open refused"
3449 );
3450 self.route_outages
3453 .record_not_serving(&target_module_id, "declared_not_ready");
3454 return Ok(vec![control_error_body_frame(
3455 &frame,
3456 ErrorBody {
3457 code: error_codes::MODULE_WARMING.to_string(),
3458 message: format!(
3459 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3460 ),
3461 detail: Some(serde_json::json!({
3462 "reason": "declared_not_ready"
3463 })),
3464 },
3465 )?]);
3466 }
3467
3468 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3487 self.counters
3488 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3489 info!(
3490 target: "control",
3491 code = error_codes::MODULE_WARMING,
3492 module_id = ?target_module_id,
3493 connection_id = ctx.connection_id.get(),
3494 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3495 capability = %capability,
3496 "route.open refused"
3497 );
3498 return Ok(vec![control_error_body_frame(
3499 &frame,
3500 ErrorBody {
3501 code: error_codes::MODULE_WARMING.to_string(),
3502 message: format!(
3503 "module_id '{target_module_id}' requires capability '{capability}', \
3504 which no registered module provides; retry"
3505 ),
3506 detail: Some(serde_json::json!({
3507 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3508 "capability": capability,
3509 })),
3510 },
3511 )?]);
3512 }
3513
3514 if !target_has_required_role(&target, ®istration.manifest.provides) {
3515 return Ok(vec![self.route_open_refusal_frame(
3516 ctx,
3517 &frame,
3518 &target_module_id,
3519 "role_not_provided",
3520 "target_unavailable",
3521 format!("module_id '{target_module_id}' does not provide the requested target"),
3522 )?]);
3523 }
3524
3525 if registration.state != ChannelState::Active {
3526 return Ok(vec![self.route_open_refusal_frame(
3527 ctx,
3528 &frame,
3529 &target_module_id,
3530 "registration_not_active",
3531 "target_unavailable",
3532 format!("module_id '{target_module_id}' is not active"),
3533 )?]);
3534 }
3535
3536 if self
3537 .forwarding
3538 .module_is_draining(&target_module_id)
3539 .map_err(RouterError::Forwarding)?
3540 {
3541 return Ok(vec![self.route_open_refusal_frame(
3542 ctx,
3543 &frame,
3544 &target_module_id,
3545 "reloading",
3546 "module_reloading",
3547 format!("module_id '{target_module_id}' is reloading"),
3548 )?]);
3549 }
3550
3551 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3552 process_liveness.process_live(&target_module_id) == Some(false)
3553 }) {
3554 if process_liveness.process_replacing(&target_module_id) {
3561 return Ok(vec![self.route_open_refusal_frame(
3562 ctx,
3563 &frame,
3564 &target_module_id,
3565 "reloading",
3566 "module_reloading",
3567 format!("module_id '{target_module_id}' is reloading"),
3568 )?]);
3569 }
3570 return Ok(vec![self.route_open_refusal_frame(
3571 ctx,
3572 &frame,
3573 &target_module_id,
3574 "supervisor_not_live",
3575 "target_unavailable",
3576 format!("module_id '{target_module_id}' is not live"),
3577 )?]);
3578 }
3579
3580 if !self
3581 .forwarding
3582 .has_live_module_connection(&target_module_id)
3583 .map_err(RouterError::Forwarding)?
3584 {
3585 return Ok(vec![self.route_open_refusal_frame(
3586 ctx,
3587 &frame,
3588 &target_module_id,
3589 "no_forwarding_connection",
3590 "target_unavailable",
3591 format!("module_id '{target_module_id}' has no live forwarding connection"),
3592 )?]);
3593 }
3594
3595 if let Some(error) =
3596 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3597 {
3598 self.observe_route_open_refusal(
3599 ctx,
3600 &target_module_id,
3601 "op_not_allowed",
3602 "op_not_allowed",
3603 );
3604 return Ok(vec![error]);
3605 }
3606
3607 let principal = match self.route_open_principal(&frame, consumer_identity)? {
3608 Ok(principal) => principal,
3609 Err(error) => {
3610 self.observe_route_open_refusal(
3611 ctx,
3612 &target_module_id,
3613 "bad_consumer_identity",
3614 "bad_consumer_identity",
3615 );
3616 return Ok(vec![error]);
3617 }
3618 };
3619
3620 if let Principal::Reserved {
3624 module_id: opening_module_id,
3625 } = &principal
3626 {
3627 if let Some(opening_registration) = self
3628 .registry
3629 .get_module(opening_module_id)
3630 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3631 {
3632 if let Some(capability) =
3633 denied_capability(&opening_registration.manifest, ®istration.manifest)
3634 {
3635 warn!(
3636 opening_module_id,
3637 target_module_id,
3638 capability,
3639 "refusing route.open because an attested capability deny edge matches"
3640 );
3641 return Ok(vec![self.route_open_refusal_frame(
3642 ctx,
3643 &frame,
3644 &target_module_id,
3645 "capability_deny_edge",
3646 "capability_forbidden",
3647 format!(
3648 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3649 ),
3650 )?]);
3651 }
3652 }
3653 }
3654
3655 if admission_facts.is_some() {
3656 let carrier_matches = matches!(
3657 &principal,
3658 Principal::Reserved { module_id }
3659 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3660 );
3661 if !carrier_matches {
3662 return Ok(vec![self.route_open_refusal_frame(
3663 ctx,
3664 &frame,
3665 &target_module_id,
3666 "admission_facts_carrier_not_permitted",
3667 "admission_facts_not_permitted",
3668 "admission facts may only be carried by the configured reserved module",
3669 )?]);
3670 }
3671
3672 let target_allowed = self
3673 .admission_facts_targets
3674 .as_ref()
3675 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3676 if !target_allowed {
3677 return Ok(vec![self.route_open_refusal_frame(
3678 ctx,
3679 &frame,
3680 &target_module_id,
3681 "admission_facts_target_not_listed",
3682 "admission_facts_target_not_allowed",
3683 format!(
3684 "admission facts are not permitted for target module_id '{target_module_id}'"
3685 ),
3686 )?]);
3687 }
3688
3689 }
3693
3694 let (bound_scope, scope_stamp) = match scope {
3700 None => (None, None),
3701 Some(selector) => {
3702 let owner_configured = match &selector.owner {
3703 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3709 _ => false,
3710 };
3711 let admitted = self
3712 .scopes
3713 .read()
3714 .unwrap_or_else(|poisoned| poisoned.into_inner())
3715 .admit(&principal, &target_module_id, &selector, owner_configured)
3716 .and_then(|admission| {
3717 crate::scopes::check_target_flow_support(
3718 &admission.stamp,
3719 &target_module_id,
3720 registration.manifest.capabilities.as_ref(),
3721 )?;
3722 crate::scopes::check_target_agent_run_support(
3723 &admission.stamp,
3724 &target_module_id,
3725 registration.manifest.capabilities.as_ref(),
3726 )?;
3727 Ok(admission)
3728 });
3729 match admitted {
3730 Ok(admission) => (
3731 Some(BoundScope {
3732 owner: admission.owner,
3733 scope_ref: admission.stamp.scope_ref.clone(),
3734 tag: admission.tag,
3735 }),
3736 Some(admission.stamp),
3737 ),
3738 Err(refusal) => {
3739 return Ok(vec![self.route_open_refusal_frame(
3740 ctx,
3741 &frame,
3742 &target_module_id,
3743 refusal.code,
3744 refusal.code,
3745 refusal.message,
3746 )?]);
3747 }
3748 }
3749 }
3750 };
3751
3752 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3774 Ok(project_root) => project_root,
3775 Err(err) => {
3776 return Ok(vec![control_error_frame(
3777 &frame,
3778 "invalid_project_root",
3779 err.to_string(),
3780 )?])
3781 }
3782 };
3783 identity.project_root = project_root.as_path().to_path_buf();
3784
3785 let _concurrency_guard = match self
3796 .route_bind_concurrency
3797 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3798 {
3799 Ok(guard) => guard,
3800 Err(in_flight) => {
3801 return Ok(vec![self.route_open_target_capacity_refusal(
3802 ctx,
3803 &frame,
3804 &target_module_id,
3805 in_flight,
3806 )?]);
3807 }
3808 };
3809
3810 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3813 RouteBindAdmission::Admitted { guard, probe } => {
3814 if probe {
3815 info!(
3816 module_id = %target_module_id,
3817 connection_id = ctx.connection_id.get(),
3818 "route.bind breaker half-open: admitting one probe"
3819 );
3820 }
3821 guard
3822 }
3823 RouteBindAdmission::Refused {
3824 consecutive_timeouts,
3825 retry_in,
3826 probe_in_flight,
3827 } => {
3828 return Ok(vec![self.route_open_breaker_refusal_frame(
3829 ctx,
3830 &frame,
3831 &target_module_id,
3832 consecutive_timeouts,
3833 retry_in,
3834 probe_in_flight,
3835 )?]);
3836 }
3837 };
3838
3839 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3844 let relay_deadline = Instant::now() + route_bind_relay_timeout;
3845 let pending = match self
3846 .forwarding
3847 .begin_route_bind_relay_for(
3848 ctx.connection_id,
3849 ctx.egress.clone(),
3850 response_version(&frame),
3851 frame.header.corr,
3852 &target_module_id,
3853 principal.clone(),
3854 bound_scope,
3855 Some(project_root),
3856 relay_deadline,
3857 )
3858 .await
3859 {
3860 Ok(pending) => pending,
3861 Err(err) => {
3862 return Ok(vec![self.route_open_refusal_frame(
3863 ctx,
3864 &frame,
3865 &target_module_id,
3866 "relay_reservation_failed",
3867 forwarding_error_code(&err),
3868 err.to_string(),
3869 )?])
3870 }
3871 };
3872 let crate::forwarding::PendingRouteBindRelay {
3873 endpoint,
3874 module_sink,
3875 negotiated_ver,
3876 client_channel,
3877 client_epoch,
3878 module_channel,
3879 module_epoch,
3880 corr: relay_corr,
3881 receiver,
3882 } = pending;
3883 let mut reservation =
3884 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3885
3886 if let Some(stamp) = scope_stamp
3891 .as_ref()
3892 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
3893 {
3894 let relay_registration = self
3895 .registry
3896 .get_module_by_connection(endpoint.connection_id)
3897 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3898 if let Err(refusal) = crate::scopes::check_target_flow_support(
3899 stamp,
3900 &target_module_id,
3901 relay_registration
3902 .as_ref()
3903 .and_then(|registration| registration.manifest.capabilities.as_ref()),
3904 )
3905 .and_then(|()| {
3906 crate::scopes::check_target_agent_run_support(
3907 stamp,
3908 &target_module_id,
3909 relay_registration
3910 .as_ref()
3911 .and_then(|registration| registration.manifest.capabilities.as_ref()),
3912 )
3913 }) {
3914 reservation.release_and_disarm();
3915 return Ok(vec![self.route_open_refusal_frame(
3916 ctx,
3917 &frame,
3918 &target_module_id,
3919 refusal.code,
3920 refusal.code,
3921 refusal.message,
3922 )?]);
3923 }
3924 }
3925
3926 debug!(
3927 connection_id = ctx.connection_id.get(),
3928 client_channel,
3929 client_epoch,
3930 module_channel,
3931 module_epoch,
3932 "reserved route handle pair"
3933 );
3934 let principal_label = principal_label(&principal);
3937 let relay = ModuleControlRequest::RouteBind {
3938 route_channel: module_channel,
3939 epoch: module_epoch,
3940 target,
3941 identity,
3942 principal: Some(principal),
3943 consumer_capabilities,
3944 role_versions,
3945 admission_facts,
3946 scope: scope_stamp,
3947 };
3948 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3949 RouterError::backend(
3950 0,
3951 frame.header.corr,
3952 format!("failed to encode route.bind request: {err}"),
3953 )
3954 })?;
3955 let relay_frame = Frame::build_with_version(
3956 negotiated_ver,
3957 FrameType::Request,
3958 control_flags(),
3959 0,
3960 0,
3961 relay_corr,
3962 relay_body,
3963 )
3964 .map_err(RouterError::FrameBuild)?;
3965
3966 if let Err(err) = module_sink.send(relay_frame).await {
3967 reservation.release_and_disarm();
3968 return Ok(vec![self.route_open_refusal_frame(
3969 ctx,
3970 &frame,
3971 &target_module_id,
3972 "relay_send_failed",
3973 "target_unavailable",
3974 err.to_string(),
3975 )?]);
3976 }
3977
3978 if !self
3979 .forwarding
3980 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
3981 .map_err(RouterError::Forwarding)?
3982 {
3983 self.send_abandoned_route_bind_goodbye(
3984 &module_sink,
3985 negotiated_ver,
3986 module_channel,
3987 module_epoch,
3988 );
3989 }
3990
3991 match timeout_at(relay_deadline, receiver).await {
3992 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
3993 reservation.disarm();
3994 if breaker.record_accepted() {
3995 info!(
3996 module_id = %target_module_id,
3997 "route.bind breaker closed: the probe was accepted"
3998 );
3999 }
4000 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4001 Ok(Vec::new())
4002 }
4003 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4004 reservation.release_and_disarm();
4005 breaker.record_inconclusive();
4009 let scope_code = match body.code.as_str() {
4013 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4014 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4015 _ => None,
4016 };
4017 if let Some(code) = scope_code {
4018 self.observe_route_open_refusal(
4019 ctx,
4020 &target_module_id,
4021 "scope_changed_before_commit",
4022 code,
4023 );
4024 return Ok(vec![control_error_body_frame(&frame, body)?]);
4025 }
4026 self.counters
4027 .increment_route_open_refused("module_rejected");
4028 info!(
4029 target: "control",
4030 code = "module_rejected",
4031 module_code = ?body.code,
4032 module_id = ?target_module_id,
4033 connection_id = ctx.connection_id.get(),
4034 "route.open refused"
4035 );
4036 Ok(vec![control_error_body_frame(&frame, body)?])
4037 }
4038 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4039 reservation.release_and_disarm();
4040 breaker.record_inconclusive();
4041 tracing::warn!(
4048 module_id = %target_module_id,
4049 "route.bind relay abandoned: {message}"
4050 );
4051 Ok(vec![self.route_open_refusal_frame(
4052 ctx,
4053 &frame,
4054 &target_module_id,
4055 "relay_abandoned",
4056 "target_unavailable",
4057 message,
4058 )?])
4059 }
4060 Ok(Err(_)) => {
4061 reservation.release_and_disarm();
4062 breaker.record_inconclusive();
4063 Ok(vec![self.route_open_refusal_frame(
4064 ctx,
4065 &frame,
4066 &target_module_id,
4067 "relay_waiter_canceled",
4068 "target_unavailable",
4069 "route.bind relay waiter was canceled before the module responded",
4070 )?])
4071 }
4072 Err(_) => {
4073 reservation.release_and_disarm();
4074 if let Some(opened) = breaker.record_timeout(
4078 self.route_bind_breaker_threshold,
4079 self.route_bind_breaker_cooldown,
4080 ) {
4081 warn!(
4082 module_id = %target_module_id,
4083 consecutive_timeouts = opened.consecutive_timeouts,
4084 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4085 reopened_after_probe = opened.reopened_after_probe,
4086 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4087 );
4088 }
4089 tracing::warn!(
4097 module_id = %target_module_id,
4098 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4099 "route.bind relay timed out: module did not ack within budget"
4100 );
4101 Ok(vec![self.route_open_refusal_frame(
4102 ctx,
4103 &frame,
4104 &target_module_id,
4105 "relay_timed_out",
4106 "module_timeout",
4107 format!(
4108 "module_id '{target_module_id}' did not answer route.bind within {:?}",
4109 route_bind_relay_timeout
4110 ),
4111 )?])
4112 }
4113 }
4114 }
4115
4116 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4117 let response = ClientControlResponse::SupervisorSpawnSnapshot {
4118 snapshot: self.supervisor.spawn_snapshot(),
4119 };
4120 Ok(vec![control_response_body_frame(
4121 &frame,
4122 &response,
4123 "ClientControlResponse::SupervisorSpawnSnapshot",
4124 )?])
4125 }
4126
4127 fn handle_supervisor_spawn_subscribe(
4128 &self,
4129 ctx: &RouteCtx,
4130 frame: Frame,
4131 since: Option<SpawnCursor>,
4132 ) -> Result<Vec<Frame>, RouterError> {
4133 match self.supervisor.subscribe_spawns(
4134 ctx.connection_id,
4135 frame.header.corr,
4136 response_version(&frame),
4137 since,
4138 ctx.egress.clone(),
4139 ) {
4140 Ok(()) => Ok(Vec::new()),
4141 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4142 Ok(vec![control_error_body_frame(
4143 &frame,
4144 ErrorBody {
4145 code: "spawn_cursor_incarnation_mismatch".to_string(),
4146 message: "spawn cursor belongs to a different daemon incarnation"
4147 .to_string(),
4148 detail: Some(serde_json::json!({
4149 "current_daemon_incarnation": current
4150 })),
4151 },
4152 )?])
4153 }
4154 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4155 &frame,
4156 ErrorBody {
4157 code: "spawn_cursor_too_old".to_string(),
4158 message: "spawn cursor predates the retained event ring".to_string(),
4159 detail: Some(serde_json::json!({
4160 "oldest_retained_cursor": oldest
4161 })),
4162 },
4163 )?]),
4164 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4165 0,
4166 frame.header.corr,
4167 format!("failed to open supervisor spawn subscription: {error}"),
4168 )),
4169 }
4170 }
4171
4172 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4173 let generation = self
4174 .registry
4175 .generation()
4176 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4177 let mut modules = Vec::new();
4178 for module in self.supervisor.list() {
4179 let status = module.status_for_control("list").map_err(|err| {
4180 RouterError::backend(
4181 0,
4182 frame.header.corr,
4183 format!("failed to read supervisor status: {err}"),
4184 )
4185 })?;
4186 let (configured, _) = module.configuration().map_err(|err| {
4187 RouterError::backend(
4188 0,
4189 frame.header.corr,
4190 format!("failed to read module configuration: {err}"),
4191 )
4192 })?;
4193 let image = module.running_image_agreement().await;
4195 let resources = Some(module.child_resource_usage());
4198 let pending_reload = Some(reload_verdict(
4199 &configured.program,
4200 status.spawned_from.as_deref(),
4201 image,
4202 ));
4203 modules.push(
4206 SupervisorEntry::new(
4207 status.module_id,
4208 status.state.to_string(),
4209 status.enabled,
4210 status.live,
4211 status.health.status,
4212 )
4213 .with_launch_nonce_env(Some(!cfg!(unix)))
4214 .with_protocol(status.protocol)
4215 .with_pending_reload(pending_reload)
4216 .with_last_probe_ms(status.health.last_probe_ms)
4217 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
4218 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
4219 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
4220 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
4221 .with_restart_count(Some(status.restart_count))
4222 .with_max_restarts(Some(status.max_restarts))
4223 .with_lifetime_restarts(Some(status.lifetime_restarts))
4224 .with_spawn_generation(Some(status.spawn_generation))
4225 .with_restart_window_secs(Some(status.restart_window.as_secs()))
4226 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
4227 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
4228 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
4229 .with_resources(resources),
4230 );
4231 }
4232 let response = ClientControlResponse::SupervisorList {
4233 generation,
4234 modules,
4235 };
4236 Ok(vec![control_response_body_frame(
4237 &frame,
4238 &response,
4239 "ClientControlResponse::SupervisorList",
4240 )?])
4241 }
4242
4243 fn handle_supervisor_stderr_tail(
4244 &self,
4245 frame: Frame,
4246 module_id: String,
4247 max_lines: Option<u32>,
4248 max_bytes: Option<u32>,
4249 ) -> Result<Vec<Frame>, RouterError> {
4250 let Some(module) = self.supervisor.get(&module_id) else {
4251 return Ok(vec![control_error_frame(
4252 &frame,
4253 "unknown_module",
4254 format!("module_id '{module_id}' is not supervised"),
4255 )?]);
4256 };
4257
4258 let snapshot = module.stderr_tail(
4259 max_lines.map(|value| value as usize),
4260 max_bytes.map(|value| value as usize),
4261 );
4262
4263 let response = ClientControlResponse::SupervisorStderrTail {
4264 module_id,
4265 tail: StderrTail {
4266 capture: match snapshot.capture {
4267 CaptureState::Captured => StderrCaptureState::Captured,
4268 CaptureState::Incomplete { reason } => {
4269 StderrCaptureState::Incomplete { reason }
4270 }
4271 CaptureState::NotCaptured { reason } => {
4272 StderrCaptureState::NotCaptured { reason }
4273 }
4274 },
4275 entries: snapshot
4276 .entries
4277 .into_iter()
4278 .map(|entry| match entry {
4279 TailEntry::Line {
4280 text,
4281 truncated,
4282 at_ms,
4283 } => StderrTailEntry::Line {
4284 text,
4285 truncated,
4286 at_ms,
4287 },
4288 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
4289 })
4290 .collect(),
4291 dropped_lines: snapshot.dropped_lines,
4292 },
4293 };
4294 Ok(vec![control_response_body_frame(
4295 &frame,
4296 &response,
4297 "ClientControlResponse::SupervisorStderrTail",
4298 )?])
4299 }
4300
4301 async fn handle_supervisor_terminals(
4302 &self,
4303 frame: Frame,
4304 module_id: String,
4305 ) -> Result<Vec<Frame>, RouterError> {
4306 let Some(module) = self.supervisor.get(&module_id) else {
4307 return Ok(vec![control_error_frame(
4308 &frame,
4309 "unknown_module",
4310 format!("module_id '{module_id}' is not supervised"),
4311 )?]);
4312 };
4313
4314 let terminals = module
4317 .read_durable_terminal_history()
4318 .await
4319 .map_err(|error| {
4320 RouterError::backend(
4321 0,
4322 frame.header.corr,
4323 format!("failed to read terminal history: {error}"),
4324 )
4325 })?;
4326 let response = ClientControlResponse::SupervisorTerminals {
4327 module_id,
4328 terminals,
4329 };
4330 Ok(vec![control_response_body_frame(
4331 &frame,
4332 &response,
4333 "ClientControlResponse::SupervisorTerminals",
4334 )?])
4335 }
4336
4337 fn handle_supervisor_routes(
4338 &self,
4339 frame: Frame,
4340 module_id: Option<String>,
4341 ) -> Result<Vec<Frame>, RouterError> {
4342 let modules = self
4343 .forwarding
4344 .route_census(module_id.as_deref())
4345 .map_err(RouterError::Forwarding)?
4346 .into_iter()
4347 .map(|(module_id, routes)| SupervisorRouteModule {
4348 module_id,
4349 routes: routes
4350 .into_iter()
4351 .map(|route| SupervisorRoute {
4352 consumer: match route.principal {
4353 Principal::Reserved { module_id } => {
4354 SupervisorRouteConsumer::Reserved { module_id }
4355 }
4356 Principal::Direct | Principal::Unverified => {
4357 SupervisorRouteConsumer::Direct {
4358 connection_id: route.goodbye_target.connection_id.get(),
4359 }
4360 }
4361 },
4362 age_ms: Instant::now()
4363 .saturating_duration_since(route.bound_at)
4364 .as_millis()
4365 .try_into()
4366 .unwrap_or(u64::MAX),
4367 draining: route.draining,
4368 drain_reason: route.drain_reason,
4369 })
4370 .collect(),
4371 })
4372 .collect();
4373 let response = ClientControlResponse::SupervisorRoutes { modules };
4374 Ok(vec![control_response_body_frame(
4375 &frame,
4376 &response,
4377 "ClientControlResponse::SupervisorRoutes",
4378 )?])
4379 }
4380
4381 async fn handle_supervisor_provenance(
4382 &self,
4383 frame: Frame,
4384 module_id: Option<String>,
4385 ) -> Result<Vec<Frame>, RouterError> {
4386 let mut selected = if let Some(module_id) = module_id {
4387 let Some(module) = self.supervisor.get(&module_id) else {
4388 return Ok(vec![control_error_frame(
4389 &frame,
4390 "unknown_module",
4391 format!("module_id '{module_id}' is not supervised"),
4392 )?]);
4393 };
4394 vec![module]
4395 } else {
4396 self.supervisor.list()
4397 };
4398
4399 let mut modules = Vec::with_capacity(selected.len());
4400 for module in selected.drain(..) {
4401 let (status, observed_image) = module
4402 .status_and_running_image_agreement()
4403 .await
4404 .map_err(|err| {
4405 RouterError::backend(
4406 0,
4407 frame.header.corr,
4408 format!("failed to read supervisor status: {err}"),
4409 )
4410 })?;
4411 let module_declared = self
4412 .registry
4413 .get_module(&status.module_id)
4414 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4415 .and_then(|registration| registration.manifest.provenance)
4416 .map(|build| ModuleDeclaredProvenance::Reported { build })
4417 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4418 #[cfg(test)]
4419 let running_image = match &self.provenance_probe_override {
4420 Some(result) => result.clone(),
4421 None => observed_image,
4422 };
4423 #[cfg(not(test))]
4424 let running_image = observed_image;
4425 modules.push(SupervisorModuleProvenance {
4426 module_id: status.module_id,
4427 module_declared,
4428 daemon_observed: SupervisorObservedProcess {
4429 pid: status.pid,
4430 spawned_at_ms: status.spawned_at_ms,
4431 spawned_from: status.spawned_from,
4432 running_image,
4433 },
4434 });
4435 }
4436 let daemon = SupervisorDaemonProvenance {
4437 daemon_build: self.daemon_provenance.build.clone(),
4438 daemon_observed: DaemonObservedProcess {
4439 pid: self.daemon_provenance.pid,
4440 started_at_ms: self
4441 .daemon_provenance
4442 .start_clock
4443 .map(|clock| clock.started_at_ms())
4444 .or(self.daemon_provenance.started_at_ms),
4445 running_image: self
4446 .daemon_provenance
4447 .probe
4448 .observe(
4449 self.daemon_provenance.pid,
4450 self.daemon_provenance.executable_path.as_deref(),
4451 self.daemon_provenance.executable_identity,
4452 self.daemon_provenance.process_start_time,
4453 )
4454 .await,
4455 },
4456 };
4457 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
4458 Ok(vec![control_response_body_frame(
4459 &frame,
4460 &response,
4461 "ClientControlResponse::SupervisorProvenance",
4462 )?])
4463 }
4464
4465 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4466 self.refresh_capability_requirements();
4467 let generation = self
4468 .registry
4469 .generation()
4470 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4471 let modules = self
4472 .supervisor
4473 .list()
4474 .into_iter()
4475 .map(|module| {
4476 let status = module.status_for_control("health").map_err(|err| {
4477 RouterError::backend(
4478 0,
4479 frame.header.corr,
4480 format!("failed to read supervisor health: {err}"),
4481 )
4482 })?;
4483 let module_id = status.module_id;
4484 let capability_detail = self
4485 .capability_evaluator
4486 .required_problem_detail(&module_id);
4487 Ok(SupervisorHealthEntry {
4488 module_id,
4489 status: status.health.status,
4490 detail: append_capability_problem_detail(
4491 status.health.detail,
4492 capability_detail,
4493 ),
4494 metrics: status.health.metrics,
4495 consecutive_failures: status.health.consecutive_failures,
4496 late_answer_count: status.health.late_answer_count,
4497 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4498 last_action: status.health.last_action,
4499 last_action_ms: status.health.last_action_ms,
4500 last_probe_ms: status.health.last_probe_ms,
4501 })
4502 })
4503 .collect::<Result<Vec<_>, RouterError>>()?;
4504 let response = ClientControlResponse::SupervisorHealth {
4505 generation,
4506 modules,
4507 };
4508 Ok(vec![control_response_body_frame(
4509 &frame,
4510 &response,
4511 "ClientControlResponse::SupervisorHealth",
4512 )?])
4513 }
4514
4515 async fn handle_supervisor_restart(
4516 &self,
4517 frame: Frame,
4518 module_id: String,
4519 drain_timeout_ms: Option<u64>,
4520 ) -> Result<Vec<Frame>, RouterError> {
4521 let operation_lock = self.supervisor.operation_lock();
4522 let _operation_guard = operation_lock.lock().await;
4523 let Some(module) = self.supervisor.get(&module_id) else {
4524 return Ok(vec![control_error_frame(
4525 &frame,
4526 "unknown_module",
4527 format!("module_id '{module_id}' is not supervised"),
4528 )?]);
4529 };
4530
4531 self.route_outages.mark_operator_action(&module_id);
4532 if let Err(err) = module.restart(drain_timeout_ms).await {
4533 self.route_outages
4534 .operator_action_ended_unrefused(&module_id);
4535 let (code, message) = match err {
4536 crate::supervise::SuperviseError::Disabled { .. } => {
4537 ("module_disabled", err.to_string())
4538 }
4539 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4540 ("swap_in_progress", err.to_string())
4541 }
4542 _ => (
4543 "target_unavailable",
4544 format!("failed to restart module_id '{module_id}': {err}"),
4545 ),
4546 };
4547 return Ok(vec![control_error_frame(&frame, code, message)?]);
4548 }
4549
4550 let response = ClientControlResponse::SupervisorAck {
4551 module_id,
4552 applied: true,
4553 };
4554 Ok(vec![control_response_body_frame(
4555 &frame,
4556 &response,
4557 "ClientControlResponse::SupervisorAck",
4558 )?])
4559 }
4560
4561 async fn handle_supervisor_swap(
4565 &self,
4566 frame: Frame,
4567 module_id: String,
4568 ready_timeout_ms: Option<u64>,
4569 ) -> Result<Vec<Frame>, RouterError> {
4570 let module = {
4577 let operation_lock = self.supervisor.operation_lock();
4578 let _operation_guard = operation_lock.lock().await;
4579 self.supervisor.get(&module_id)
4580 };
4581 let Some(module) = module else {
4582 return Ok(vec![control_error_frame(
4583 &frame,
4584 "unknown_module",
4585 format!("module_id '{module_id}' is not supervised"),
4586 )?]);
4587 };
4588
4589 self.route_outages.mark_operator_action(&module_id);
4590 if let Err(err) = module
4591 .swap(ready_timeout_ms.map(Duration::from_millis))
4592 .await
4593 {
4594 self.route_outages
4595 .operator_action_ended_unrefused(&module_id);
4596 use crate::supervise::SuperviseError;
4597 let message = err.to_string();
4598 let error = match err {
4599 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4600 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4601 code: "swap_refused".to_string(),
4602 message,
4603 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4604 },
4605 SuperviseError::SwapFailed {
4606 arm,
4607 candidate_exit,
4608 ..
4609 } => ErrorBody {
4610 code: "swap_failed".to_string(),
4611 message,
4612 detail: Some(serde_json::json!({
4613 "arm": arm.as_str(),
4614 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4615 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4616 })),
4617 },
4618 _ => ErrorBody::new(
4619 "target_unavailable",
4620 format!("failed to swap module_id '{module_id}': {message}"),
4621 ),
4622 };
4623 return Ok(vec![control_error_body_frame(&frame, error)?]);
4624 }
4625 self.route_outages
4629 .operator_action_ended_unrefused(&module_id);
4630
4631 let response = ClientControlResponse::SupervisorAck {
4632 module_id,
4633 applied: true,
4634 };
4635 Ok(vec![control_response_body_frame(
4636 &frame,
4637 &response,
4638 "ClientControlResponse::SupervisorAck",
4639 )?])
4640 }
4641
4642 async fn handle_supervisor_reload(
4643 &self,
4644 frame: Frame,
4645 module_id: String,
4646 ) -> Result<Vec<Frame>, RouterError> {
4647 let operation_lock = self.supervisor.operation_lock();
4648 let _operation_guard = operation_lock.lock().await;
4649 let Some(module) = self.supervisor.get(&module_id) else {
4650 return Ok(vec![control_error_frame(
4651 &frame,
4652 "unknown_module",
4653 format!("module_id '{module_id}' is not supervised"),
4654 )?]);
4655 };
4656
4657 self.route_outages.mark_operator_action(&module_id);
4658 if let Err(err) = module.reload().await {
4659 self.route_outages
4660 .operator_action_ended_unrefused(&module_id);
4661 let (code, message) = match err {
4662 crate::supervise::SuperviseError::Disabled { .. } => {
4663 ("module_disabled", err.to_string())
4664 }
4665 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4666 ("swap_in_progress", err.to_string())
4667 }
4668 _ => (
4669 "reload_failed",
4670 format!("failed to reload module_id '{module_id}': {err}"),
4671 ),
4672 };
4673 return Ok(vec![control_error_frame(&frame, code, message)?]);
4674 }
4675
4676 let response = ClientControlResponse::SupervisorAck {
4677 module_id,
4678 applied: true,
4679 };
4680 Ok(vec![control_response_body_frame(
4681 &frame,
4682 &response,
4683 "ClientControlResponse::SupervisorAck",
4684 )?])
4685 }
4686
4687 async fn handle_supervisor_rescan(
4688 &self,
4689 frame: Frame,
4690 preview: bool,
4691 ) -> Result<Vec<Frame>, RouterError> {
4692 let Some(context) = self.rescan.clone() else {
4693 return Ok(vec![control_error_frame(
4694 &frame,
4695 "rescan_unavailable",
4696 "the daemon was not started with a reloadable config path".to_string(),
4697 )?]);
4698 };
4699
4700 let operation_lock = self.supervisor.operation_lock();
4701 let _operation_guard = operation_lock.lock().await;
4702 let loaded = match crate::daemon_config::load(&context.config_path) {
4703 Ok(config) => config,
4704 Err(err) => {
4705 return Ok(vec![control_error_frame(
4706 &frame,
4707 "invalid_daemon_config",
4708 format!("supervisor rescan rejected daemon config: {err}"),
4709 )?])
4710 }
4711 };
4712 let Some(config) = loaded else {
4720 return Ok(vec![control_error_frame(
4721 &frame,
4722 "invalid_daemon_config",
4723 format!(
4724 "daemon config not found at {}; refusing to rescan (an absent config would \
4725 retire every supervised module)",
4726 context.config_path.display()
4727 ),
4728 )?]);
4729 };
4730 let (
4731 configured_port,
4732 storage_config,
4733 admission_facts_carrier_module_id,
4734 admission_facts_targets,
4735 scope_authority_owners,
4736 modules,
4737 reserved_capabilities,
4738 ) = (
4739 config.port,
4740 config.storage,
4741 config.admission_facts_carrier_module_id,
4742 config.admission_facts_targets,
4743 config.scope_authority_owners,
4744 config.modules,
4745 config.reserved_capabilities,
4746 );
4747
4748 let mut restart_required = Vec::new();
4757 for section in RestartRequiredSection::ALL {
4758 let changed = match section {
4759 RestartRequiredSection::Port => configured_port != context.configured_port,
4760 RestartRequiredSection::Storage => storage_config != context.storage_config,
4761 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4762 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4763 }
4764 RestartRequiredSection::AdmissionFactsTargets => {
4765 admission_facts_targets != context.admission_facts_targets
4766 }
4767 RestartRequiredSection::ScopeAuthorityOwners => {
4768 scope_authority_owners != context.scope_authority_owners
4769 }
4770 };
4771 if changed {
4772 restart_required.push(section.label().to_string());
4773 }
4774 }
4775 if !restart_required.is_empty() {
4776 warn!(
4777 config_path = %context.config_path.display(),
4778 sections = %restart_required.join(", "),
4779 "daemon config changed outside the modules section; restart the daemon to apply those changes"
4780 );
4781 }
4782
4783 for configured in &modules {
4784 if let Err(err) = validate_spec(&configured.module_spec()) {
4785 return Ok(vec![control_error_frame(
4786 &frame,
4787 "invalid_daemon_config",
4788 format!("supervisor rescan rejected daemon config: {err}"),
4789 )?]);
4790 }
4791 }
4792
4793 let configured_capabilities = modules
4794 .iter()
4795 .map(|module| (module.module_id.clone(), module.enabled))
4796 .collect::<Vec<_>>();
4797 let preview_capability_warnings = if preview {
4798 let (_, registrations) = self.runtime_capability_snapshot()?;
4799 let current_modules = self
4800 .supervisor
4801 .list()
4802 .into_iter()
4803 .map(|module| module.module_id().to_string())
4804 .collect::<BTreeSet<_>>();
4805 let resulting_modules = configured_capabilities.clone();
4806 let removed = current_modules
4807 .into_iter()
4808 .filter(|module_id| {
4809 !resulting_modules
4810 .iter()
4811 .any(|(configured_id, _)| configured_id == module_id)
4812 })
4813 .collect::<Vec<_>>();
4814 self.capability_evaluator.preview_removal_warnings(
4815 resulting_modules,
4816 &removed,
4817 ®istrations,
4818 )
4819 } else {
4820 Vec::new()
4821 };
4822 let result = match self
4823 .reconcile_supervised_modules(&context.supervisor, modules, preview)
4824 .await
4825 {
4826 Ok(result) => result,
4827 Err(message) => {
4828 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4829 }
4830 };
4831 if !preview {
4832 self.capability_evaluator
4833 .configure(configured_capabilities, reserved_capabilities);
4834 self.capability_evaluator.wake_deadline_loop();
4835 self.refresh_capability_requirements();
4836 }
4837 let mut result = result;
4838 result.restart_required = restart_required;
4839 result.capability_warnings = preview_capability_warnings;
4840 let response = ClientControlResponse::SupervisorRescan { result };
4841 Ok(vec![control_response_body_frame(
4842 &frame,
4843 &response,
4844 "ClientControlResponse::SupervisorRescan",
4845 )?])
4846 }
4847
4848 async fn handle_supervisor_release_reserved(
4849 &self,
4850 frame: Frame,
4851 module_id: String,
4852 ) -> Result<Vec<Frame>, RouterError> {
4853 let Some(context) = self.rescan.clone() else {
4854 return Ok(vec![control_error_frame(
4855 &frame,
4856 "release_unavailable",
4857 "reserved-id release requires a daemon started with a reloadable config path",
4858 )?]);
4859 };
4860 let operation_lock = self.supervisor.operation_lock();
4861 let _operation_guard = operation_lock.lock().await;
4862 let loaded = match crate::daemon_config::load(&context.config_path) {
4863 Ok(Some(config)) => config,
4864 Ok(None) => {
4865 return Ok(vec![control_error_frame(
4866 &frame,
4867 "invalid_daemon_config",
4868 format!(
4869 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4870 context.config_path.display()
4871 ),
4872 )?])
4873 }
4874 Err(err) => {
4875 return Ok(vec![control_error_frame(
4876 &frame,
4877 "invalid_daemon_config",
4878 format!("unable to verify reserved-id release against daemon config: {err}"),
4879 )?])
4880 }
4881 };
4882 if loaded
4883 .modules
4884 .iter()
4885 .any(|configured| configured.module_id == module_id)
4886 {
4887 return Ok(vec![control_error_frame(
4888 &frame,
4889 "reserved_module_configured",
4890 format!(
4891 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4892 ),
4893 )?]);
4894 }
4895 if !self.supervisor.release_retained_reserved_gate(&module_id) {
4896 return Ok(vec![control_error_frame(
4897 &frame,
4898 "reserved_gate_not_retained",
4899 format!(
4900 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4901 ),
4902 )?]);
4903 }
4904
4905 let response = ClientControlResponse::SupervisorAck {
4906 module_id,
4907 applied: true,
4908 };
4909 Ok(vec![control_response_body_frame(
4910 &frame,
4911 &response,
4912 "ClientControlResponse::SupervisorAck",
4913 )?])
4914 }
4915
4916 async fn reconcile_supervised_modules(
4925 &self,
4926 supervisor: &Supervisor,
4927 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4928 preview: bool,
4929 ) -> Result<SupervisorRescanResult, String> {
4930 let mut current = BTreeMap::new();
4931 for module in self.supervisor.list() {
4932 let (spec, health) = module.configuration().map_err(|err| {
4933 format!(
4934 "failed to read configuration for module_id '{}': {err}",
4935 module.module_id()
4936 )
4937 })?;
4938 let enabled = module
4939 .status()
4940 .map_err(|err| {
4941 format!(
4942 "failed to read status for module_id '{}': {err}",
4943 module.module_id()
4944 )
4945 })?
4946 .enabled;
4947 current.insert(
4948 module.module_id().to_string(),
4949 (module, spec, health, enabled),
4950 );
4951 }
4952 let configured = configured_modules
4953 .into_iter()
4954 .map(|module| (module.module_id.clone(), module))
4955 .collect::<BTreeMap<_, _>>();
4956
4957 let added = configured
4958 .keys()
4959 .filter(|module_id| !current.contains_key(*module_id))
4960 .cloned()
4961 .collect::<Vec<_>>();
4962 let removed = current
4963 .keys()
4964 .filter(|module_id| !configured.contains_key(*module_id))
4965 .cloned()
4966 .collect::<Vec<_>>();
4967 let mut changed_pending_reload = Vec::new();
4968 let mut configuration_changes = BTreeSet::new();
4969 let mut enabled_changes = BTreeSet::new();
4970 let mut unchanged = 0_u32;
4971
4972 for (module_id, configured_module) in &configured {
4973 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
4974 else {
4975 continue;
4976 };
4977 let launch_changed = *current_spec != configured_module.module_spec();
4981 let configuration_changed =
4982 launch_changed || *current_health != configured_module.health;
4983 let enabled_changed = *current_enabled != configured_module.enabled;
4984 if configuration_changed {
4985 configuration_changes.insert(module_id.clone());
4986 }
4987 if launch_changed {
4988 changed_pending_reload.push(module_id.clone());
4989 }
4990 if enabled_changed {
4991 enabled_changes.insert(module_id.clone());
4992 }
4993 if !configuration_changed && !enabled_changed {
4994 unchanged = unchanged.saturating_add(1);
4995 }
4996 }
4997
4998 if preview {
5004 return Ok(SupervisorRescanResult {
5005 added,
5006 removed,
5007 changed_pending_reload,
5008 enabled_changes: enabled_changes.iter().cloned().collect(),
5009 unchanged,
5010 preview: true,
5011 restart_required: Vec::new(),
5015 capability_warnings: Vec::new(),
5016 });
5017 }
5018
5019 for module_id in &removed {
5020 let module = ¤t
5021 .get(module_id)
5022 .expect("removed module came from current supervisor state")
5023 .0;
5024 module.retire().await.map_err(|err| {
5025 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5026 })?;
5027 self.supervisor.record_rescan_removal(module_id);
5061 self.supervisor.retire(module_id);
5062 self.route_outages.forget(module_id);
5063 }
5064
5065 for module_id in configured.keys() {
5066 let Some((module, _, _, _)) = current.get(module_id) else {
5067 continue;
5068 };
5069 let configured_module = configured
5070 .get(module_id)
5071 .expect("configured module id came from configured map");
5072 if configuration_changes.contains(module_id) {
5073 module
5074 .update_configuration(
5075 configured_module.module_spec(),
5076 configured_module.health.clone(),
5077 configured_module.drain_timeout_ms,
5078 )
5079 .await
5080 .map_err(|err| {
5081 format!(
5082 "failed to update module_id '{module_id}' configuration during rescan: {err}"
5083 )
5084 })?;
5085 }
5086 if enabled_changes.contains(module_id) {
5087 self.route_outages.mark_operator_action(module_id);
5090 module
5091 .set_enabled(configured_module.enabled)
5092 .await
5093 .map_err(|err| {
5094 self.route_outages.operator_action_ended_unrefused(module_id);
5095 format!(
5096 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5097 configured_module.enabled
5098 )
5099 })?;
5100 }
5101 }
5102
5103 for module_id in &added {
5104 let configured_module = configured
5105 .get(module_id)
5106 .expect("added module id came from configured map");
5107 supervisor
5108 .supervise_configured_with_health(
5109 configured_module.module_spec(),
5110 configured_module.enabled,
5111 configured_module.health.clone(),
5112 configured_module.drain_timeout_ms,
5113 configured_module.restart,
5114 )
5115 .map_err(|err| {
5116 format!("failed to add module_id '{module_id}' during rescan: {err}")
5117 })?;
5118 }
5119
5120 Ok(SupervisorRescanResult {
5121 added,
5122 removed,
5123 changed_pending_reload,
5124 enabled_changes: enabled_changes.iter().cloned().collect(),
5125 unchanged,
5126 preview: false,
5127 restart_required: Vec::new(),
5130 capability_warnings: Vec::new(),
5131 })
5132 }
5133
5134 async fn handle_supervisor_set_enabled(
5135 &self,
5136 frame: Frame,
5137 module_id: String,
5138 enabled: bool,
5139 ) -> Result<Vec<Frame>, RouterError> {
5140 let operation_lock = self.supervisor.operation_lock();
5141 let _operation_guard = operation_lock.lock().await;
5142 let Some(module) = self.supervisor.get(&module_id) else {
5143 return Ok(vec![control_error_frame(
5144 &frame,
5145 "unknown_module",
5146 format!("module_id '{module_id}' is not supervised"),
5147 )?]);
5148 };
5149
5150 self.route_outages.mark_operator_action(&module_id);
5153 let applied = match module.set_enabled(enabled).await {
5154 Ok(applied) => applied,
5155 Err(err) => {
5156 self.route_outages
5157 .operator_action_ended_unrefused(&module_id);
5158 return Ok(vec![control_error_frame(
5159 &frame,
5160 "target_unavailable",
5161 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
5162 )?]);
5163 }
5164 };
5165 if !applied {
5166 self.route_outages
5169 .operator_action_ended_unrefused(&module_id);
5170 }
5171
5172 self.capability_evaluator.wake_deadline_loop();
5173 self.refresh_capability_requirements();
5174 let response = ClientControlResponse::SupervisorAck { module_id, applied };
5175 Ok(vec![control_response_body_frame(
5176 &frame,
5177 &response,
5178 "ClientControlResponse::SupervisorAck",
5179 )?])
5180 }
5181
5182 async fn handle_supervisor_health_probe(
5183 &self,
5184 frame: Frame,
5185 module_id: String,
5186 ) -> Result<Vec<Frame>, RouterError> {
5187 self.refresh_capability_requirements();
5188 let Some(registration) = self
5189 .registry
5190 .get_module(&module_id)
5191 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5192 else {
5193 return Ok(vec![control_error_frame(
5194 &frame,
5195 "unknown_module",
5196 format!("module_id '{module_id}' is not registered"),
5197 )?]);
5198 };
5199
5200 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
5213 {
5214 return Ok(vec![control_error_frame(
5215 &frame,
5216 "health_not_advertised",
5217 format!("module_id '{module_id}' did not advertise health.check"),
5218 )?]);
5219 }
5220
5221 let deadline = Instant::now() + self.health_probe_timeout;
5222 let pending = match self.forwarding.begin_module_control_rpc_for(
5223 &module_id,
5224 MODULE_CONTROL_OP_HEALTH_CHECK,
5225 deadline,
5226 ) {
5227 Ok(pending) => pending,
5228 Err(err) => {
5229 return Ok(vec![control_error_frame(
5230 &frame,
5231 forwarding_error_code(&err),
5232 err.to_string(),
5233 )?])
5234 }
5235 };
5236
5237 let PendingModuleControlRpc {
5238 endpoint,
5239 module_sink,
5240 negotiated_ver,
5241 corr: probe_corr,
5242 receiver,
5243 } = pending;
5244 let mut guard =
5245 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
5246 let probe_body =
5247 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5248 RouterError::backend(
5249 0,
5250 frame.header.corr,
5251 format!("failed to encode health.check request: {err}"),
5252 )
5253 })?;
5254 let probe_frame = Frame::build_with_version(
5255 negotiated_ver,
5256 FrameType::Request,
5257 control_flags(),
5258 0,
5259 0,
5260 probe_corr,
5261 probe_body,
5262 )
5263 .map_err(RouterError::FrameBuild)?;
5264
5265 if let Err(err) = module_sink.send(probe_frame).await {
5266 return Ok(vec![control_error_frame(
5267 &frame,
5268 "target_unavailable",
5269 err.to_string(),
5270 )?]);
5271 }
5272
5273 match timeout_at(deadline, receiver).await {
5274 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5275 guard.disarm();
5276 let Some(report) = response.health_report() else {
5277 return Ok(vec![control_error_frame(
5278 &frame,
5279 "invalid_control_body",
5280 "health.check RPC returned a non-health response",
5281 )?]);
5282 };
5283 let HealthReport {
5288 status,
5289 detail,
5290 metrics,
5291 } = report;
5292 let capability_detail = self
5293 .capability_evaluator
5294 .required_problem_detail(&module_id);
5295 let response = ClientControlResponse::SupervisorHealthProbe {
5296 module_id,
5297 status,
5298 detail: append_capability_problem_detail(detail, capability_detail),
5299 metrics,
5300 };
5301 Ok(vec![control_response_body_frame(
5302 &frame,
5303 &response,
5304 "ClientControlResponse::SupervisorHealthProbe",
5305 )?])
5306 }
5307 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
5308 guard.disarm();
5309 Ok(vec![control_error_body_frame(&frame, body)?])
5310 }
5311 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5312 guard.disarm();
5313 Ok(vec![control_error_frame(
5314 &frame,
5315 "target_unavailable",
5316 message,
5317 )?])
5318 }
5319 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5320 guard.disarm();
5321 Ok(vec![control_error_frame(
5322 &frame,
5323 "invalid_control_body",
5324 message,
5325 )?])
5326 }
5327 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5328 guard.disarm();
5329 Ok(vec![control_error_frame(
5330 &frame,
5331 "invalid_control_body",
5332 format!("expected module-control op '{expected}', got '{actual}'"),
5333 )?])
5334 }
5335 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
5336 guard.disarm();
5337 Ok(vec![control_error_frame(
5338 &frame,
5339 "module_timeout",
5340 format!(
5341 "module_id '{module_id}' answered health.check after {:?}",
5342 self.health_probe_timeout
5343 ),
5344 )?])
5345 }
5346 Ok(Err(_)) => Ok(vec![control_error_frame(
5347 &frame,
5348 "target_unavailable",
5349 "health.check waiter was canceled before the module responded",
5350 )?]),
5351 Err(_) => Ok(vec![control_error_frame(
5352 &frame,
5353 "module_timeout",
5354 format!(
5355 "module_id '{module_id}' did not answer health.check within {:?}",
5356 self.health_probe_timeout
5357 ),
5358 )?]),
5359 }
5360 }
5361
5362 fn supervisor_status(
5363 &self,
5364 module_id: &str,
5365 corr: u64,
5366 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5367 self.supervisor
5368 .get(module_id)
5369 .map(|module| {
5370 let warming = module.is_warming_for_control("status").map_err(|err| {
5371 RouterError::backend(
5372 0,
5373 corr,
5374 format!(
5375 "failed to read supervisor warming state for module_id '{module_id}': {err}"
5376 ),
5377 )
5378 })?;
5379 module.status_for_control("status").map_err(|err| {
5380 RouterError::backend(
5381 0,
5382 corr,
5383 format!(
5384 "failed to read supervisor status for module_id '{module_id}': {err}"
5385 ),
5386 )
5387 }).map(|status| (status, warming))
5388 })
5389 .transpose()
5390 }
5391
5392 fn guard_module_control_op(
5393 &self,
5394 frame: &Frame,
5395 module_id: &str,
5396 op: &str,
5397 ) -> Result<Option<Frame>, RouterError> {
5398 if self.module_grants_op(module_id, op, frame.header.corr)? {
5399 return Ok(None);
5400 }
5401
5402 Ok(Some(control_error_frame(
5403 frame,
5404 "op_not_allowed",
5405 format!("module_id '{module_id}' did not grant control op '{op}'"),
5406 )?))
5407 }
5408
5409 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
5410 let Some(registration) = self
5411 .registry
5412 .get_module(module_id)
5413 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
5414 else {
5415 return Ok(false);
5416 };
5417 Ok(module_registration_grants_op(®istration.control_ops, op))
5418 }
5419
5420 fn handle_status_update(
5421 &self,
5422 endpoint: ModuleEndpointId,
5423 frame: Frame,
5424 ) -> Result<Vec<Frame>, RouterError> {
5425 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
5426 Ok(update) => update,
5427 Err(err) => {
5428 if is_known_module_push_op(&frame.body) {
5433 return Ok(vec![control_error_frame(
5434 &frame,
5435 "invalid_control_body",
5436 format!("malformed module control push body: {err}"),
5437 )?]);
5438 }
5439 return Ok(Vec::new());
5440 }
5441 };
5442
5443 match update {
5444 ModuleControlPush::RouteStatus {
5445 route_channel,
5446 route_epoch,
5447 status,
5448 } => {
5449 self.forwarding
5450 .cache_status(endpoint, route_channel, route_epoch, status)
5451 .map_err(RouterError::Forwarding)?;
5452 }
5453 }
5454 Ok(Vec::new())
5455 }
5456
5457 fn handle_route_poll(
5458 &self,
5459 ctx: &RouteCtx,
5460 frame: Frame,
5461 route_channel: u16,
5462 route_epoch: u32,
5463 kind: PollKind,
5464 ) -> Result<Vec<Frame>, RouterError> {
5465 let snapshot = self
5466 .forwarding
5467 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
5468 .map_err(RouterError::Forwarding)?;
5469 let response = match (kind, snapshot) {
5470 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
5471 ClientControlResponse::RoutePoll {
5472 route_channel,
5473 route_epoch,
5474 status,
5475 live: None,
5476 }
5477 }
5478 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5479 route_channel,
5480 route_epoch,
5481 status: None,
5482 live: None,
5483 },
5484 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5485 let live = self
5511 .process_liveness
5512 .as_ref()
5513 .and_then(|source| source.process_live(&module_id))
5514 .unwrap_or(true);
5515 ClientControlResponse::RoutePoll {
5516 route_channel,
5517 route_epoch,
5518 status: None,
5519 live: Some(live),
5520 }
5521 }
5522 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5523 route_channel,
5524 route_epoch,
5525 status: None,
5526 live: Some(false),
5527 },
5528 };
5529
5530 Ok(vec![control_response_body_frame(
5531 &frame,
5532 &response,
5533 "ClientControlResponse::RoutePoll",
5534 )?])
5535 }
5536
5537 pub(crate) fn observe_module_control_completion(
5538 &self,
5539 completion: ModuleControlRpcCompletion,
5540 ) -> bool {
5541 match completion {
5542 ModuleControlRpcCompletion::Unknown => false,
5543 ModuleControlRpcCompletion::Settled => true,
5544 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5545 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5546 info!(
5547 module_id = %module_id,
5548 latency_ms,
5549 "late health.check answer proves the module is alive"
5550 );
5551 match self
5552 .supervisor
5553 .record_late_health_answer(&module_id, latency_ms)
5554 {
5555 Ok(true) => {}
5556 Ok(false) => debug!(
5557 module_id = %module_id,
5558 latency_ms,
5559 "late health.check answer has no active supervisor snapshot"
5560 ),
5561 Err(err) => warn!(
5562 module_id = %module_id,
5563 latency_ms,
5564 error = %err,
5565 "failed to record late health.check answer"
5566 ),
5567 }
5568 true
5569 }
5570 }
5571 }
5572
5573 fn refuse_to_end_module_connection_for_a_client(
5594 &self,
5595 module_connection_id: ConnectionId,
5596 corr: u64,
5597 err: ForwardingError,
5598 ) -> Result<(), RouterError> {
5599 if let ForwardingError::ConnectionClosing { connection_id } = err {
5600 if connection_id != module_connection_id {
5601 warn!(
5602 module_connection_id = module_connection_id.get(),
5603 client_connection_id = connection_id.get(),
5604 corr,
5605 "dropping a route.bind response for a closing client; the module connection keeps serving"
5606 );
5607 return Ok(());
5608 }
5609 }
5610 Err(RouterError::Forwarding(err))
5611 }
5612
5613 fn handle_module_relay_response(
5614 &self,
5615 connection_id: ConnectionId,
5616 frame: Frame,
5617 ) -> Result<Vec<Frame>, RouterError> {
5618 let mut secondary_error = None;
5619 let outcome = match frame.header.ty {
5620 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5621 Ok(probe) if probe.op == "route.bind" => {
5622 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5623 Ok(ModuleControlResponse::RouteBindAck {}) => {
5624 RouteBindRelayOutcome::Accepted
5625 }
5626 Ok(other) => {
5627 let message =
5628 format!("route.bind response carried unexpected body: {other:?}");
5629 secondary_error = Some(control_error_frame(
5630 &frame,
5631 "invalid_control_body",
5632 message.clone(),
5633 )?);
5634 RouteBindRelayOutcome::ModuleGone(message)
5635 }
5636 Err(err) => {
5637 let message = format!("malformed route.bind response body: {err}");
5638 secondary_error = Some(control_error_frame(
5639 &frame,
5640 "invalid_control_body",
5641 message.clone(),
5642 )?);
5643 RouteBindRelayOutcome::ModuleGone(message)
5644 }
5645 }
5646 }
5647 Ok(probe) => {
5648 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5649 {
5650 Ok(response) => ModuleControlRpcOutcome::Response(response),
5651 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5652 "malformed {} response body: {err}",
5653 probe.op
5654 )),
5655 };
5656 let completion = self
5657 .forwarding
5658 .complete_module_control_rpc(
5659 connection_id,
5660 frame.header.corr,
5661 Some(&probe.op),
5662 outcome,
5663 )
5664 .map_err(RouterError::Forwarding)?;
5665 if !self.observe_module_control_completion(completion) {
5666 debug!(
5667 connection_id = connection_id.get(),
5668 corr = frame.header.corr,
5669 op = %probe.op,
5670 "dropping late or unknown module-control RPC response"
5671 );
5672 }
5673 return Ok(Vec::new());
5674 }
5675 Err(err) => {
5676 if let Some(expected_op) = self
5677 .forwarding
5678 .pending_module_control_op(connection_id, frame.header.corr)
5679 .map_err(RouterError::Forwarding)?
5680 {
5681 let completion = self
5682 .forwarding
5683 .complete_module_control_rpc(
5684 connection_id,
5685 frame.header.corr,
5686 None,
5687 ModuleControlRpcOutcome::MalformedResponse(format!(
5688 "malformed {expected_op} response body: {err}"
5689 )),
5690 )
5691 .map_err(RouterError::Forwarding)?;
5692 if !self.observe_module_control_completion(completion) {
5693 debug!(
5694 connection_id = connection_id.get(),
5695 corr = frame.header.corr,
5696 "dropping late malformed module-control RPC response"
5697 );
5698 }
5699 return Ok(Vec::new());
5700 }
5701 let message = format!("malformed route.bind response body: {err}");
5702 secondary_error = Some(control_error_frame(
5703 &frame,
5704 "invalid_control_body",
5705 message.clone(),
5706 )?);
5707 RouteBindRelayOutcome::ModuleGone(message)
5708 }
5709 },
5710 FrameType::Error => {
5711 if self
5712 .forwarding
5713 .pending_module_control_op(connection_id, frame.header.corr)
5714 .map_err(RouterError::Forwarding)?
5715 .is_some()
5716 {
5717 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5718 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5719 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5720 "malformed module-control ERROR body: {err}"
5721 )),
5722 };
5723 let completion = self
5724 .forwarding
5725 .complete_module_control_rpc(
5726 connection_id,
5727 frame.header.corr,
5728 None,
5729 outcome,
5730 )
5731 .map_err(RouterError::Forwarding)?;
5732 if !self.observe_module_control_completion(completion) {
5733 debug!(
5734 connection_id = connection_id.get(),
5735 corr = frame.header.corr,
5736 "dropping late or unknown module-control RPC error"
5737 );
5738 }
5739 return Ok(Vec::new());
5740 }
5741 match serde_json::from_slice::<ErrorBody>(&frame.body) {
5742 Ok(body) => RouteBindRelayOutcome::Rejected(body),
5743 Err(err) => {
5744 let message = format!("malformed route.bind ERROR body: {err}");
5745 secondary_error = Some(control_error_frame(
5746 &frame,
5747 "invalid_control_body",
5748 message.clone(),
5749 )?);
5750 RouteBindRelayOutcome::ModuleGone(message)
5751 }
5752 }
5753 }
5754 ty => {
5755 return Ok(vec![control_error_frame(
5756 &frame,
5757 "unsupported_control_frame",
5758 format!("unsupported module channel-0 frame {ty:?}"),
5759 )?])
5760 }
5761 };
5762
5763 let settled =
5764 self.forwarding
5765 .complete_pending_relay(connection_id, frame.header.corr, outcome);
5766 let completion = match settled {
5767 Ok(completion) => completion,
5768 Err(err) => {
5769 self.refuse_to_end_module_connection_for_a_client(
5770 connection_id,
5771 frame.header.corr,
5772 err,
5773 )?;
5774 return Ok(secondary_error.into_iter().collect());
5775 }
5776 };
5777 if let Some(target) = completion.abandoned.as_ref() {
5778 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5779 }
5780 if !completion.settled {
5781 debug!(
5782 connection_id = connection_id.get(),
5783 corr = frame.header.corr,
5784 frame_type = ?frame.header.ty,
5785 "dropping late or unknown route.bind relay response"
5786 );
5787 }
5788 Ok(secondary_error.into_iter().collect())
5789 }
5790
5791 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5792 debug!(connection_id = connection_id.get(), "handling GOODBYE");
5793 self.cleanup_connection_with_end_reason(
5797 connection_id,
5798 RegistrationEndReason::ExplicitGoodbye,
5799 )
5800 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5801 Ok(Vec::new())
5802 }
5803}
5804
5805impl Default for ControlHandler {
5806 fn default() -> Self {
5807 Self::new(Arc::new(Registry::default()))
5808 }
5809}
5810
5811impl crate::supervise::SwapPromotionObserver for ControlHandler {
5812 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5813 self.apply_registration_capabilities(registration);
5814 }
5815}
5816
5817fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5818 CapabilityRequirementStatus {
5819 consumer: status.consumer,
5820 capability: status.capability,
5821 need: match status.need {
5822 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5823 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5824 },
5825 verdict: status.verdict.as_str().to_string(),
5826 episode_seq: status.episode_seq,
5827 config_satisfiable: status.config_satisfiable,
5828 runtime_available: status.runtime_available,
5829 detail: status.detail,
5830 }
5831}
5832
5833fn append_capability_problem_detail(
5834 detail: Option<String>,
5835 capability_detail: Option<String>,
5836) -> Option<String> {
5837 match (detail, capability_detail) {
5838 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5839 (Some(detail), None) => Some(detail),
5840 (None, Some(capability_detail)) => Some(capability_detail),
5841 (None, None) => None,
5842 }
5843}
5844
5845fn subc_ops() -> Vec<String> {
5846 SUBC_CONTROL_OPS
5847 .iter()
5848 .map(|op| (*op).to_string())
5849 .collect()
5850}
5851
5852fn module_subc_ops() -> Vec<String> {
5853 SUBC_CONTROL_OPS
5854 .iter()
5855 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5856 .map(|op| (*op).to_string())
5857 .collect()
5858}
5859
5860#[cfg(test)]
5861fn module_baseline_control_ops() -> Vec<String> {
5862 MODULE_BASELINE_CONTROL_OPS
5863 .iter()
5864 .map(|op| (*op).to_string())
5865 .collect()
5866}
5867
5868fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5869 let mut seen = HashSet::new();
5870 let mut effective = Vec::new();
5871 for op in MODULE_BASELINE_CONTROL_OPS {
5872 if seen.insert((*op).to_string()) {
5873 effective.push((*op).to_string());
5874 }
5875 }
5876 for op in declared.unwrap_or_default() {
5877 if seen.insert(op.clone()) {
5878 effective.push(op);
5879 }
5880 }
5881 effective
5882}
5883
5884fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5885 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5886}
5887
5888fn target_module_id(target: &RouteTarget) -> &str {
5889 match target {
5890 RouteTarget::ToolProvider { module_id }
5891 | RouteTarget::ManagementSurface { module_id }
5892 | RouteTarget::InternalService { module_id, .. } => module_id,
5893 }
5894}
5895
5896fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5897 roles.iter().any(|role| match (target, role) {
5898 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5899 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5900 (
5901 RouteTarget::InternalService { service_id, .. },
5902 ProviderRole::InternalService {
5903 service_id: provided,
5904 ..
5905 },
5906 ) => service_id == provided,
5907 _ => false,
5908 })
5909}
5910
5911fn is_routable_role(role: &ProviderRole) -> bool {
5912 matches!(
5913 role,
5914 ProviderRole::ToolProvider { .. }
5915 | ProviderRole::ManagementSurface { .. }
5916 | ProviderRole::InternalService { .. }
5917 )
5918}
5919
5920#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5921enum ControlRequestBodyError {
5922 UnknownOp,
5923 InvalidBody,
5924}
5925
5926#[derive(Debug, Deserialize)]
5927struct ControlOpProbe {
5928 op: String,
5929}
5930
5931const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5934
5935fn is_known_module_push_op(body: &[u8]) -> bool {
5936 serde_json::from_slice::<ControlOpProbe>(body)
5937 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5938 .unwrap_or(false)
5939}
5940
5941fn is_known_module_request_op(body: &[u8]) -> bool {
5942 serde_json::from_slice::<ControlOpProbe>(body)
5943 .map(|probe| is_module_to_subc_op(&probe.op))
5944 .unwrap_or(false)
5945}
5946
5947fn is_module_to_subc_op(op: &str) -> bool {
5948 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5949}
5950
5951fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5952 debug!(
5953 op = %op,
5954 connection_id = connection_id.get(),
5955 corr,
5956 "control dispatch"
5957 );
5958}
5959
5960fn log_slow_control_dispatch(
5961 dispatch_started_at: Option<StdInstant>,
5962 op: &'static str,
5963 connection_id: ConnectionId,
5964 corr: u64,
5965) {
5966 let Some(dispatch_started_at) = dispatch_started_at else {
5967 return;
5968 };
5969 let elapsed = dispatch_started_at.elapsed();
5970 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
5971 warn!(
5972 op = %op,
5973 connection_id = connection_id.get(),
5974 corr,
5975 elapsed_ms = elapsed.as_millis() as u64,
5976 "slow control dispatch"
5977 );
5978 }
5979}
5980
5981fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
5982 match request {
5983 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
5984 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
5985 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
5986 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
5987 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
5988 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
5989 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
5990 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
5991 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
5992 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
5993 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
5994 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
5995 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
5996 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
5997 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
5998 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
5999 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6000 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6001 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6002 }
6003}
6004
6005fn principal_label(principal: &Principal) -> String {
6006 match principal {
6007 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6008 Principal::Direct => "direct".to_string(),
6009 other => format!("{other:?}"),
6010 }
6011}
6012
6013fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6014 match request {
6015 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6016 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6017 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6018 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6019 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6020 }
6021}
6022
6023fn parse_client_control_request(
6024 body: &[u8],
6025) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6026 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6027 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6028 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6029 ControlRequestBodyError::InvalidBody
6030 }
6031 Ok(_) => ControlRequestBodyError::UnknownOp,
6032 Err(_) => ControlRequestBodyError::InvalidBody,
6033 };
6034 (err, classification)
6035 })
6036}
6037
6038fn parse_module_control_request_from_module(
6039 body: &[u8],
6040) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6041 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6042 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6043 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6044 Ok(_) => ControlRequestBodyError::UnknownOp,
6045 Err(_) => ControlRequestBodyError::InvalidBody,
6046 };
6047 (err, classification)
6048 })
6049}
6050
6051#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6052enum ProviderRoleKind {
6053 ToolProvider,
6054 PipelineStage,
6055 ManagementSurface,
6056 InternalService,
6057}
6058
6059fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6060 match role {
6061 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6062 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6063 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6064 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
6065 }
6066}
6067
6068fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
6069 roles.iter().map(provider_role_kind).collect()
6070}
6071
6072const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
6075
6076#[derive(Debug, Default, PartialEq, Eq)]
6078struct ScopeOutcomeCounts {
6079 created: usize,
6080 replaced: usize,
6081 updated: usize,
6082 unchanged: usize,
6083 refused: usize,
6084}
6085
6086impl ScopeOutcomeCounts {
6087 fn of(results: &[ScopeRecordResult]) -> Self {
6088 let mut counts = Self::default();
6089 for result in results {
6090 let slot = match result.outcome {
6091 ScopeRecordOutcome::Created => &mut counts.created,
6092 ScopeRecordOutcome::Replaced => &mut counts.replaced,
6093 ScopeRecordOutcome::Updated => &mut counts.updated,
6094 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
6095 ScopeRecordOutcome::Refused => &mut counts.refused,
6096 };
6097 *slot += 1;
6098 }
6099 counts
6100 }
6101}
6102
6103#[cfg(test)]
6104mod scope_outcome_count_tests {
6105 use super::*;
6106
6107 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
6108 ScopeRecordResult {
6109 scope_ref: "r".to_string(),
6110 scope_epoch: 1,
6111 outcome,
6112 code: None,
6113 message: None,
6114 version: None,
6115 parent_state: None,
6116 }
6117 }
6118
6119 #[test]
6122 fn every_outcome_is_counted_in_its_own_field() {
6123 let results = [
6124 result(ScopeRecordOutcome::Created),
6125 result(ScopeRecordOutcome::Created),
6126 result(ScopeRecordOutcome::Replaced),
6127 result(ScopeRecordOutcome::Updated),
6128 result(ScopeRecordOutcome::Unchanged),
6129 result(ScopeRecordOutcome::Refused),
6130 result(ScopeRecordOutcome::Refused),
6131 result(ScopeRecordOutcome::Refused),
6132 ];
6133 assert_eq!(
6134 ScopeOutcomeCounts::of(&results),
6135 ScopeOutcomeCounts {
6136 created: 2,
6137 replaced: 1,
6138 updated: 1,
6139 unchanged: 1,
6140 refused: 3,
6141 }
6142 );
6143 }
6144}
6145
6146fn capability_census_trigger(
6150 old: Option<&CapabilityDeclarations>,
6151 new: Option<&CapabilityDeclarations>,
6152) -> bool {
6153 let old_provides = old
6154 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
6155 .unwrap_or_default();
6156 let old_denies = old
6157 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
6158 .unwrap_or_default();
6159 let new = new.cloned().unwrap_or(CapabilityDeclarations {
6160 provides: Vec::new(),
6161 requires: Vec::new(),
6162 must_never_reach: Vec::new(),
6163 });
6164
6165 new.provides
6166 .iter()
6167 .any(|capability| !old_provides.contains(capability))
6168 || new
6169 .must_never_reach
6170 .iter()
6171 .any(|capability| !old_denies.contains(capability))
6172}
6173
6174fn denied_capability<'a>(
6177 opening_manifest: &'a ModuleManifest,
6178 target_manifest: &ModuleManifest,
6179) -> Option<&'a str> {
6180 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
6181 let target_capabilities = target_manifest.capabilities.as_ref()?;
6182 opening_capabilities
6183 .must_never_reach
6184 .iter()
6185 .find(|denied| {
6186 target_capabilities
6187 .provides
6188 .iter()
6189 .any(|provided| provided == *denied)
6190 })
6191 .map(String::as_str)
6192}
6193
6194fn catalog_update_frozen_field_message(
6195 registered: &ModuleManifest,
6196 provides: &[ProviderRole],
6197) -> Option<String> {
6198 let old_has_provides = !registered.provides.is_empty();
6199 let new_has_provides = !provides.is_empty();
6200 if old_has_provides != new_has_provides {
6201 return Some(format!(
6202 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
6203 registered.module_id
6204 ));
6205 }
6206
6207 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
6208 return Some(format!(
6209 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
6210 registered.module_id
6211 ));
6212 }
6213
6214 let registered_concurrency = manifest_concurrency(registered);
6215 let mut candidate = registered.clone();
6216 candidate.provides = provides.to_vec();
6217 let candidate_concurrency = manifest_concurrency(&candidate);
6218 if candidate_concurrency != registered_concurrency {
6219 return Some(format!(
6220 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
6221 registered.module_id, registered_concurrency, candidate_concurrency
6222 ));
6223 }
6224
6225 None
6228}
6229
6230fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
6231 manifest.provides.iter().any(is_routable_role)
6232}
6233
6234fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
6240 manifest
6241 .provides
6242 .iter()
6243 .find_map(|provider| match provider {
6244 ProviderRole::ToolProvider { concurrency, .. }
6245 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
6246 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
6247 })
6248 .unwrap_or(Concurrency::ModuleManaged)
6249}
6250
6251fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
6258 let has_management_surface = manifest
6259 .provides
6260 .iter()
6261 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
6262 if !has_management_surface {
6263 return false;
6264 }
6265 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
6266 return false;
6267 };
6268 let Some(provides) = raw
6269 .get("manifest")
6270 .and_then(|manifest| manifest.get("provides"))
6271 .and_then(serde_json::Value::as_array)
6272 else {
6273 return false;
6274 };
6275 provides.iter().any(|role| {
6280 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
6281 && role.get("concurrency").is_none()
6282 })
6283}
6284
6285fn negotiate_version(peer_version: u8) -> Result<u8, String> {
6286 if peer_version != PROTOCOL_VERSION {
6287 return Err(format!(
6288 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
6289 ));
6290 }
6291 Ok(PROTOCOL_VERSION)
6292}
6293
6294fn pong(frame: &Frame) -> Result<Frame, RouterError> {
6295 Frame::build_with_version(
6296 response_version(frame),
6297 FrameType::Pong,
6298 frame.header.flags,
6299 0,
6300 0,
6301 frame.header.corr,
6302 Vec::new(),
6303 )
6304 .map_err(RouterError::FrameBuild)
6305}
6306
6307fn control_error_frame(
6308 frame: &Frame,
6309 code: &'static str,
6310 message: impl Into<String>,
6311) -> Result<Frame, RouterError> {
6312 control_error_body_frame(
6313 frame,
6314 ErrorBody {
6315 code: code.to_string(),
6316 message: message.into(),
6317 detail: None,
6318 },
6319 )
6320}
6321
6322fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
6323 let body = serde_json::to_vec(&error).map_err(|err| {
6324 RouterError::backend(
6325 0,
6326 frame.header.corr,
6327 format!("failed to encode control ERROR: {err}"),
6328 )
6329 })?;
6330
6331 Frame::build_with_version(
6332 response_version(frame),
6333 FrameType::Error,
6334 control_flags(),
6335 0,
6336 0,
6337 frame.header.corr,
6338 body,
6339 )
6340 .map_err(RouterError::FrameBuild)
6341}
6342
6343fn control_response_body_frame<T: Serialize>(
6344 frame: &Frame,
6345 reply: &T,
6346 label: &'static str,
6347) -> Result<Frame, RouterError> {
6348 let body = serde_json::to_vec(reply).map_err(|err| {
6349 RouterError::backend(
6350 0,
6351 frame.header.corr,
6352 format!("failed to encode {label}: {err}"),
6353 )
6354 })?;
6355
6356 Frame::build_with_version(
6357 response_version(frame),
6358 FrameType::Response,
6359 control_flags(),
6360 0,
6361 0,
6362 frame.header.corr,
6363 body,
6364 )
6365 .map_err(RouterError::FrameBuild)
6366}
6367
6368fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6398 match err {
6399 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6400 ForwardingError::NoModuleConnection => "target_unavailable",
6401 ForwardingError::ModuleReloading { .. } => "module_reloading",
6402 ForwardingError::ClientRouteChannelExhausted { .. }
6403 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6404 ForwardingError::StaleModuleEndpoint
6405 | ForwardingError::UnknownReservation { .. }
6406 | ForwardingError::ConnectionClosing { .. }
6407 | ForwardingError::ClientEgressClosed { .. }
6408 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
6409 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
6412 ForwardingError::RelayCorrelationExhausted
6413 | ForwardingError::RouteOpenBuild(_)
6414 | ForwardingError::Poisoned => "forwarding_error",
6415 }
6416}
6417
6418fn response_version(frame: &Frame) -> u8 {
6419 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
6420 frame.header.ver
6421 } else {
6422 PROTOCOL_VERSION
6423 }
6424}
6425
6426fn control_flags() -> Flags {
6427 Flags::new(false, Priority::Passive, false)
6428}
6429
6430fn send_goodbye_target_best_effort(
6435 counters: &DaemonCounters,
6436 target: &GoodbyeTarget,
6437 context: &'static str,
6438) {
6439 let Ok(frame) = Frame::build_with_version(
6440 target.negotiated_ver,
6441 FrameType::Goodbye,
6442 control_flags(),
6443 target.channel,
6444 target.epoch,
6445 0,
6446 Vec::new(),
6447 ) else {
6448 return;
6449 };
6450 crate::forwarding::send_module_route_goodbye(
6451 counters,
6452 &target.sink,
6453 frame,
6454 target.module_id.as_deref(),
6455 context,
6456 );
6457}
6458
6459pub(crate) fn send_route_control_pushes(
6460 forwarding: &ForwardingTable,
6461 routes: Vec<EndpointRoute>,
6462 push: ClientControlPush,
6463) {
6464 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
6465 for route in routes {
6466 let target = route.goodbye_target;
6467 if let Some((existing, channels)) = targets
6468 .iter_mut()
6469 .find(|(existing, _)| existing.connection_id == target.connection_id)
6470 {
6471 debug_assert_eq!(
6472 existing.negotiated_ver, target.negotiated_ver,
6473 "one connection cannot negotiate multiple frame versions"
6474 );
6475 if !channels.contains(&target.channel) {
6476 channels.push(target.channel);
6477 }
6478 continue;
6479 }
6480 let channel = target.channel;
6481 targets.push((target, vec![channel]));
6482 }
6483 for (target, mut channels) in targets {
6484 channels.sort_unstable();
6485 let mut push = push.clone();
6486 match &mut push {
6487 ClientControlPush::RouteClosing {
6488 channels: covered, ..
6489 }
6490 | ClientControlPush::RouteClosed {
6491 channels: covered, ..
6492 } => *covered = channels,
6493 }
6494 let body = match serde_json::to_vec(&push) {
6495 Ok(body) => body,
6496 Err(err) => {
6497 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
6498 continue;
6499 }
6500 };
6501 let frame = match Frame::build_with_version(
6502 target.negotiated_ver,
6503 FrameType::Push,
6504 control_flags(),
6505 0,
6506 0,
6507 0,
6508 body.clone(),
6509 ) {
6510 Ok(frame) => frame,
6511 Err(err) => {
6512 warn!(
6513 route_channel = target.channel,
6514 error = %err,
6515 "failed to build route lifecycle control PUSH frame"
6516 );
6517 continue;
6518 }
6519 };
6520 if let Err(err) = target.sink.try_send(frame) {
6521 if target.close_on_delivery_failure() {
6522 warn!(
6523 target_connection_id = target.connection_id.get(),
6524 route_channel = target.channel,
6525 error = %err,
6526 "route lifecycle control PUSH was not delivered to client; closing target connection"
6527 );
6528 let _ = forwarding.escalate_client_delivery_failure(
6529 target.connection_id,
6530 target.channel,
6531 target.epoch,
6532 CloseReason::new(
6533 "route_lifecycle_push_delivery_failed",
6534 format!(
6535 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6536 target.channel
6537 ),
6538 ),
6539 crate::forwarding::UndeliveredFrame {
6540 module_id: target.module_id.as_deref(),
6541 sink: &target.sink,
6542 },
6543 );
6544 }
6545 }
6546 }
6547}
6548
6549#[cfg(test)]
6550mod tests {
6551 #[cfg(unix)]
6552 #[tokio::test]
6553 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
6554 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
6555 let path = dir.join("subc.jsonc");
6556 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
6557 "program":"/bin/sleep","args":["60"],"protocol":"none",
6558 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
6559 }}}).to_string()).unwrap();
6560 let mut configured = crate::daemon_config::load(&path)
6561 .unwrap()
6562 .unwrap()
6563 .modules
6564 .pop()
6565 .unwrap();
6566 let registry = std::sync::Arc::new(crate::Registry::default());
6567 let handle = crate::SupervisorHandle::new();
6568 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
6569 .with_handle(handle.clone());
6570 let module = supervisor
6571 .supervise_configured_with_health(
6572 configured.module_spec(),
6573 true,
6574 configured.health.clone(),
6575 None,
6576 configured.restart,
6577 )
6578 .unwrap();
6579 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
6580 let before = module.status().unwrap().pid;
6581 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
6582 configured.health.cadence = std::time::Duration::from_secs(3600);
6583 let health_only = handler
6584 .reconcile_supervised_modules(&supervisor, vec![configured.clone()], false)
6585 .await
6586 .unwrap();
6587 assert!(
6588 health_only.changed_pending_reload.is_empty(),
6589 "health policy is already applied live"
6590 );
6591 assert_eq!(module.status().unwrap().pid, before);
6592 assert_eq!(
6593 module.configuration().unwrap().1.http,
6594 configured.health.http
6595 );
6596 configured.args = vec!["61".into()];
6597 let launch = handler
6598 .reconcile_supervised_modules(&supervisor, vec![configured], false)
6599 .await
6600 .unwrap();
6601 assert_eq!(launch.changed_pending_reload, ["stock"]);
6602 assert_eq!(
6603 module.status().unwrap().pid,
6604 before,
6605 "a launch edit is stored until reload"
6606 );
6607 module.drain().await.unwrap();
6608 }
6609 use cortexkit_test_support::ScratchDir;
6610 use std::{
6611 collections::BTreeMap,
6612 fmt,
6613 path::PathBuf,
6614 sync::{Arc, Mutex},
6615 time::Duration,
6616 };
6617
6618 use serde_json::{json, Value};
6619 use subc_protocol::{
6620 manifest::{
6621 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6622 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6623 },
6624 session::HealthStatus,
6625 FrameType,
6626 };
6627
6628 use super::*;
6629 use crate::{
6630 forwarding::{DataRoute, DataRouteState},
6631 registry::ChannelState,
6632 router::FrameSink,
6633 stderr_tail::DEFAULT_MAX_LINE_BYTES,
6634 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6635 RouteCtx, Router,
6636 };
6637 use tokio::{
6638 sync::mpsc,
6639 time::{sleep, Instant},
6640 };
6641 use tracing::{
6642 field::{Field, Visit},
6643 Event, Subscriber,
6644 };
6645 use tracing_subscriber::{layer::Context, prelude::*, Layer};
6646
6647 fn fake_aft_stub_path() -> PathBuf {
6666 let mut path = std::env::current_exe().expect("current_exe available in tests");
6667 path.pop(); path.pop(); path.push(if cfg!(windows) {
6670 "fake-aft-stub.exe"
6671 } else {
6672 "fake-aft-stub"
6673 });
6674 assert!(
6675 path.exists(),
6676 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6677 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6678 path.display()
6679 );
6680 path
6681 }
6682
6683 fn client_retries(code: &str) -> bool {
6688 subc_protocol::error_codes::is_retryable_route_open(code)
6689 }
6690
6691 #[test]
6702 fn retryability_of_forwarding_codes_matches_the_failure() {
6703 let transient = [
6706 ForwardingError::NoModuleConnection,
6707 ForwardingError::ModuleReloading {
6708 module_id: "m".into(),
6709 },
6710 ForwardingError::StaleModuleEndpoint,
6711 ForwardingError::UnknownReservation {
6712 client_channel: 1,
6713 module_channel: 1,
6714 },
6715 ForwardingError::ConnectionClosing {
6716 connection_id: ConnectionId::new(1),
6717 },
6718 ForwardingError::ClientEgressClosed {
6719 connection_id: ConnectionId::new(1),
6720 },
6721 ForwardingError::ModuleEgressUnavailable {
6722 connection_id: ConnectionId::new(1),
6723 },
6724 ];
6725 for err in transient {
6726 let code = forwarding_error_code(&err);
6727 assert!(
6728 client_retries(code),
6729 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6730 );
6731 }
6732
6733 let permanent = [
6738 ForwardingError::ConnectionRoleConflict {
6739 connection_id: ConnectionId::new(1),
6740 },
6741 ForwardingError::ClientRouteChannelExhausted {
6742 connection_id: ConnectionId::new(1),
6743 },
6744 ForwardingError::ModuleRouteChannelExhausted {
6745 endpoint: ModuleEndpointId {
6746 connection_id: ConnectionId::new(1),
6747 generation: 1,
6748 },
6749 },
6750 ForwardingError::RelayCorrelationExhausted,
6751 ForwardingError::RouteOpenBuild("x".into()),
6752 ForwardingError::Poisoned,
6753 ];
6754 for err in permanent {
6755 let code = forwarding_error_code(&err);
6756 assert!(
6757 !client_retries(code),
6758 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6759 );
6760 }
6761 }
6762
6763 #[tokio::test]
6791 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6792 let handler = ControlHandler::default();
6793 let frame =
6794 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6795
6796 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6800 assert_eq!(
6801 stamped,
6802 Principal::Direct,
6803 "a caller that proved nothing must not be stamped as a supervised module"
6804 );
6805
6806 let forged = handler
6811 .route_open_principal(
6812 &frame,
6813 Some(ConsumerIdentity {
6814 module_id: "aft".to_string(),
6815 launch_nonce: "not-a-real-nonce".to_string(),
6816 }),
6817 )
6818 .unwrap();
6819 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6820 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6821 }
6822
6823 #[test]
6840 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6841 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6842 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6843 let ClientControlRequest::RouteOpen {
6844 consumer_identity, ..
6845 } = parsed
6846 else {
6847 panic!("route.open body must parse as RouteOpen");
6848 };
6849 assert_eq!(
6850 consumer_identity,
6851 Some(ConsumerIdentity {
6852 module_id: "aft".to_string(),
6853 launch_nonce: "n".to_string(),
6854 }),
6855 "the wire field name must reach the value route_open_principal reads"
6856 );
6857 }
6858
6859 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6860 ModuleManifest::builder(module_id, "0.1.0")
6861 .protocol_ver(protocol_ver)
6862 .provides(vec![ProviderRole::ToolProvider {
6863 tools: vec![Tool {
6864 name: "read".to_string(),
6865 description: None,
6866 execution_mode: ExecutionMode::Pure,
6867 schema: json!({"type": "object"}),
6868 }],
6869 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6870 concurrency: Concurrency::ModuleManaged,
6871 emits_push: true,
6872 sub_supervises: true,
6873 }])
6874 .build()
6875 }
6876
6877 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6878 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6879 }
6880
6881 fn hello_frame_with_control_ops(
6882 module_id: &str,
6883 protocol_ver: u8,
6884 corr: u64,
6885 control_ops: Option<Vec<String>>,
6886 ) -> Frame {
6887 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6888 }
6889
6890 fn hello_frame_with_nonce(
6891 module_id: &str,
6892 protocol_ver: u8,
6893 corr: u64,
6894 launch_nonce: Option<&str>,
6895 ) -> Frame {
6896 hello_frame_full(
6897 module_id,
6898 protocol_ver,
6899 corr,
6900 None,
6901 launch_nonce.map(ToOwned::to_owned),
6902 )
6903 }
6904
6905 fn hello_frame_full(
6906 module_id: &str,
6907 protocol_ver: u8,
6908 corr: u64,
6909 control_ops: Option<Vec<String>>,
6910 launch_nonce: Option<String>,
6911 ) -> Frame {
6912 let body = serde_json::to_vec(&ModuleHelloBody {
6913 manifest: manifest(module_id, protocol_ver),
6914 protocol_ver,
6915 control_ops,
6916 launch_nonce,
6917 })
6918 .unwrap();
6919 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6920 }
6921
6922 fn non_routable_hello_frame_with_control_ops(
6923 module_id: &str,
6924 corr: u64,
6925 control_ops: Option<Vec<String>>,
6926 ) -> Frame {
6927 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6928 manifest.provides.clear();
6929 let body = serde_json::to_vec(&ModuleHelloBody {
6930 manifest,
6931 protocol_ver: PROTOCOL_VERSION,
6932 control_ops,
6933 launch_nonce: None,
6934 })
6935 .unwrap();
6936 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6937 }
6938
6939 fn capability_grammar_hello_frame(
6940 capabilities: Value,
6941 runtime_computed: Option<Value>,
6942 corr: u64,
6943 ) -> Frame {
6944 let mut body = serde_json::to_value(ModuleHelloBody {
6945 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6946 protocol_ver: PROTOCOL_VERSION,
6947 control_ops: None,
6948 launch_nonce: None,
6949 })
6950 .expect("HELLO body serializes");
6951 body["manifest"]["capabilities"] = capabilities;
6952 if let Some(runtime_computed) = runtime_computed {
6953 body["runtime_computed"] = runtime_computed;
6954 }
6955 Frame::build(
6956 FrameType::Hello,
6957 control_flags(),
6958 0,
6959 0,
6960 corr,
6961 serde_json::to_vec(&body).expect("HELLO body reserializes"),
6962 )
6963 .expect("HELLO frame builds")
6964 }
6965
6966 fn channel_request(channel: u16, corr: u64) -> Frame {
6967 Frame::build(
6968 FrameType::Request,
6969 Flags::new(true, Priority::Interactive, false),
6970 channel,
6971 0,
6972 corr,
6973 b"opaque".to_vec(),
6974 )
6975 .unwrap()
6976 }
6977
6978 fn route_ctx(
6979 connection_id: ConnectionId,
6980 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
6981 let (tx, rx) = mpsc::channel(8);
6982 (
6983 RouteCtx {
6984 connection_id,
6985 egress: FrameSink::new(tx),
6986 },
6987 rx,
6988 )
6989 }
6990
6991 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
6992 serde_json::from_slice(&frame.body).unwrap()
6993 }
6994
6995 async fn hello_via_sink(
7000 handler: &ControlHandler,
7001 ctx: &RouteCtx,
7002 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7003 hello: Frame,
7004 ) -> Frame {
7005 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7006 assert!(
7007 replies.is_empty(),
7008 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7009 );
7010 let ack = rx
7011 .try_recv()
7012 .expect("HELLO_ACK is queued on the module sink")
7013 .frame;
7014 assert_eq!(ack.header.ty, FrameType::HelloAck);
7015 ack
7016 }
7017
7018 fn parse_error(frame: &Frame) -> Value {
7019 serde_json::from_slice(&frame.body).unwrap()
7020 }
7021
7022 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7023 serde_json::from_slice(&frame.body).unwrap()
7024 }
7025
7026 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7027 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7028 route_channel,
7029 route_epoch: 0,
7030 kind,
7031 })
7032 .unwrap();
7033 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7034 }
7035
7036 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7037 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7038 module_id: module_id.to_string(),
7039 })
7040 .unwrap();
7041 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7042 }
7043
7044 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7045 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7046 }
7047
7048 fn route_open_frame_with_consumer_capabilities(
7049 corr: u64,
7050 module_id: &str,
7051 project_root: ScratchDir,
7052 consumer_capabilities: Option<Vec<String>>,
7053 ) -> Frame {
7054 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7055 target: RouteTarget::ToolProvider {
7056 module_id: module_id.to_string(),
7057 },
7058 identity: BindIdentity::new(
7059 project_root.path().to_path_buf(),
7060 "unit".to_string(),
7061 "session".to_string(),
7062 ),
7063 consumer_identity: None,
7064 consumer_capabilities,
7065 role_versions: None,
7066 admission_facts: None,
7067 scope: None,
7068 })
7069 .unwrap();
7070 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7071 }
7072
7073 fn route_open_frame_with_role_versions(
7074 corr: u64,
7075 module_id: &str,
7076 project_root: ScratchDir,
7077 role_versions: Option<BTreeMap<String, String>>,
7078 ) -> Frame {
7079 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7080 target: RouteTarget::ToolProvider {
7081 module_id: module_id.to_string(),
7082 },
7083 identity: BindIdentity::new(
7084 project_root.path().to_path_buf(),
7085 "unit".to_string(),
7086 format!("session-{corr}"),
7087 ),
7088 consumer_identity: None,
7089 consumer_capabilities: None,
7090 role_versions,
7091 admission_facts: None,
7092 scope: None,
7093 })
7094 .unwrap();
7095 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7096 }
7097
7098 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
7099 entries
7100 .iter()
7101 .map(|(role, version)| (role.to_string(), version.to_string()))
7102 .collect()
7103 }
7104
7105 fn route_open_frame_with_admission_facts(
7106 corr: u64,
7107 module_id: &str,
7108 project_root: ScratchDir,
7109 consumer_identity: Option<subc_control::ConsumerIdentity>,
7110 facts: Option<Value>,
7111 ) -> Frame {
7112 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7113 target: RouteTarget::ToolProvider {
7114 module_id: module_id.to_string(),
7115 },
7116 identity: BindIdentity::new(
7117 project_root.path().to_path_buf(),
7118 "unit".to_string(),
7119 format!("session-{corr}"),
7120 ),
7121 consumer_identity,
7122 consumer_capabilities: None,
7123 role_versions: None,
7124 admission_facts: facts,
7125 scope: None,
7126 })
7127 .unwrap();
7128 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7129 }
7130
7131 #[derive(Clone, Default)]
7132 struct EventCapture {
7133 events: Arc<Mutex<Vec<CapturedEvent>>>,
7134 }
7135
7136 #[derive(Clone, Debug)]
7137 struct CapturedEvent {
7138 target: String,
7139 level: tracing::Level,
7140 fields: BTreeMap<String, String>,
7141 }
7142
7143 impl EventCapture {
7144 fn events(&self) -> Vec<CapturedEvent> {
7145 self.events.lock().unwrap().clone()
7146 }
7147 }
7148
7149 impl<S> Layer<S> for EventCapture
7150 where
7151 S: Subscriber,
7152 {
7153 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
7154 let mut visitor = EventFieldVisitor::default();
7155 event.record(&mut visitor);
7156 self.events.lock().unwrap().push(CapturedEvent {
7157 target: event.metadata().target().to_string(),
7158 level: *event.metadata().level(),
7159 fields: visitor.fields,
7160 });
7161 }
7162 }
7163
7164 #[derive(Default)]
7165 struct EventFieldVisitor {
7166 fields: BTreeMap<String, String>,
7167 }
7168
7169 impl Visit for EventFieldVisitor {
7170 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
7171 self.fields
7172 .insert(field.name().to_string(), format!("{value:?}"));
7173 }
7174 }
7175
7176 fn health_response(corr: u64, status: HealthStatus) -> Frame {
7177 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
7178 status,
7179 detail: Some("warming".to_string()),
7180 metrics: Some(json!({"queue_depth": 3})),
7181 })
7182 .unwrap();
7183 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7184 }
7185
7186 fn route_bind_ack(corr: u64) -> Frame {
7187 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
7188 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7189 }
7190
7191 fn unique_project_root(label: &str) -> ScratchDir {
7192 ScratchDir::new(label)
7193 }
7194
7195 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
7196 match parse_route_poll(frame) {
7197 ClientControlResponse::RoutePoll {
7198 status: None,
7199 live: Some(live),
7200 ..
7201 } => assert_eq!(live, expected_live),
7202 other => panic!("unexpected route.poll response: {other:?}"),
7203 }
7204 }
7205
7206 fn bind_liveness_route(
7207 registry: &Registry,
7208 forwarding: &ForwardingTable,
7209 module_id: &str,
7210 ) -> (RouteCtx, u16, u32) {
7211 let module_connection = ConnectionId::new(101);
7212 let client_connection = ConnectionId::new(202);
7213 let registration = registry
7214 .register_with_control_ops(
7215 manifest(module_id, PROTOCOL_VERSION),
7216 PROTOCOL_VERSION,
7217 module_connection,
7218 module_baseline_control_ops(),
7219 )
7220 .unwrap();
7221 let (module_tx, _module_rx) = mpsc::channel(8);
7222 let endpoint = forwarding
7223 .register_module_connection(
7224 module_connection,
7225 module_id.to_string(),
7226 PROTOCOL_VERSION,
7227 manifest_concurrency(®istration.manifest),
7228 FrameSink::new(module_tx),
7229 )
7230 .unwrap();
7231 let (client_ctx, _client_rx) = route_ctx(client_connection);
7232 let pending = forwarding
7233 .begin_route_bind_relay_for_test(
7234 client_connection,
7235 client_ctx.egress.clone(),
7236 1,
7237 module_id,
7238 )
7239 .unwrap();
7240 assert_eq!(pending.endpoint, endpoint);
7241 let route_channel = pending.client_channel;
7242 let route_epoch = pending.client_epoch;
7243 forwarding
7244 .complete_pending_relay(
7245 module_connection,
7246 pending.corr,
7247 RouteBindRelayOutcome::Accepted,
7248 )
7249 .unwrap();
7250 (client_ctx, route_channel, route_epoch)
7251 }
7252
7253 struct FakeProcessLiveness {
7254 live: Option<bool>,
7255 }
7256
7257 impl ModuleProcessLiveness for FakeProcessLiveness {
7258 fn process_live(&self, _module_id: &str) -> Option<bool> {
7259 self.live
7260 }
7261 }
7262
7263 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7264 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
7265 {
7266 let registry = Arc::new(Registry::default());
7267 let supervisor_handle = SupervisorHandle::new();
7268 let supervisor = Supervisor::new_for_test(
7269 Arc::clone(®istry),
7270 RestartPolicy::new(1, Duration::from_millis(10)),
7271 )
7272 .with_handle(supervisor_handle.clone());
7273 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
7274 let module = supervisor
7275 .spawn(ModuleSpec {
7276 module_id: "stderr-tail-wire".to_string(),
7277 program: fake_aft_stub_path(),
7278 args: Vec::new(),
7279 env: vec![
7280 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
7281 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
7282 ],
7283 reserved: false,
7284 reserved_prefixes: Vec::new(),
7285 protocol: ModuleProtocol::Subc,
7286 overlap: Default::default(),
7287 })
7288 .unwrap();
7289
7290 let deadline = Instant::now() + Duration::from_secs(5);
7291 loop {
7292 let tail = module.stderr_tail(None, None);
7293 if tail
7294 .entries
7295 .iter()
7296 .any(|entry| matches!(entry, TailEntry::ProcessStart))
7297 && tail.entries.iter().any(|entry| {
7298 matches!(
7299 entry,
7300 TailEntry::Line {
7301 truncated: true,
7302 ..
7303 }
7304 )
7305 })
7306 {
7307 break;
7308 }
7309 assert!(
7310 Instant::now() < deadline,
7311 "module did not produce a truncated line and restart boundary: {tail:?}"
7312 );
7313 sleep(Duration::from_millis(10)).await;
7314 }
7315
7316 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7317 let request = ClientControlRequest::SupervisorStderrTail {
7318 module_id: "stderr-tail-wire".to_string(),
7319 max_lines: None,
7320 max_bytes: None,
7321 };
7322 let frame = Frame::build(
7323 FrameType::Request,
7324 control_flags(),
7325 0,
7326 0,
7327 1,
7328 serde_json::to_vec(&request).unwrap(),
7329 )
7330 .unwrap();
7331 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7332 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7333 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
7334 serde_json::from_slice(&responses[0].body).unwrap()
7335 else {
7336 panic!("expected supervisor.stderr_tail response");
7337 };
7338
7339 assert!(
7340 tail.entries
7341 .iter()
7342 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
7343 "the control response lost the restart boundary"
7344 );
7345 let Some(StderrTailEntry::Line {
7346 text,
7347 truncated,
7348 at_ms,
7349 }) = tail.entries.iter().find(|entry| {
7350 matches!(
7351 entry,
7352 StderrTailEntry::Line {
7353 truncated: true,
7354 ..
7355 }
7356 )
7357 })
7358 else {
7359 panic!("the control response lost the truncated line");
7360 };
7361 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
7362 assert!(*truncated);
7363 assert!(
7364 at_ms.is_some(),
7365 "the control response lost the line's capture time"
7366 );
7367 }
7368
7369 #[tokio::test(flavor = "current_thread")]
7374 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
7375 let dir = ScratchDir::new("terminals-off-worker");
7376 let journal_path = dir.join("terminals.jsonl");
7377 let registry = Arc::new(Registry::default());
7378 let supervisor_handle = SupervisorHandle::new();
7379 let supervisor =
7380 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7381 .with_handle(supervisor_handle.clone())
7382 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7383 let module = supervisor
7384 .spawn(ModuleSpec {
7385 module_id: "terminal-off-worker".to_string(),
7386 program: fake_aft_stub_path(),
7387 args: Vec::new(),
7388 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7389 reserved: false,
7390 reserved_prefixes: Vec::new(),
7391 protocol: ModuleProtocol::Subc,
7392 overlap: Default::default(),
7393 })
7394 .unwrap();
7395 let deadline = Instant::now() + Duration::from_secs(5);
7396 while module.terminal_history().entries.len() != 2 {
7397 assert!(Instant::now() < deadline, "module did not record two exits");
7398 sleep(Duration::from_millis(10)).await;
7399 }
7400
7401 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
7402 let handler =
7403 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
7404 let frame = Frame::build(
7405 FrameType::Request,
7406 control_flags(),
7407 0,
7408 0,
7409 1,
7410 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
7411 module_id: "terminal-off-worker".to_string(),
7412 })
7413 .unwrap(),
7414 )
7415 .unwrap();
7416 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7417 let spawned_at = std::time::Instant::now();
7418 let read = tokio::spawn({
7419 let handler = Arc::clone(&handler);
7420 async move { handler.handle_control_frame(&ctx, frame).await }
7421 });
7422 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
7425 .await
7426 .unwrap()
7427 .expect("the history read reached its pause");
7428 let elapsed = spawned_at.elapsed();
7429 assert!(
7430 elapsed < Duration::from_secs(2) && !read.is_finished(),
7431 "this task could not run while the history read was paused \
7432 (resumed after {elapsed:?}, read finished: {})",
7433 read.is_finished()
7434 );
7435
7436 drop(release);
7437 let responses = read.await.unwrap().unwrap();
7438 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7439 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
7440 panic!("expected supervisor.terminals response");
7441 };
7442 assert_eq!(terminals.entries.len(), 2);
7443 assert_eq!(terminals.journal_skipped_lines, 0);
7444 assert_eq!(terminals.journal_read_errors, 0);
7445 }
7446
7447 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7448 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
7449 let registry = Arc::new(Registry::default());
7450 let supervisor_handle = SupervisorHandle::new();
7451 let supervisor =
7452 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7453 .with_handle(supervisor_handle.clone());
7454 let module = supervisor
7455 .spawn(ModuleSpec {
7456 module_id: "terminal-golden".to_string(),
7457 program: fake_aft_stub_path(),
7458 args: Vec::new(),
7459 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7460 reserved: false,
7461 reserved_prefixes: Vec::new(),
7462 protocol: ModuleProtocol::Subc,
7463 overlap: Default::default(),
7464 })
7465 .unwrap();
7466
7467 let deadline = Instant::now() + Duration::from_secs(5);
7468 while module.terminal_history().entries.len() != 2 {
7469 assert!(
7470 Instant::now() < deadline,
7471 "module did not retain two terminal exits: {:?}",
7472 module.terminal_history()
7473 );
7474 sleep(Duration::from_millis(10)).await;
7475 }
7476
7477 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7478 let request = ClientControlRequest::SupervisorTerminals {
7479 module_id: "terminal-golden".to_string(),
7480 };
7481 let frame = Frame::build(
7482 FrameType::Request,
7483 control_flags(),
7484 0,
7485 0,
7486 1,
7487 serde_json::to_vec(&request).unwrap(),
7488 )
7489 .unwrap();
7490 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7491 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7492 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7493 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
7494 panic!("expected supervisor.terminals response");
7495 };
7496 assert_eq!(terminals.entries.len(), 2);
7497 assert_eq!(terminals.dropped, 0);
7498
7499 let mut rendered = serde_json::to_value(response).unwrap();
7500 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
7503 for (index, entry) in rendered["entries"]
7504 .as_array_mut()
7505 .expect("terminal response entries array")
7506 .iter_mut()
7507 .enumerate()
7508 {
7509 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
7510 }
7511
7512 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7513 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
7514 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
7515 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7516 std::fs::write(&golden_path, &serialized).unwrap();
7517 }
7518 let expected: Value =
7519 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
7520 assert_eq!(rendered, expected);
7521 }
7522
7523 #[test]
7524 fn hello_registers_manifest_and_returns_ack() {
7525 let registry = Arc::new(Registry::default());
7526 let handler = ControlHandler::new(Arc::clone(®istry));
7527 let conn = ConnectionId::new(1);
7528
7529 let responses = handler
7530 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
7531 .unwrap();
7532
7533 assert_eq!(responses.len(), 1);
7534 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7535 assert_eq!(responses[0].header.channel, 0);
7536 assert_eq!(responses[0].header.corr, 7);
7537 let ack = parse_ack(&responses[0]);
7538 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
7539 assert!(ack
7540 .subc_capabilities
7541 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
7542 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
7543 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
7544 assert!(ack
7545 .subc_ops
7546 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
7547 assert!(ack
7548 .subc_ops
7549 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
7550
7551 let registration = registry.get_module("aft").unwrap().unwrap();
7552 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
7553 assert_eq!(registration.state, ChannelState::Active);
7554 assert_eq!(registration.connection_id, conn);
7555 assert_eq!(registration.control_ops, module_baseline_control_ops());
7556 }
7557
7558 #[test]
7559 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7560 let invalid_identifiers = [
7561 ("case_change", "credentials-Provider/v1"),
7562 ("leading_zero", "credentials-provider/v01"),
7563 ("trailing_hyphen", "credentials-provider-/v1"),
7564 ("consecutive_hyphens", "credentials--provider/v1"),
7565 ("uppercase", "Credentials-provider/v1"),
7566 ("missing_v", "credentials-provider/1"),
7567 ("whitespace", "credentials provider/v1"),
7568 ("zero_version", "credentials-provider/v0"),
7569 ("out_of_range_version", "credentials-provider/v4294967296"),
7570 (
7571 "overlength_name",
7572 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7573 ),
7574 ];
7575 let mut cases = invalid_identifiers
7576 .into_iter()
7577 .map(|(name, identifier)| {
7578 (
7579 format!("identifier_{name}"),
7580 "capabilities.provides[0]".to_string(),
7581 identifier.to_string(),
7582 json!({ "provides": [identifier] }),
7583 None,
7584 )
7585 })
7586 .collect::<Vec<_>>();
7587 cases.extend([
7588 (
7589 "unknown_need".to_string(),
7590 "capabilities.requires[0].need".to_string(),
7591 "deferred".to_string(),
7592 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7593 None,
7594 ),
7595 (
7596 "duplicate_provides".to_string(),
7597 "capabilities.provides[1]".to_string(),
7598 "credentials-provider/v1".to_string(),
7599 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7600 None,
7601 ),
7602 (
7603 "duplicate_must_never_reach".to_string(),
7604 "capabilities.must_never_reach[1]".to_string(),
7605 "credentials-provider/v1".to_string(),
7606 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7607 None,
7608 ),
7609 (
7610 "duplicate_requires_same_need".to_string(),
7611 "capabilities.requires[1]".to_string(),
7612 "credentials-provider/v1".to_string(),
7613 json!({ "requires": [
7614 { "capability": "credentials-provider/v1", "need": "required" },
7615 { "capability": "credentials-provider/v1", "need": "required" }
7616 ] }),
7617 None,
7618 ),
7619 (
7620 "duplicate_requires_conflicting_need".to_string(),
7621 "capabilities.requires[1]".to_string(),
7622 "credentials-provider/v1".to_string(),
7623 json!({ "requires": [
7624 { "capability": "credentials-provider/v1", "need": "required" },
7625 { "capability": "credentials-provider/v1", "need": "optional" }
7626 ] }),
7627 None,
7628 ),
7629 (
7630 "capabilities_root_pointer".to_string(),
7631 "runtime_computed[0]".to_string(),
7632 "/capabilities".to_string(),
7633 json!({}),
7634 Some(json!(["/capabilities"])),
7635 ),
7636 (
7637 "capabilities_descendant_pointer".to_string(),
7638 "runtime_computed[0]".to_string(),
7639 "/capabilities/provides".to_string(),
7640 json!({}),
7641 Some(json!(["/capabilities/provides"])),
7642 ),
7643 (
7644 "malformed_pointer_without_leading_slash".to_string(),
7645 "runtime_computed[0]".to_string(),
7646 "capabilities".to_string(),
7647 json!({}),
7648 Some(json!(["capabilities"])),
7649 ),
7650 (
7651 "malformed_pointer_escape".to_string(),
7652 "runtime_computed[0]".to_string(),
7653 "/roles/~2/tools".to_string(),
7654 json!({}),
7655 Some(json!(["/roles/~2/tools"])),
7656 ),
7657 (
7658 "unknown_capabilities_field".to_string(),
7659 "capabilities.future".to_string(),
7660 "<array>".to_string(),
7661 json!({ "future": [] }),
7662 None,
7663 ),
7664 ]);
7665
7666 for (index, (name, field, value, capabilities, runtime_computed)) in
7667 cases.into_iter().enumerate()
7668 {
7669 let registry = Arc::new(Registry::default());
7670 let handler = ControlHandler::new(Arc::clone(®istry));
7671 let response = handler
7672 .handle_control(
7673 ConnectionId::new((index + 1) as u64),
7674 capability_grammar_hello_frame(
7675 capabilities,
7676 runtime_computed,
7677 index as u64 + 1,
7678 ),
7679 )
7680 .expect("invalid HELLO returns a refusal");
7681
7682 assert_eq!(response.len(), 1, "{name} must emit one refusal");
7683 let error = parse_error(&response[0]);
7684 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7685 let message = error["message"]
7686 .as_str()
7687 .expect("error message is a string");
7688 assert!(
7689 message.contains(&field),
7690 "{name}: field missing from {message}"
7691 );
7692 assert!(
7693 message.contains(&value),
7694 "{name}: value missing from {message}"
7695 );
7696 assert_eq!(
7697 registry
7698 .active_registration_count()
7699 .expect("registry reads"),
7700 0,
7701 "{name}: refused HELLO must not create a catalog entry"
7702 );
7703 }
7704 }
7705
7706 #[test]
7707 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7708 let registry = Arc::new(Registry::default());
7709 let handler = ControlHandler::new(Arc::clone(®istry));
7710 let capabilities = json!({
7711 "provides": ["credentials-provider/v1"],
7712 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7713 "must_never_reach": ["federation-transport/v1"]
7714 });
7715 let response = handler
7716 .handle_control(
7717 ConnectionId::new(99),
7718 capability_grammar_hello_frame(
7719 capabilities.clone(),
7720 Some(json!(["/roles/0/tools"])),
7721 99,
7722 ),
7723 )
7724 .expect("valid HELLO registers");
7725 assert_eq!(response[0].header.ty, FrameType::HelloAck);
7726
7727 let request = Frame::build(
7728 FrameType::Request,
7729 control_flags(),
7730 0,
7731 0,
7732 100,
7733 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7734 .expect("catalog request serializes"),
7735 )
7736 .expect("catalog request frame builds");
7737 let response = handler
7738 .handle_catalog_list(request, None)
7739 .expect("catalog list succeeds");
7740 let ClientControlResponse::CatalogList { modules, .. } =
7741 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7742 else {
7743 panic!("catalog request must return catalog.list");
7744 };
7745 assert_eq!(modules.len(), 1);
7746 assert_eq!(
7747 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7748 capabilities
7749 );
7750 }
7751
7752 #[test]
7753 fn catalog_list_mirrors_management_operation_description() {
7754 let registry = Arc::new(Registry::default());
7755 let handler = ControlHandler::new(Arc::clone(®istry));
7756 let description = "List managed records and return their identifiers and metadata.";
7757 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7758 manifest.provides = vec![ProviderRole::ManagementSurface {
7759 operations: vec![ManagementOperation {
7760 name: "records.list".to_string(),
7761 kind: ManagementOperationKind::Query,
7762 description: Some(description.to_string()),
7763 }],
7764 config_schema: json!({"type": "object"}),
7765 observability: vec![ObservabilitySurface {
7766 name: "records.stats".to_string(),
7767 kind: ObservabilityKind::Snapshot,
7768 }],
7769 identity_scope: vec![IdentityScope::Project],
7770 concurrency: Concurrency::ModuleManaged,
7771 }];
7772 registry
7773 .register_with_control_ops(
7774 manifest,
7775 PROTOCOL_VERSION,
7776 ConnectionId::new(99),
7777 Vec::new(),
7778 )
7779 .expect("described management manifest registers");
7780
7781 let request = Frame::build(
7782 FrameType::Request,
7783 control_flags(),
7784 0,
7785 0,
7786 100,
7787 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7788 .expect("catalog request serializes"),
7789 )
7790 .expect("catalog request frame builds");
7791 let response = handler
7792 .handle_catalog_list(request, None)
7793 .expect("catalog list succeeds");
7794 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7795 assert_eq!(
7796 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7797 "catalog.list must preserve the declared operation description verbatim"
7798 );
7799 }
7800
7801 #[test]
7802 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7803 let registry = Arc::new(Registry::default());
7804 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7805 [("vault".to_string(), true), ("squatter".to_string(), true)],
7806 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7807 );
7808 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7809 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7810 provides: vec!["credentials-provider/v1".to_string()],
7811 requires: Vec::new(),
7812 must_never_reach: Vec::new(),
7813 });
7814 let frame = Frame::build(
7815 FrameType::Hello,
7816 control_flags(),
7817 0,
7818 0,
7819 77,
7820 serde_json::to_vec(&ModuleHelloBody {
7821 manifest: squatter,
7822 protocol_ver: PROTOCOL_VERSION,
7823 control_ops: None,
7824 launch_nonce: None,
7825 })
7826 .expect("HELLO serializes"),
7827 )
7828 .expect("HELLO frame builds");
7829 let response = handler
7830 .handle_control(ConnectionId::new(77), frame)
7831 .expect("reserved claim receives a typed refusal");
7832 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7833 assert_eq!(
7834 registry
7835 .active_registration_count()
7836 .expect("registry reads"),
7837 0,
7838 "a reserved capability refusal must not leave a catalog entry"
7839 );
7840 }
7841
7842 #[test]
7843 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
7844 for settlement in ["timeout", "inconclusive", "drop"] {
7845 let breakers = RouteBindBreakers::default();
7846 let RouteBindAdmission::Admitted {
7847 guard: mut old,
7848 probe: false,
7849 } = breakers.admit("prov")
7850 else {
7851 panic!("ordinary relay admitted")
7852 };
7853 let RouteBindAdmission::Admitted {
7854 guard: mut opener, ..
7855 } = breakers.admit("prov")
7856 else {
7857 panic!("second relay admitted")
7858 };
7859 assert!(
7860 !opener
7861 .record_timeout(1, Duration::ZERO)
7862 .unwrap()
7863 .reopened_after_probe
7864 );
7865 let RouteBindAdmission::Admitted {
7866 guard: mut probe,
7867 probe: true,
7868 } = breakers.admit("prov")
7869 else {
7870 panic!("one cooldown probe admitted")
7871 };
7872 match settlement {
7873 "timeout" => assert!(
7874 !old.record_timeout(1, Duration::ZERO)
7875 .unwrap()
7876 .reopened_after_probe
7877 ),
7878 "inconclusive" => old.record_inconclusive(),
7879 "drop" => drop(old),
7880 _ => unreachable!(),
7881 }
7882 assert!(
7883 matches!(
7884 breakers.admit("prov"),
7885 RouteBindAdmission::Refused {
7886 probe_in_flight: true,
7887 ..
7888 }
7889 ),
7890 "{settlement} of a pre-open relay cannot release the real probe"
7891 );
7892 assert!(
7893 probe
7894 .record_timeout(1, Duration::ZERO)
7895 .unwrap()
7896 .reopened_after_probe
7897 );
7898 assert!(matches!(
7899 breakers.admit("prov"),
7900 RouteBindAdmission::Admitted { probe: true, .. }
7901 ));
7902 }
7903 let breakers = RouteBindBreakers::default();
7904 let admit = || match breakers.admit("prov") {
7905 RouteBindAdmission::Admitted { guard, .. } => guard,
7906 _ => panic!("relay admitted"),
7907 };
7908 admit().record_timeout(1, Duration::ZERO);
7909 let mut old_probe = admit();
7910 breakers.reset_for_new_module_connection("prov");
7911 admit().record_timeout(1, Duration::ZERO);
7912 let _new_probe = admit();
7913 old_probe.record_inconclusive();
7914 assert!(matches!(
7915 breakers.admit("prov"),
7916 RouteBindAdmission::Refused {
7917 probe_in_flight: true,
7918 ..
7919 }
7920 ));
7921 }
7922
7923 #[tokio::test]
7924 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
7925 for candidate in [false, true] {
7926 let registry = Arc::new(Registry::default());
7927 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7928 [("vault".to_string(), true), ("squatter".to_string(), true)],
7929 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7930 );
7931 let conn = ConnectionId::new(77);
7932 let (ctx, mut rx) = route_ctx(conn);
7933 let initial = capability_manifest("squatter", &[], &[]);
7934 if candidate {
7935 registry
7936 .register_candidate_with_control_ops(
7937 initial.clone(),
7938 PROTOCOL_VERSION,
7939 conn,
7940 module_baseline_control_ops(),
7941 )
7942 .unwrap();
7943 handler
7944 .forwarding
7945 .register_candidate_module_connection(
7946 conn,
7947 "squatter".to_string(),
7948 PROTOCOL_VERSION,
7949 manifest_concurrency(&initial),
7950 ctx.egress.clone(),
7951 )
7952 .unwrap();
7953 } else {
7954 hello_via_sink(
7955 &handler,
7956 &ctx,
7957 &mut rx,
7958 hello_frame_with_manifest(initial.clone(), 1),
7959 )
7960 .await;
7961 }
7962 let response = handler
7963 .handle_control_frame(
7964 &ctx,
7965 catalog_update_with_capabilities_frame(
7966 2,
7967 capability_manifest("squatter", &["credentials-provider/v1"], &[])
7968 .capabilities
7969 .unwrap(),
7970 ),
7971 )
7972 .await
7973 .unwrap();
7974 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7975 assert_eq!(
7976 registry
7977 .get_module_by_connection(conn)
7978 .unwrap()
7979 .unwrap()
7980 .manifest,
7981 initial
7982 );
7983 }
7984 }
7985
7986 #[test]
7987 fn server_describe_surfaces_required_capability_verdict_fields() {
7988 let registry = Arc::new(Registry::default());
7989 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7990 [
7991 ("consumer".to_string(), true),
7992 ("provider".to_string(), false),
7993 ],
7994 BTreeMap::new(),
7995 );
7996 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
7997 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7998 provides: Vec::new(),
7999 requires: vec![subc_protocol::manifest::CapabilityRequirement {
8000 capability: "credentials-provider/v1".to_string(),
8001 need: subc_protocol::manifest::CapabilityNeed::Required,
8002 }],
8003 must_never_reach: Vec::new(),
8004 });
8005 let hello = Frame::build(
8006 FrameType::Hello,
8007 control_flags(),
8008 0,
8009 0,
8010 78,
8011 serde_json::to_vec(&ModuleHelloBody {
8012 manifest: consumer,
8013 protocol_ver: PROTOCOL_VERSION,
8014 control_ops: None,
8015 launch_nonce: None,
8016 })
8017 .expect("HELLO serializes"),
8018 )
8019 .expect("HELLO frame builds");
8020 handler
8021 .handle_control(ConnectionId::new(78), hello)
8022 .expect("consumer registers");
8023 let describe = Frame::build(
8024 FrameType::Request,
8025 control_flags(),
8026 0,
8027 0,
8028 79,
8029 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8030 .expect("request serializes"),
8031 )
8032 .expect("describe frame builds");
8033 let response = handler
8034 .handle_server_describe(describe)
8035 .expect("server.describe succeeds");
8036 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8037 let requirement = &rendered["capability_requirements"][0];
8038 assert_eq!(requirement["consumer"], "consumer");
8039 assert_eq!(requirement["verdict"], "never_provided");
8040 assert_eq!(requirement["episode_seq"], 1);
8041 assert_eq!(requirement["config_satisfiable"], false);
8042 assert_eq!(requirement["runtime_available"], false);
8043 assert!(requirement["detail"]
8044 .as_str()
8045 .expect("detail string")
8046 .contains("credentials-provider/v1"));
8047 }
8048
8049 #[test]
8050 fn catalog_list_omits_capabilities_for_legacy_manifest() {
8051 let registry = Arc::new(Registry::default());
8052 let handler = ControlHandler::new(Arc::clone(®istry));
8053 let hello = handler
8054 .handle_control(
8055 ConnectionId::new(101),
8056 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
8057 )
8058 .expect("legacy HELLO registers");
8059 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
8060
8061 let request = Frame::build(
8062 FrameType::Request,
8063 control_flags(),
8064 0,
8065 0,
8066 102,
8067 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8068 .expect("catalog request serializes"),
8069 )
8070 .expect("catalog request frame builds");
8071 let response = handler
8072 .handle_catalog_list(request, None)
8073 .expect("catalog list succeeds");
8074 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8075 assert!(
8076 body["modules"][0].get("capabilities").is_none(),
8077 "legacy manifest must retain an absent capabilities field on catalog.list"
8078 );
8079 }
8080
8081 #[test]
8082 fn hello_ack_omits_storage_when_no_storage_config() {
8083 let registry = Arc::new(Registry::default());
8084 let handler = ControlHandler::new(Arc::clone(®istry));
8085 let responses = handler
8086 .handle_control(
8087 ConnectionId::new(1),
8088 hello_frame("aft", PROTOCOL_VERSION, 7),
8089 )
8090 .unwrap();
8091 let ack = parse_ack(&responses[0]);
8092 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
8093 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
8094 }
8095
8096 #[tokio::test]
8097 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
8098 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
8099 let registry = Arc::new(Registry::default());
8100 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
8101 let responses = handler
8102 .handle_control(
8103 ConnectionId::new(1),
8104 hello_frame("aft", PROTOCOL_VERSION, 7),
8105 )
8106 .unwrap();
8107 let ack = parse_ack(&responses[0]);
8108 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
8109
8110 let described = handler
8111 .handle_control_frame(
8112 &route_ctx(ConnectionId::new(2)).0,
8113 Frame::build(
8114 FrameType::Request,
8115 control_flags(),
8116 0,
8117 0,
8118 9,
8119 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8120 )
8121 .unwrap(),
8122 )
8123 .await
8124 .unwrap();
8125 let ClientControlResponse::ServerDescribe { machine_id, .. } =
8126 serde_json::from_slice(&described[0].body).unwrap()
8127 else {
8128 panic!("server.describe answered with another shape");
8129 };
8130 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
8131 }
8132
8133 #[test]
8134 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
8135 let registry = Arc::new(Registry::default());
8138 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
8139 crate::daemon_config::StorageConfig::Sqlite {
8140 data_home: std::path::PathBuf::from("/data"),
8141 },
8142 ));
8143
8144 let responses = handler
8145 .handle_control(
8146 ConnectionId::new(1),
8147 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
8148 )
8149 .unwrap();
8150 let ack = parse_ack(&responses[0]);
8151 assert_eq!(
8152 ack.storage,
8153 Some(serde_json::json!({
8154 "module_id": "alfonso-routing",
8155 "storage_namespace": "default",
8156 "isolation": { "kind": "module" },
8157 "backend": {
8158 "backend": "sqlite",
8159 "path": "/data/cortexkit/alfonso-routing/store.db"
8160 }
8161 })),
8162 "the delivered descriptor is the module's own sqlite store path"
8163 );
8164 }
8165
8166 #[test]
8167 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
8168 let registry = Arc::new(Registry::default());
8169 let handler = ControlHandler::new(Arc::clone(®istry));
8170 let conn = ConnectionId::new(1);
8171 let responses = handler
8172 .handle_control(
8173 conn,
8174 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8175 )
8176 .unwrap();
8177 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8178 let registration = registry.get_module("aft").unwrap().unwrap();
8179 assert_eq!(registration.control_ops, module_baseline_control_ops());
8180
8181 let frame =
8182 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
8183 assert!(handler
8184 .guard_module_control_op(&frame, "aft", "route.bind")
8185 .unwrap()
8186 .is_none());
8187 let error = handler
8188 .guard_module_control_op(&frame, "aft", "test.synthetic")
8189 .unwrap()
8190 .expect("synthetic ungranted op should be rejected");
8191 assert_eq!(error.header.ty, FrameType::Error);
8192 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
8193 }
8194
8195 #[test]
8196 fn hello_control_ops_some_adds_optional_grants() {
8197 let registry = Arc::new(Registry::default());
8198 let handler = ControlHandler::new(Arc::clone(®istry));
8199 handler
8200 .handle_control(
8201 ConnectionId::new(1),
8202 hello_frame_with_control_ops(
8203 "aft",
8204 PROTOCOL_VERSION,
8205 7,
8206 Some(vec![
8207 "future.synthetic".to_string(),
8208 "route.bind".to_string(),
8209 ]),
8210 ),
8211 )
8212 .unwrap();
8213 let registration = registry.get_module("aft").unwrap().unwrap();
8214 assert_eq!(
8215 registration.control_ops,
8216 vec![
8217 "route.bind".to_string(),
8218 "route.status".to_string(),
8219 "future.synthetic".to_string(),
8220 ]
8221 );
8222 let frame =
8223 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
8224 assert!(handler
8225 .guard_module_control_op(&frame, "aft", "future.synthetic")
8226 .unwrap()
8227 .is_none());
8228 }
8229
8230 #[tokio::test]
8231 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
8232 let registry = Arc::new(Registry::default());
8233 let forwarding = Arc::new(ForwardingTable::default());
8234 let handler =
8235 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8236 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
8237 hello_via_sink(
8238 &handler,
8239 &module_ctx,
8240 &mut module_rx,
8241 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8242 )
8243 .await;
8244
8245 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
8246 let responses = handler
8247 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
8248 .await
8249 .unwrap();
8250 assert_eq!(responses.len(), 1);
8251 assert_eq!(responses[0].header.ty, FrameType::Error);
8252 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
8253 assert!(module_rx.try_recv().is_err());
8254 }
8255
8256 #[tokio::test]
8257 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
8258 let registry = Arc::new(Registry::default());
8259 let forwarding = Arc::new(ForwardingTable::default());
8260 let handler =
8261 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8262 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
8263 hello_via_sink(
8264 &handler,
8265 &module_ctx,
8266 &mut module_rx,
8267 hello_frame_with_control_ops(
8268 "aft",
8269 PROTOCOL_VERSION,
8270 7,
8271 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8272 ),
8273 )
8274 .await;
8275
8276 let project_root = unique_project_root("demux");
8277 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
8278 let route_handler = handler.clone();
8279 let route_task = tokio::spawn(async move {
8280 route_handler
8281 .handle_control_frame(
8282 &route_client_ctx,
8283 route_open_frame(100, "aft", project_root),
8284 )
8285 .await
8286 .unwrap()
8287 });
8288 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8289 .await
8290 .unwrap()
8291 .unwrap();
8292 assert!(matches!(
8293 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
8294 ModuleControlRequest::RouteBind { .. }
8295 ));
8296
8297 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
8298 let health_handler = handler.clone();
8299 let health_task = tokio::spawn(async move {
8300 health_handler
8301 .handle_control_frame(
8302 &health_client_ctx,
8303 supervisor_health_probe_frame(101, "aft"),
8304 )
8305 .await
8306 .unwrap()
8307 });
8308 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8309 .await
8310 .unwrap()
8311 .unwrap();
8312 assert_eq!(
8313 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8314 ModuleControlRequest::HealthCheck {}
8315 );
8316
8317 handler
8318 .handle_control_frame(
8319 &module_ctx,
8320 health_response(health_frame.header.corr, HealthStatus::Degraded),
8321 )
8322 .await
8323 .unwrap();
8324 let health_response = health_task.await.unwrap();
8325 assert_eq!(health_response.len(), 1);
8326 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8327 ClientControlResponse::SupervisorHealthProbe {
8328 module_id,
8329 status,
8330 detail,
8331 metrics,
8332 } => {
8333 assert_eq!(module_id, "aft");
8334 assert_eq!(status, HealthStatus::Degraded);
8335 assert_eq!(detail.as_deref(), Some("warming"));
8336 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
8337 }
8338 other => panic!("unexpected health response: {other:?}"),
8339 }
8340
8341 handler
8342 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8343 .await
8344 .unwrap();
8345 let route_response = route_task.await.unwrap();
8346 assert!(route_response.is_empty());
8347 let published = route_client_rx.recv().await.unwrap();
8348 assert!(matches!(
8349 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8350 ClientControlResponse::RouteOpen { .. }
8351 ));
8352 }
8353
8354 async fn relay_route_open(
8359 handler: &ControlHandler,
8360 client_connection: ConnectionId,
8361 client_egress: &FrameSink,
8362 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8363 corr: u64,
8364 module_id: &str,
8365 project_root_label: &str,
8366 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
8367 let ctx = RouteCtx {
8368 connection_id: client_connection,
8369 egress: client_egress.clone(),
8370 };
8371 let handler = handler.clone();
8372 let project_root = unique_project_root(project_root_label);
8373 let module_id = module_id.to_string();
8374 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
8375 let task = tokio::spawn(async move {
8376 let _guard = tracing::dispatcher::set_default(&dispatch);
8377 handler
8378 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8379 .await
8380 .unwrap()
8381 });
8382 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8383 .await
8384 .expect("module receives the relayed route.bind")
8385 .expect("module egress is open");
8386 (task, bind.frame)
8387 }
8388
8389 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8390 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8391 ModuleControlRequest::RouteBind {
8392 route_channel,
8393 epoch,
8394 ..
8395 } => (route_channel, epoch),
8396 other => panic!("expected a route.bind request, got {other:?}"),
8397 }
8398 }
8399
8400 fn published_route(frame: &Frame) -> (u16, u32) {
8401 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
8402 ClientControlResponse::RouteOpen {
8403 route_channel,
8404 route_epoch,
8405 } => (route_channel, route_epoch),
8406 other => panic!("expected a route.open response, got {other:?}"),
8407 }
8408 }
8409
8410 #[tokio::test]
8430 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
8431 let registry = Arc::new(Registry::default());
8432 let forwarding = Arc::new(ForwardingTable::default());
8433 let handler =
8434 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8435
8436 let module_connection = ConnectionId::new(30);
8437 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8438 hello_via_sink(
8439 &handler,
8440 &module_ctx,
8441 &mut module_rx,
8442 hello_frame("aft", PROTOCOL_VERSION, 7),
8443 )
8444 .await;
8445
8446 let dying_client = ConnectionId::new(31);
8447 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
8448
8449 let (first_task, first_bind) = relay_route_open(
8452 &handler,
8453 dying_client,
8454 &dying_ctx.egress,
8455 &mut module_rx,
8456 100,
8457 "aft",
8458 "closing-first",
8459 )
8460 .await;
8461 handler
8462 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
8463 .await
8464 .unwrap();
8465 assert!(first_task.await.unwrap().is_empty());
8466 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
8467
8468 let (second_task, second_bind) = relay_route_open(
8470 &handler,
8471 dying_client,
8472 &dying_ctx.egress,
8473 &mut module_rx,
8474 101,
8475 "aft",
8476 "closing-second",
8477 )
8478 .await;
8479 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
8480
8481 assert!(forwarding
8484 .escalate_client_delivery_failure(
8485 dying_client,
8486 first_channel,
8487 first_epoch,
8488 CloseReason::new(
8489 "module_to_client_delivery_failed",
8490 "client egress refused a module frame",
8491 ),
8492 crate::forwarding::UndeliveredFrame {
8493 module_id: None,
8494 sink: &dying_ctx.egress,
8495 },
8496 )
8497 .unwrap());
8498 assert!(!dying_ctx.egress.is_closed());
8499
8500 let ack = handler
8502 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
8503 .await;
8504 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
8505 if module_loop_error.is_some() {
8506 handler.cleanup_connection(module_connection).unwrap();
8510 }
8511 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8515 .await
8516 .ok()
8517 .flatten();
8518
8519 assert!(
8521 registry
8522 .get_module_by_connection(module_connection)
8523 .unwrap()
8524 .is_some(),
8525 "one client's closing connection ended the shared module connection: \
8526 {module_loop_error:?}"
8527 );
8528 let cotenant = ConnectionId::new(32);
8530 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
8531 let (cotenant_task, cotenant_bind) = relay_route_open(
8532 &handler,
8533 cotenant,
8534 &cotenant_ctx.egress,
8535 &mut module_rx,
8536 102,
8537 "aft",
8538 "closing-cotenant",
8539 )
8540 .await;
8541 handler
8542 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
8543 .await
8544 .unwrap();
8545 assert!(cotenant_task.await.unwrap().is_empty());
8546 let (cotenant_channel, cotenant_epoch) =
8547 published_route(&cotenant_rx.recv().await.unwrap());
8548 assert!(matches!(
8549 forwarding
8550 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
8551 .unwrap(),
8552 DataRoute::Client(DataRouteState::Bound(_))
8553 ));
8554
8555 let goodbye = post_ack_module_frame
8558 .expect("module receives a GOODBYE for the abandoned route channel");
8559 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8560 assert_eq!(goodbye.header.channel, abandoned_channel);
8561 assert_eq!(goodbye.header.epoch, abandoned_epoch);
8562
8563 assert!(dying_rx.try_recv().is_err());
8567 let second_response = second_task.await.unwrap();
8568 assert_eq!(second_response.len(), 1);
8569 assert_eq!(
8570 parse_error(&second_response[0])["code"],
8571 "target_unavailable"
8572 );
8573 }
8574
8575 #[test]
8582 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
8583 let handler = ControlHandler::default();
8584 let module_connection = ConnectionId::new(30);
8585 let client_connection = ConnectionId::new(31);
8586
8587 handler
8588 .refuse_to_end_module_connection_for_a_client(
8589 module_connection,
8590 77,
8591 ForwardingError::ConnectionClosing {
8592 connection_id: client_connection,
8593 },
8594 )
8595 .expect("a closing client must never end the module connection");
8596
8597 assert!(matches!(
8598 handler.refuse_to_end_module_connection_for_a_client(
8599 module_connection,
8600 78,
8601 ForwardingError::ConnectionClosing {
8602 connection_id: module_connection,
8603 },
8604 ),
8605 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
8606 connection_id
8607 })) if connection_id == module_connection
8608 ));
8609 assert!(matches!(
8610 handler.refuse_to_end_module_connection_for_a_client(
8611 module_connection,
8612 79,
8613 ForwardingError::Poisoned,
8614 ),
8615 Err(RouterError::Forwarding(ForwardingError::Poisoned))
8616 ));
8617 assert!(matches!(
8618 handler.refuse_to_end_module_connection_for_a_client(
8619 module_connection,
8620 80,
8621 ForwardingError::StaleModuleEndpoint,
8622 ),
8623 Err(RouterError::Forwarding(
8624 ForwardingError::StaleModuleEndpoint
8625 ))
8626 ));
8627 }
8628
8629 #[tokio::test]
8642 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
8643 let registry = Arc::new(Registry::default());
8644 let forwarding = Arc::new(ForwardingTable::default());
8645 let supervisor = SupervisorHandle::new();
8646 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8647 let handler =
8648 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8649 .with_supervisor(supervisor);
8650
8651 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8652 hello_via_sink(
8653 &handler,
8654 &target_ctx,
8655 &mut target_rx,
8656 hello_frame("target", PROTOCOL_VERSION, 1),
8657 )
8658 .await;
8659
8660 let wrong_nonce = handler
8664 .handle_control_frame(
8665 &route_ctx(ConnectionId::new(91)).0,
8666 route_open_frame_with_admission_facts(
8667 20,
8668 "target",
8669 unique_project_root("admission-facts"),
8670 Some(subc_control::ConsumerIdentity {
8671 module_id: "fed".to_string(),
8672 launch_nonce: "not-the-real-nonce".to_string(),
8673 }),
8674 None,
8675 ),
8676 )
8677 .await
8678 .unwrap();
8679 assert_eq!(
8680 parse_error(&wrong_nonce[0])["code"],
8681 "bad_consumer_identity",
8682 "a mismatched launch nonce must be refused, not stamped Reserved"
8683 );
8684
8685 let never_spawned = handler
8689 .handle_control_frame(
8690 &route_ctx(ConnectionId::new(92)).0,
8691 route_open_frame_with_admission_facts(
8692 21,
8693 "target",
8694 unique_project_root("admission-facts"),
8695 Some(subc_control::ConsumerIdentity {
8696 module_id: "never-spawned".to_string(),
8697 launch_nonce: "any-nonce".to_string(),
8698 }),
8699 None,
8700 ),
8701 )
8702 .await
8703 .unwrap();
8704 assert_eq!(
8705 parse_error(&never_spawned[0])["code"],
8706 "bad_consumer_identity",
8707 "an unspawned module_id must be refused rather than accepted for lack of a record"
8708 );
8709 }
8710
8711 #[tokio::test]
8734 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
8735 let registry = Arc::new(Registry::default());
8736 let forwarding = Arc::new(ForwardingTable::default());
8737 let supervisor = SupervisorHandle::new();
8738 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8739 let handler =
8740 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8741 .with_supervisor(supervisor);
8742
8743 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
8744 hello_via_sink(
8745 &handler,
8746 &target_ctx,
8747 &mut target_rx,
8748 hello_frame("target", PROTOCOL_VERSION, 1),
8749 )
8750 .await;
8751
8752 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
8753 let route_handler = handler.clone();
8754 let route_task = tokio::spawn(async move {
8755 route_handler
8756 .handle_control_frame(
8757 &client_ctx,
8758 route_open_frame_with_admission_facts(
8759 30,
8760 "target",
8761 unique_project_root("admission-facts"),
8762 Some(subc_control::ConsumerIdentity {
8763 module_id: "fed".to_string(),
8764 launch_nonce: "fed-nonce".to_string(),
8765 }),
8766 None,
8767 ),
8768 )
8769 .await
8770 .unwrap()
8771 });
8772
8773 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8780 .await
8781 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
8782 .expect("module control channel closed before route.bind");
8783 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
8784 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
8785 panic!("expected route.bind")
8786 };
8787 assert_eq!(
8788 principal,
8789 Some(Principal::Reserved {
8790 module_id: "fed".to_string()
8791 }),
8792 "a correctly attested consumer must be stamped Reserved for its own id"
8793 );
8794
8795 handler
8796 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
8797 .await
8798 .unwrap();
8799 assert!(route_task.await.unwrap().is_empty());
8800 assert!(
8801 matches!(
8802 serde_json::from_slice::<ClientControlResponse>(
8803 &client_rx.recv().await.unwrap().body
8804 )
8805 .unwrap(),
8806 ClientControlResponse::RouteOpen { .. }
8807 ),
8808 "the route must actually open, not merely avoid an error"
8809 );
8810 }
8811
8812 #[tokio::test(start_paused = true)]
8813 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
8814 let registry = Arc::new(Registry::default());
8815 let forwarding = Arc::new(ForwardingTable::default());
8816 let supervisor = SupervisorHandle::new();
8817 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8818 let handler =
8819 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8820 .with_supervisor(supervisor);
8821
8822 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
8823 hello_via_sink(
8824 &handler,
8825 &target_ctx,
8826 &mut target_rx,
8827 hello_frame("target", PROTOCOL_VERSION, 1),
8828 )
8829 .await;
8830
8831 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
8832 let direct_handler = handler.clone();
8833 let direct_open = tokio::spawn(async move {
8834 direct_handler
8835 .handle_control_frame(
8836 &direct_ctx,
8837 route_open_frame(2, "target", unique_project_root("route-census-direct")),
8838 )
8839 .await
8840 .unwrap()
8841 });
8842 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8843 .await
8844 .expect("no direct route.bind within 5s")
8845 .expect("target control channel closed before direct route.bind");
8846 handler
8847 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
8848 .await
8849 .unwrap();
8850 assert!(direct_open.await.unwrap().is_empty());
8851 let _ = direct_rx.recv().await.unwrap();
8852
8853 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
8854 let reserved_handler = handler.clone();
8855 let reserved_open = tokio::spawn(async move {
8856 reserved_handler
8857 .handle_control_frame(
8858 &reserved_ctx,
8859 route_open_frame_with_admission_facts(
8860 3,
8861 "target",
8862 unique_project_root("admission-facts"),
8863 Some(ConsumerIdentity {
8864 module_id: "fed".to_string(),
8865 launch_nonce: "fed-nonce".to_string(),
8866 }),
8867 None,
8868 ),
8869 )
8870 .await
8871 .unwrap()
8872 });
8873 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8874 .await
8875 .expect("no reserved route.bind within 5s")
8876 .expect("target control channel closed before reserved route.bind");
8877 handler
8878 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
8879 .await
8880 .unwrap();
8881 assert!(reserved_open.await.unwrap().is_empty());
8882 let _ = reserved_rx.recv().await.unwrap();
8883
8884 forwarding
8885 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
8886 .unwrap();
8887 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
8888 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
8889 module_id: Some("target".to_string()),
8890 })
8891 .unwrap();
8892 let census_frame =
8893 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
8894 let response = handler
8895 .handle_control_frame(&census_ctx, census_frame)
8896 .await
8897 .unwrap()
8898 .pop()
8899 .unwrap();
8900 let actual: Value = serde_json::from_slice(&response.body).unwrap();
8901 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
8902 assert!(matches!(
8903 decoded,
8904 ClientControlResponse::SupervisorRoutes { .. }
8905 ));
8906 let routes = actual["modules"][0]["routes"].as_array().unwrap();
8907 assert_eq!(routes.len(), 2);
8908 assert!(routes.iter().all(|route| route["draining"] == true));
8909 assert!(
8912 routes.iter().all(|route| route["drain_reason"] == "reload"),
8913 "draining routes must name the drain's reason: {routes:?}"
8914 );
8915 assert!(routes.iter().any(|route| {
8916 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
8917 }));
8918 assert!(routes.iter().any(|route| {
8919 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
8920 }));
8921
8922 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8923 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
8924 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8925 std::fs::write(
8926 &golden_path,
8927 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
8928 )
8929 .unwrap();
8930 }
8931 let expected: Value =
8932 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
8933 assert_eq!(actual, expected);
8934 }
8935
8936 async fn query_live_roots(
8937 handler: &ControlHandler,
8938 module_ctx: &RouteCtx,
8939 ) -> ModuleControlResponseToModule {
8940 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
8941 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
8942 let response = handler
8943 .handle_control_frame(module_ctx, frame)
8944 .await
8945 .unwrap()
8946 .pop()
8947 .unwrap();
8948 serde_json::from_slice(&response.body).unwrap()
8949 }
8950
8951 #[tokio::test(start_paused = true)]
8952 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
8953 let registry = Arc::new(Registry::default());
8954 let forwarding = Arc::new(ForwardingTable::default());
8955 let handler = ControlHandler::with_forwarding(registry, forwarding);
8956 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
8957 hello_via_sink(
8958 &handler,
8959 &target_ctx,
8960 &mut target_rx,
8961 hello_frame("target", PROTOCOL_VERSION, 1),
8962 )
8963 .await;
8964 let root = unique_project_root("live-roots-known");
8965 let path = ProjectRootId::from_path_allowing_missing(root.path())
8966 .unwrap()
8967 .as_path()
8968 .to_path_buf();
8969 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
8970 let open_handler = handler.clone();
8971 let opened = tokio::spawn(async move {
8972 open_handler
8973 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
8974 .await
8975 .unwrap()
8976 });
8977 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
8978 .await
8979 .unwrap()
8980 .unwrap();
8981 handler
8982 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
8983 .await
8984 .unwrap();
8985 assert!(opened.await.unwrap().is_empty());
8986 let _ = client_rx.recv().await.unwrap();
8987
8988 let root = unique_project_root("live-roots-pending");
8989 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
8990 .unwrap()
8991 .as_path()
8992 .to_path_buf();
8993 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
8994 let open_handler = handler.clone();
8995 let pending = tokio::spawn(async move {
8996 open_handler
8997 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
8998 .await
8999 .unwrap()
9000 });
9001 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9002 .await
9003 .unwrap()
9004 .unwrap();
9005 let actual = query_live_roots(&handler, &target_ctx).await;
9006 let ModuleControlResponseToModule::LiveRoots {
9007 roots,
9008 unknown_root_bindings,
9009 total_bindings,
9010 } = actual
9011 else {
9012 panic!("expected live roots")
9013 };
9014 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
9015 assert_eq!(unknown_root_bindings, 0);
9016 assert_eq!(
9017 roots.len(),
9018 2,
9019 "root-known arm must retain each canonical root"
9020 );
9021 assert_eq!(
9022 total_bindings,
9023 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9024 );
9025 let counts = roots
9026 .iter()
9027 .map(|root| (root.project_root.clone(), root.bound, root.pending))
9028 .collect::<Vec<_>>();
9029 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
9030 expected.sort_by(|a, b| a.0.cmp(&b.0));
9031 assert_eq!(
9032 counts, expected,
9033 "roots must sort by path and count pending separately"
9034 );
9035 handler
9036 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
9037 .await
9038 .unwrap();
9039 assert!(pending.await.unwrap().is_empty());
9040 }
9041
9042 #[tokio::test(start_paused = true)]
9043 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
9044 let forwarding = Arc::new(ForwardingTable::default());
9045 let handler =
9046 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9047 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
9048 hello_via_sink(
9049 &handler,
9050 &target_ctx,
9051 &mut target_rx,
9052 hello_frame("target", PROTOCOL_VERSION, 1),
9053 )
9054 .await;
9055 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
9056 let pending = forwarding
9057 .begin_route_bind_relay_for_test(
9058 client_ctx.connection_id,
9059 client_ctx.egress.clone(),
9060 2,
9061 "target",
9062 )
9063 .unwrap();
9064 forwarding
9065 .complete_pending_relay(
9066 target_ctx.connection_id,
9067 pending.corr,
9068 RouteBindRelayOutcome::Accepted,
9069 )
9070 .unwrap();
9071 let actual = query_live_roots(&handler, &target_ctx).await;
9072 let ModuleControlResponseToModule::LiveRoots {
9073 roots,
9074 unknown_root_bindings,
9075 total_bindings,
9076 } = actual
9077 else {
9078 panic!("expected live roots")
9079 };
9080 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
9081 assert_eq!(
9082 unknown_root_bindings, 1,
9083 "unknown-root arm must not read as no bindings"
9084 );
9085 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
9086 assert_eq!(
9087 total_bindings,
9088 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9089 );
9090 }
9091
9092 #[tokio::test(start_paused = true)]
9100 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
9101 let forwarding = Arc::new(ForwardingTable::default());
9102 let handler =
9103 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9104 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
9105 let replies = handler
9106 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
9107 .await
9108 .unwrap();
9109 let queued_by_hello = module_rx.len();
9110
9111 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
9112 let open_handler = handler.clone();
9113 let open = tokio::spawn(async move {
9114 open_handler
9115 .handle_control_frame(
9116 &client_ctx,
9117 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
9118 )
9119 .await
9120 .unwrap()
9121 });
9122 let mut spins = 0;
9124 while module_rx.len() == queued_by_hello {
9125 spins += 1;
9126 assert!(spins < 10_000, "route.open never queued a route.bind");
9127 tokio::task::yield_now().await;
9128 }
9129
9130 for reply in replies {
9132 module_ctx.egress.send(reply).await.unwrap();
9133 }
9134
9135 let first = module_rx.recv().await.unwrap().frame;
9136 assert_eq!(
9137 first.header.ty,
9138 FrameType::HelloAck,
9139 "the first frame a registering module reads must be its HELLO_ACK"
9140 );
9141 assert_eq!(first.header.corr, 7);
9142 let second = module_rx.recv().await.unwrap().frame;
9143 assert_eq!(second.header.ty, FrameType::Request);
9144 assert!(
9145 matches!(
9146 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
9147 ModuleControlRequest::RouteBind { .. }
9148 ),
9149 "the route.bind follows the ack"
9150 );
9151 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
9152
9153 handler
9154 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
9155 .await
9156 .unwrap();
9157 assert!(open.await.unwrap().is_empty());
9158 let _ = client_rx.recv().await.unwrap();
9159 }
9160
9161 #[tokio::test(start_paused = true)]
9162 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
9163 let handler = ControlHandler::with_forwarding(
9164 Arc::new(Registry::default()),
9165 Arc::new(ForwardingTable::default()),
9166 );
9167 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
9168 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
9169 hello_via_sink(
9170 &handler,
9171 &first_ctx,
9172 &mut first_rx,
9173 hello_frame("first", PROTOCOL_VERSION, 1),
9174 )
9175 .await;
9176 hello_via_sink(
9177 &handler,
9178 &second_ctx,
9179 &mut second_rx,
9180 hello_frame("second", PROTOCOL_VERSION, 2),
9181 )
9182 .await;
9183 let root = unique_project_root("second-only");
9184 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
9185 let cloned = handler.clone();
9186 let open = tokio::spawn(async move {
9187 cloned
9188 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
9189 .await
9190 .unwrap()
9191 });
9192 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
9193 .await
9194 .unwrap()
9195 .unwrap();
9196 let first = query_live_roots(&handler, &first_ctx).await;
9197 let second = query_live_roots(&handler, &second_ctx).await;
9198 assert!(
9199 matches!(
9200 first,
9201 ModuleControlResponseToModule::LiveRoots {
9202 total_bindings: 0,
9203 ..
9204 }
9205 ),
9206 "cross-module scope must not expose another module's roots"
9207 );
9208 assert!(
9209 matches!(
9210 second,
9211 ModuleControlResponseToModule::LiveRoots {
9212 total_bindings: 1,
9213 ..
9214 }
9215 ),
9216 "second module must see its pending route"
9217 );
9218 handler
9219 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
9220 .await
9221 .unwrap();
9222 assert!(open.await.unwrap().is_empty());
9223 }
9224
9225 #[tokio::test(start_paused = true)]
9226 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
9227 let handler = ControlHandler::with_forwarding(
9228 Arc::new(Registry::default()),
9229 Arc::new(ForwardingTable::default()),
9230 );
9231 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
9232 hello_via_sink(
9233 &handler,
9234 &target_ctx,
9235 &mut target_rx,
9236 hello_frame("target", PROTOCOL_VERSION, 1),
9237 )
9238 .await;
9239 let actual = query_live_roots(&handler, &target_ctx).await;
9240 let ModuleControlResponseToModule::LiveRoots {
9241 roots,
9242 unknown_root_bindings,
9243 total_bindings,
9244 } = actual
9245 else {
9246 panic!("expected live roots")
9247 };
9248 assert!(roots.is_empty());
9249 assert_eq!(unknown_root_bindings, 0);
9250 assert_eq!(total_bindings, 0);
9251 assert_eq!(
9252 total_bindings,
9253 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9254 );
9255 }
9256
9257 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
9264 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9265 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
9266 let text = std::fs::read_to_string(&path)
9267 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
9268 let vectors: Vec<(String, Value)> = text
9269 .lines()
9270 .filter(|line| !line.trim().is_empty())
9271 .map(|line| {
9272 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
9273 let id = entry["corpus_id"]
9274 .as_str()
9275 .expect("every vector carries a corpus_id")
9276 .to_string();
9277 (id, entry["package"].clone())
9278 })
9279 .collect();
9280 assert_eq!(
9284 vectors.len(),
9285 3,
9286 "vendored fed corpus changed size; re-sync from subc-federation"
9287 );
9288
9289 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
9304 let richest = vectors
9305 .iter()
9306 .filter_map(|(_, package)| package.as_object())
9307 .map(|object| {
9308 object
9309 .keys()
9310 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
9311 .count()
9312 })
9313 .max()
9314 .unwrap_or(0);
9315 assert!(
9316 richest >= 2,
9317 "vendored corpus no longer carries a package with unmodelled fields, \
9318 so the relay test can no longer distinguish a verbatim relay from a lossy one"
9319 );
9320
9321 vectors
9322 }
9323
9324 #[tokio::test]
9332 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
9333 for (corpus_id, package) in fed_admission_facts_vectors() {
9334 let registry = Arc::new(Registry::default());
9335 let forwarding = Arc::new(ForwardingTable::default());
9336 let supervisor = SupervisorHandle::new();
9337 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9338 let handler =
9339 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9340 .with_supervisor(supervisor)
9341 .with_admission_facts_config(
9342 Some("fed".to_string()),
9343 Some(vec!["target".to_string()]),
9344 );
9345
9346 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9347 hello_via_sink(
9348 &handler,
9349 &target_ctx,
9350 &mut target_rx,
9351 hello_frame("target", PROTOCOL_VERSION, 1),
9352 )
9353 .await;
9354
9355 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
9356 let route_handler = handler.clone();
9357 let expected = package.clone();
9358 let route_task = tokio::spawn(async move {
9359 route_handler
9360 .handle_control_frame(
9361 &client_ctx,
9362 route_open_frame_with_admission_facts(
9363 20,
9364 "target",
9365 unique_project_root("admission-facts"),
9366 Some(subc_control::ConsumerIdentity {
9367 module_id: "fed".to_string(),
9368 launch_nonce: "fed-nonce".to_string(),
9369 }),
9370 Some(package),
9371 ),
9372 )
9373 .await
9374 .unwrap()
9375 });
9376
9377 let bind_frame = target_rx.recv().await.unwrap();
9378 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9379 let ModuleControlRequest::RouteBind {
9380 admission_facts, ..
9381 } = bind
9382 else {
9383 panic!("{corpus_id}: expected route.bind")
9384 };
9385 assert_eq!(
9386 admission_facts,
9387 Some(expected),
9388 "{corpus_id}: relay must not add, drop or reshape any field"
9389 );
9390
9391 handler
9392 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9393 .await
9394 .unwrap();
9395 route_task.await.unwrap();
9396 }
9397 }
9398
9399 #[tokio::test]
9400 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
9401 let registry = Arc::new(Registry::default());
9402 let forwarding = Arc::new(ForwardingTable::default());
9403 let supervisor = SupervisorHandle::new();
9404 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9405 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
9406 let handler =
9407 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9408 .with_supervisor(supervisor)
9409 .with_admission_facts_config(
9410 Some("fed".to_string()),
9411 Some(vec!["target".to_string()]),
9412 );
9413
9414 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
9415 hello_via_sink(
9416 &handler,
9417 &target_ctx,
9418 &mut target_rx,
9419 hello_frame("target", PROTOCOL_VERSION, 1),
9420 )
9421 .await;
9422 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
9423 hello_via_sink(
9424 &handler,
9425 &other_ctx,
9426 &mut other_rx,
9427 hello_frame("other", PROTOCOL_VERSION, 2),
9428 )
9429 .await;
9430
9431 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
9432 let expected_facts = facts.clone();
9433 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
9434 let route_handler = handler.clone();
9435 let route_task = tokio::spawn(async move {
9436 route_handler
9437 .handle_control_frame(
9438 &client_ctx,
9439 route_open_frame_with_admission_facts(
9440 10,
9441 "target",
9442 unique_project_root("admission-facts"),
9443 Some(subc_control::ConsumerIdentity {
9444 module_id: "fed".to_string(),
9445 launch_nonce: "fed-nonce".to_string(),
9446 }),
9447 Some(facts.clone()),
9448 ),
9449 )
9450 .await
9451 .unwrap()
9452 });
9453 let bind_frame = target_rx.recv().await.unwrap();
9454 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9455 let ModuleControlRequest::RouteBind {
9456 admission_facts, ..
9457 } = bind
9458 else {
9459 panic!("expected route.bind")
9460 };
9461 assert_eq!(admission_facts, Some(expected_facts));
9462 handler
9463 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9464 .await
9465 .unwrap();
9466 assert!(route_task.await.unwrap().is_empty());
9467 assert!(matches!(
9468 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
9469 .unwrap(),
9470 ClientControlResponse::RouteOpen { .. }
9471 ));
9472
9473 let direct = handler
9474 .handle_control_frame(
9475 &route_ctx(ConnectionId::new(73)).0,
9476 route_open_frame_with_admission_facts(
9477 11,
9478 "target",
9479 unique_project_root("admission-facts"),
9480 None,
9481 Some(json!({"x": 1})),
9482 ),
9483 )
9484 .await
9485 .unwrap();
9486 assert_eq!(
9487 parse_error(&direct[0])["code"],
9488 "admission_facts_not_permitted"
9489 );
9490
9491 let different_reserved = handler
9492 .handle_control_frame(
9493 &route_ctx(ConnectionId::new(77)).0,
9494 route_open_frame_with_admission_facts(
9495 15,
9496 "target",
9497 unique_project_root("admission-facts"),
9498 Some(subc_control::ConsumerIdentity {
9499 module_id: "other".to_string(),
9500 launch_nonce: "other-nonce".to_string(),
9501 }),
9502 Some(json!({"x": 1})),
9503 ),
9504 )
9505 .await
9506 .unwrap();
9507 assert_eq!(
9508 parse_error(&different_reserved[0])["code"],
9509 "admission_facts_not_permitted"
9510 );
9511
9512 let other_target = handler
9513 .handle_control_frame(
9514 &route_ctx(ConnectionId::new(74)).0,
9515 route_open_frame_with_admission_facts(
9516 12,
9517 "other",
9518 unique_project_root("admission-facts"),
9519 Some(subc_control::ConsumerIdentity {
9520 module_id: "fed".to_string(),
9521 launch_nonce: "fed-nonce".to_string(),
9522 }),
9523 Some(json!({"x": 1})),
9524 ),
9525 )
9526 .await
9527 .unwrap();
9528 assert_eq!(
9529 parse_error(&other_target[0])["code"],
9530 "admission_facts_target_not_allowed"
9531 );
9532
9533 let nonexistent = handler
9534 .handle_control_frame(
9535 &route_ctx(ConnectionId::new(75)).0,
9536 route_open_frame_with_admission_facts(
9537 13,
9538 "missing",
9539 unique_project_root("admission-facts"),
9540 None,
9541 Some(json!({"x": 1})),
9542 ),
9543 )
9544 .await
9545 .unwrap();
9546 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
9547
9548 let described = handler
9549 .handle_control_frame(
9550 &route_ctx(ConnectionId::new(76)).0,
9551 Frame::build(
9552 FrameType::Request,
9553 control_flags(),
9554 0,
9555 0,
9556 14,
9557 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9558 )
9559 .unwrap(),
9560 )
9561 .await
9562 .unwrap();
9563 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9564 serde_json::from_slice(&described[0].body).unwrap()
9565 else {
9566 panic!("expected server.describe response")
9567 };
9568 assert!(capabilities
9569 .iter()
9570 .any(|cap| cap == "admission_facts_relay_v1"));
9571 }
9572
9573 #[tokio::test]
9574 async fn admission_facts_without_configured_carrier_are_rejected() {
9575 let registry = Arc::new(Registry::default());
9576 let forwarding = Arc::new(ForwardingTable::default());
9577 let handler = ControlHandler::with_forwarding(registry, forwarding);
9578 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
9579 hello_via_sink(
9580 &handler,
9581 &target_ctx,
9582 &mut target_rx,
9583 hello_frame("target", PROTOCOL_VERSION, 1),
9584 )
9585 .await;
9586
9587 let responses = handler
9588 .handle_control_frame(
9589 &route_ctx(ConnectionId::new(79)).0,
9590 route_open_frame_with_admission_facts(
9591 16,
9592 "target",
9593 unique_project_root("admission-facts"),
9594 None,
9595 Some(json!({"x": 1})),
9596 ),
9597 )
9598 .await
9599 .unwrap();
9600 assert_eq!(
9601 parse_error(&responses[0])["code"],
9602 "admission_facts_not_permitted"
9603 );
9604 }
9605
9606 #[tokio::test]
9607 async fn route_open_relays_consumer_capabilities_verbatim() {
9608 let registry = Arc::new(Registry::default());
9609 let forwarding = Arc::new(ForwardingTable::default());
9610 let handler =
9611 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9612 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
9613 hello_via_sink(
9614 &handler,
9615 &module_ctx,
9616 &mut module_rx,
9617 hello_frame("aft", PROTOCOL_VERSION, 7),
9618 )
9619 .await;
9620
9621 let expected = vec!["elicitation".to_string(), "roots".to_string()];
9622 let expected_for_request = expected.clone();
9623 let project_root = unique_project_root("consumer-capabilities-present");
9624 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
9625 let route_handler = handler.clone();
9626 let route_task = tokio::spawn(async move {
9627 route_handler
9628 .handle_control_frame(
9629 &client_ctx,
9630 route_open_frame_with_consumer_capabilities(
9631 401,
9632 "aft",
9633 project_root,
9634 Some(expected_for_request),
9635 ),
9636 )
9637 .await
9638 .unwrap()
9639 });
9640 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9641 .await
9642 .unwrap()
9643 .unwrap();
9644 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9645 let ModuleControlRequest::RouteBind {
9646 consumer_capabilities,
9647 ..
9648 } = bind
9649 else {
9650 panic!("expected route.bind request, got {bind:?}");
9651 };
9652 assert_eq!(consumer_capabilities, Some(expected.clone()));
9653
9654 handler
9655 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9656 .await
9657 .unwrap();
9658 let route_response = route_task.await.unwrap();
9659 assert!(route_response.is_empty());
9660 let published = client_rx.recv().await.unwrap();
9661 assert!(matches!(
9662 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9663 ClientControlResponse::RouteOpen { .. }
9664 ));
9665 }
9666
9667 #[tokio::test]
9668 async fn route_open_without_consumer_capabilities_relays_none() {
9669 let registry = Arc::new(Registry::default());
9670 let forwarding = Arc::new(ForwardingTable::default());
9671 let handler =
9672 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9673 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
9674 hello_via_sink(
9675 &handler,
9676 &module_ctx,
9677 &mut module_rx,
9678 hello_frame("aft", PROTOCOL_VERSION, 7),
9679 )
9680 .await;
9681
9682 let project_root = unique_project_root("consumer-capabilities-absent");
9683 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
9684 let route_handler = handler.clone();
9685 let route_task = tokio::spawn(async move {
9686 route_handler
9687 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
9688 .await
9689 .unwrap()
9690 });
9691 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9692 .await
9693 .unwrap()
9694 .unwrap();
9695 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9696 let ModuleControlRequest::RouteBind {
9697 consumer_capabilities,
9698 ..
9699 } = bind
9700 else {
9701 panic!("expected route.bind request, got {bind:?}");
9702 };
9703 assert_eq!(consumer_capabilities, None);
9704
9705 handler
9706 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9707 .await
9708 .unwrap();
9709 let route_response = route_task.await.unwrap();
9710 assert!(route_response.is_empty());
9711 let published = client_rx.recv().await.unwrap();
9712 assert!(matches!(
9713 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9714 ClientControlResponse::RouteOpen { .. }
9715 ));
9716 }
9717
9718 async fn bind_role_versions_for(
9722 sent: Option<BTreeMap<String, String>>,
9723 connection: u64,
9724 ) -> Option<BTreeMap<String, String>> {
9725 let registry = Arc::new(Registry::default());
9726 let forwarding = Arc::new(ForwardingTable::default());
9727 let handler =
9728 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9729 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
9730 hello_via_sink(
9731 &handler,
9732 &module_ctx,
9733 &mut module_rx,
9734 hello_frame("aft", PROTOCOL_VERSION, 7),
9735 )
9736 .await;
9737 let project_root = unique_project_root("role-versions");
9738 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
9739 let route_handler = handler.clone();
9740 let route_task = tokio::spawn(async move {
9741 route_handler
9742 .handle_control_frame(
9743 &client_ctx,
9744 route_open_frame_with_role_versions(403, "aft", project_root, sent),
9745 )
9746 .await
9747 .unwrap()
9748 });
9749 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9750 .await
9751 .expect("a well-formed route.open reaches the module as a bind")
9752 .unwrap();
9753 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9754 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
9755 panic!("expected route.bind request, got {bind:?}");
9756 };
9757 handler
9758 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9759 .await
9760 .unwrap();
9761 assert!(route_task.await.unwrap().is_empty());
9762 let published = client_rx.recv().await.unwrap();
9763 assert!(matches!(
9764 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9765 ClientControlResponse::RouteOpen { .. }
9766 ));
9767 role_versions
9768 }
9769
9770 #[tokio::test]
9771 async fn route_open_relays_role_versions_verbatim() {
9772 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
9773 assert_eq!(
9774 bind_role_versions_for(Some(sent.clone()), 141).await,
9775 Some(sent)
9776 );
9777 }
9778
9779 #[tokio::test]
9782 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
9783 assert_eq!(bind_role_versions_for(None, 143).await, None);
9784 assert_eq!(
9785 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
9786 None
9787 );
9788 }
9789
9790 #[tokio::test]
9791 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
9792 let registry = Arc::new(Registry::default());
9793 let forwarding = Arc::new(ForwardingTable::default());
9794 let handler =
9795 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9796 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
9797 hello_via_sink(
9798 &handler,
9799 &module_ctx,
9800 &mut module_rx,
9801 hello_frame("aft", PROTOCOL_VERSION, 7),
9802 )
9803 .await;
9804
9805 let nine: BTreeMap<String, String> = (0..9)
9806 .map(|index| (format!("role-{index}"), "v1".to_string()))
9807 .collect();
9808 for (label, malformed) in [
9809 (
9810 "invalid role name",
9811 role_versions(&[("Tool_Provider", "v1")]),
9812 ),
9813 ("invalid version", role_versions(&[("tool-provider", "v0")])),
9814 ("nine entries", nine),
9815 ] {
9816 let responses = tokio::time::timeout(
9819 Duration::from_secs(1),
9820 handler.handle_control_frame(
9821 &route_ctx(ConnectionId::new(148)).0,
9822 route_open_frame_with_role_versions(
9823 404,
9824 "aft",
9825 unique_project_root("role-versions-malformed"),
9826 Some(malformed),
9827 ),
9828 ),
9829 )
9830 .await
9831 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
9832 .unwrap();
9833 assert_eq!(responses.len(), 1, "{label}");
9834 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
9835 let error = parse_error(&responses[0]);
9836 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
9837 assert_eq!(
9838 error["detail"]["field"], "role_versions",
9839 "{label}: {error}"
9840 );
9841 assert!(
9842 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
9843 "{label}: a malformed declaration is terminal"
9844 );
9845 assert!(
9846 module_rx.try_recv().is_err(),
9847 "{label}: the module must never see a bind"
9848 );
9849 }
9850 }
9851
9852 #[tokio::test]
9856 async fn route_role_versions_capability_is_advertised() {
9857 let handler = ControlHandler::new(Arc::new(Registry::default()));
9858 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
9859 let ack = hello_via_sink(
9860 &handler,
9861 &ctx,
9862 &mut rx,
9863 hello_frame("m", PROTOCOL_VERSION, 1),
9864 )
9865 .await;
9866 let ack = parse_ack(&ack);
9867 assert!(
9868 ack.subc_capabilities
9869 .iter()
9870 .any(|c| c == "route-role-versions/v1"),
9871 "{:?}",
9872 ack.subc_capabilities
9873 );
9874
9875 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
9876 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
9877 let reply = handler
9878 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
9879 .await
9880 .unwrap()
9881 .pop()
9882 .unwrap();
9883 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9884 serde_json::from_slice(&reply.body).unwrap()
9885 else {
9886 panic!("not a server.describe reply");
9887 };
9888 assert!(
9889 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
9890 "{capabilities:?}"
9891 );
9892 }
9893
9894 #[tokio::test]
9895 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
9896 let registry = Arc::new(Registry::default());
9897 let forwarding = Arc::new(ForwardingTable::default());
9898 let handler =
9899 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9900 .with_health_probe_timeout(Duration::from_secs(5));
9901 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
9902 hello_via_sink(
9903 &handler,
9904 &module_ctx,
9905 &mut module_rx,
9906 non_routable_hello_frame_with_control_ops(
9907 "mcp",
9908 300,
9909 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9910 ),
9911 )
9912 .await;
9913 assert!(registry
9914 .get_module("mcp")
9915 .unwrap()
9916 .unwrap()
9917 .manifest
9918 .provides
9919 .is_empty());
9920
9921 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
9922 let route_response = handler
9923 .handle_control_frame(
9924 &route_client_ctx,
9925 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
9926 )
9927 .await
9928 .unwrap();
9929 assert_eq!(route_response[0].header.ty, FrameType::Error);
9930 assert_eq!(
9931 parse_error(&route_response[0])["code"],
9932 "target_unavailable"
9933 );
9934 assert!(parse_error(&route_response[0])["message"]
9935 .as_str()
9936 .unwrap()
9937 .contains("does not provide the requested target"));
9938 assert!(module_rx.try_recv().is_err());
9939
9940 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
9941 let health_handler = handler.clone();
9942 let health_task = tokio::spawn(async move {
9943 health_handler
9944 .handle_control_frame(
9945 &health_client_ctx,
9946 supervisor_health_probe_frame(302, "mcp"),
9947 )
9948 .await
9949 .unwrap()
9950 });
9951 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9952 .await
9953 .unwrap()
9954 .unwrap();
9955 assert_eq!(
9956 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9957 ModuleControlRequest::HealthCheck {}
9958 );
9959 handler
9960 .handle_control_frame(
9961 &module_ctx,
9962 health_response(health_frame.header.corr, HealthStatus::Ok),
9963 )
9964 .await
9965 .unwrap();
9966 let health_response = health_task.await.unwrap();
9967 assert_eq!(health_response[0].header.ty, FrameType::Response);
9968 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9969 ClientControlResponse::SupervisorHealthProbe {
9970 module_id, status, ..
9971 } => {
9972 assert_eq!(module_id, "mcp");
9973 assert_eq!(status, HealthStatus::Ok);
9974 }
9975 other => panic!("unexpected health response: {other:?}"),
9976 }
9977
9978 forwarding
9983 .cleanup_connection(module_ctx.connection_id)
9984 .unwrap();
9985 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
9986 let cleanup_response = tokio::time::timeout(
9987 Duration::from_millis(200),
9988 handler.handle_control_frame(
9989 &cleanup_probe_ctx,
9990 supervisor_health_probe_frame(303, "mcp"),
9991 ),
9992 )
9993 .await
9994 .expect("probe should fail immediately when the control lane is gone")
9995 .unwrap();
9996 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
9997 assert_eq!(
9998 parse_error(&cleanup_response[0])["code"],
9999 "target_unavailable"
10000 );
10001 assert!(parse_error(&cleanup_response[0])["message"]
10002 .as_str()
10003 .unwrap()
10004 .contains("no module connection"));
10005
10006 handler
10007 .cleanup_connection(module_ctx.connection_id)
10008 .unwrap();
10009 }
10010
10011 #[tokio::test]
10012 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
10013 let registry = Arc::new(Registry::default());
10014 let supervisor_handle = SupervisorHandle::new();
10015 let supervisor =
10016 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10017 .with_handle(supervisor_handle.clone())
10018 .with_connection_file_path(
10019 std::env::temp_dir()
10020 .join(format!("subc-route-open-warming-{}", std::process::id())),
10021 );
10022 let module = supervisor
10023 .supervise_configured(
10024 ModuleSpec {
10025 module_id: "warming".to_string(),
10026 program: fake_aft_stub_path(),
10027 args: Vec::new(),
10028 env: Vec::new(),
10029 reserved: false,
10030 reserved_prefixes: Vec::new(),
10031 protocol: ModuleProtocol::Subc,
10032 overlap: Default::default(),
10033 },
10034 true,
10035 )
10036 .unwrap();
10037 assert_eq!(module.state().unwrap(), ModuleState::Running);
10038
10039 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10040 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
10041 let response = handler
10042 .handle_control_frame(
10043 &ctx,
10044 route_open_frame(304, "warming", unique_project_root("warming")),
10045 )
10046 .await
10047 .unwrap();
10048 module.stop().await.unwrap();
10049
10050 assert_eq!(response[0].header.ty, FrameType::Error);
10051 let error = parse_error(&response[0]);
10052 assert_eq!(error["code"], "module_warming");
10053 assert!(error["message"]
10054 .as_str()
10055 .unwrap()
10056 .contains("state=running, enabled=true, live=false"));
10057 }
10058
10059 #[test]
10060 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
10061 let handler = ControlHandler::new(Arc::new(Registry::default()));
10062 let capture = EventCapture::default();
10063 let _subscriber =
10064 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10065 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
10066 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
10067 let pending = (0..limit).collect::<Vec<_>>();
10068 let response = handler
10069 .route_open_capacity_refusal(
10070 &ctx,
10071 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
10072 "busy",
10073 pending.len(),
10074 limit,
10075 )
10076 .unwrap();
10077 assert_eq!(parse_error(&response)["code"], "target_unavailable");
10078 let event = capture
10079 .events()
10080 .into_iter()
10081 .find(|event| {
10082 event.target == "control"
10083 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
10084 })
10085 .expect("connection admission refusal event");
10086 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10087 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10088 }
10089
10090 #[test]
10091 fn route_open_target_cap_logs_admission_reason_and_capacity() {
10092 let handler = ControlHandler::new(Arc::new(Registry::default()));
10093 let capture = EventCapture::default();
10094 let _subscriber =
10095 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10096 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
10097 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
10098 let guards = (0..limit)
10099 .map(|_| {
10100 handler
10101 .route_bind_concurrency
10102 .try_admit("busy", limit)
10103 .unwrap()
10104 })
10105 .collect::<Vec<_>>();
10106 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
10107 Err(in_flight) => in_flight,
10108 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
10109 };
10110 let response = handler
10111 .route_open_target_capacity_refusal(
10112 &ctx,
10113 &route_open_frame(397, "busy", unique_project_root("target-cap")),
10114 "busy",
10115 in_flight,
10116 )
10117 .unwrap();
10118 assert_eq!(parse_error(&response)["code"], "target_unavailable");
10119 let event = capture
10120 .events()
10121 .into_iter()
10122 .find(|event| {
10123 event.target == "control"
10124 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
10125 })
10126 .expect("target admission refusal event");
10127 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10128 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10129 drop(guards);
10130 }
10131
10132 #[tokio::test(flavor = "current_thread")]
10133 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
10134 let handler = ControlHandler::new(Arc::new(Registry::default()));
10135 let capture = EventCapture::default();
10136 let _subscriber =
10137 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10138 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
10139 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10140
10141 let _ = handler
10142 .handle_client_control_request(
10143 &ctx,
10144 frame,
10145 ClientControlRequest::SupervisorSetEnabled {
10146 module_id: "broca".to_string(),
10147 enabled: false,
10148 },
10149 )
10150 .await
10151 .unwrap();
10152
10153 let events = capture
10154 .events()
10155 .into_iter()
10156 .filter(|event| {
10157 event.target == "control"
10158 && event.fields.get("message").map(String::as_str)
10159 == Some("supervisor request received")
10160 })
10161 .collect::<Vec<_>>();
10162 assert_eq!(events.len(), 1, "one supervisor request log line");
10163 let event = &events[0];
10164 assert_eq!(event.level, tracing::Level::INFO);
10165 assert_eq!(
10166 event.fields.get("op"),
10167 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
10168 );
10169 assert_eq!(
10170 event.fields.get("module_id"),
10171 Some(&"\"broca\"".to_string())
10172 );
10173 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
10174 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
10175 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
10176 }
10177
10178 #[tokio::test(flavor = "current_thread")]
10179 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
10180 let registry = Arc::new(Registry::default());
10181 let module_connection = ConnectionId::new(303);
10182 registry
10183 .register_with_control_ops(
10184 manifest("aft", PROTOCOL_VERSION),
10185 PROTOCOL_VERSION,
10186 module_connection,
10187 module_baseline_control_ops(),
10188 )
10189 .unwrap();
10190 let handler = ControlHandler::new(registry);
10191 let capture = EventCapture::default();
10192 let _subscriber =
10193 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10194 let (ctx, _rx) = route_ctx(module_connection);
10195 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10196
10197 let _ = handler
10198 .handle_client_control_request(
10199 &ctx,
10200 frame,
10201 ClientControlRequest::SupervisorSetEnabled {
10202 module_id: "broca".to_string(),
10203 enabled: false,
10204 },
10205 )
10206 .await
10207 .unwrap();
10208
10209 let callers = capture
10210 .events()
10211 .into_iter()
10212 .filter(|event| {
10213 event.fields.get("message").map(String::as_str)
10214 == Some("supervisor request received")
10215 })
10216 .map(|event| event.fields.get("caller").cloned())
10217 .collect::<Vec<_>>();
10218 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
10219 }
10220
10221 #[tokio::test(flavor = "current_thread")]
10222 async fn supervisor_list_does_not_log_request_received() {
10223 let handler = ControlHandler::new(Arc::new(Registry::default()));
10224 let capture = EventCapture::default();
10225 let _subscriber =
10226 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10227 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
10228 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
10229
10230 handler
10231 .handle_client_control_request(&ctx, frame, ClientControlRequest::SupervisorList {})
10232 .await
10233 .unwrap();
10234
10235 assert!(capture.events().into_iter().all(|event| {
10236 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10237 }));
10238 }
10239
10240 #[tokio::test(flavor = "current_thread")]
10241 async fn supervisor_rescan_preview_does_not_log_request_received() {
10242 let handler = ControlHandler::new(Arc::new(Registry::default()));
10243 let capture = EventCapture::default();
10244 let _subscriber =
10245 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10246 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
10247 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
10248
10249 handler
10250 .handle_client_control_request(
10251 &ctx,
10252 frame,
10253 ClientControlRequest::SupervisorRescan { preview: true },
10254 )
10255 .await
10256 .unwrap();
10257
10258 assert!(capture.events().into_iter().all(|event| {
10259 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10260 }));
10261 }
10262
10263 #[tokio::test]
10268 async fn route_open_refusal_names_the_check_that_refused() {
10269 let handler = ControlHandler::new(Arc::new(Registry::default()));
10270 let capture = EventCapture::default();
10271 let _subscriber =
10272 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10273 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10274 let response = handler
10275 .handle_control_frame(
10276 &ctx,
10277 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
10278 )
10279 .await
10280 .unwrap();
10281
10282 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10283 let event = capture
10284 .events()
10285 .into_iter()
10286 .find(|event| {
10287 event.target == "control"
10288 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10289 })
10290 .expect("route.open refusal event");
10291 assert_eq!(
10292 event.fields.get("reason"),
10293 Some(&"\"not_registered\"".to_string())
10294 );
10295 }
10296
10297 #[tokio::test]
10298 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
10299 let registry = Arc::new(Registry::default());
10300 let supervisor_handle = SupervisorHandle::new();
10301 let supervisor =
10302 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10303 .with_handle(supervisor_handle.clone())
10304 .with_connection_file_path(std::env::temp_dir().join(format!(
10305 "subc-route-open-refusal-info-{}",
10306 std::process::id()
10307 )));
10308 let module = supervisor
10309 .supervise_configured(
10310 ModuleSpec {
10311 module_id: "warming".to_string(),
10312 program: fake_aft_stub_path(),
10313 args: Vec::new(),
10314 env: Vec::new(),
10315 reserved: false,
10316 reserved_prefixes: Vec::new(),
10317 protocol: ModuleProtocol::Subc,
10318 overlap: Default::default(),
10319 },
10320 true,
10321 )
10322 .unwrap();
10323 assert_eq!(module.state().unwrap(), ModuleState::Running);
10324
10325 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10326 assert!(handler
10327 .counters()
10328 .snapshot()
10329 .get("route_open_refused_by_code")
10330 .is_none());
10331 let capture = EventCapture::default();
10332 let _subscriber =
10333 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10334 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
10335 let response = handler
10336 .handle_control_frame(
10337 &ctx,
10338 route_open_frame(394, "warming", unique_project_root("refusal-info")),
10339 )
10340 .await
10341 .unwrap();
10342 module.stop().await.unwrap();
10343
10344 assert_eq!(parse_error(&response[0])["code"], "module_warming");
10345 let event = capture
10346 .events()
10347 .into_iter()
10348 .find(|event| {
10349 event.target == "control"
10350 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
10351 })
10352 .expect("route.open refusal event");
10353 assert_eq!(
10354 event.fields.get("module_id"),
10355 Some(&"\"warming\"".to_string())
10356 );
10357 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
10358 assert_eq!(
10359 event.fields.get("reason"),
10360 Some(&"\"supervised_not_registered\"".to_string())
10361 );
10362 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
10363 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
10364 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
10365 assert_eq!(
10366 handler.counters().snapshot()["route_open_refused_by_code"],
10367 json!({ "module_warming": 1 })
10368 );
10369 }
10370
10371 const OUTAGE_START: &str = "route.open refusing module: not serving";
10372 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
10373
10374 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
10375 capture
10376 .events()
10377 .into_iter()
10378 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
10379 .collect()
10380 }
10381
10382 fn supervise_stub(
10383 registry: &Arc<Registry>,
10384 module_id: &str,
10385 enabled: bool,
10386 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
10387 let supervisor_handle = SupervisorHandle::new();
10388 let supervisor =
10389 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
10390 .with_handle(supervisor_handle.clone())
10391 .with_connection_file_path(std::env::temp_dir().join(format!(
10392 "subc-route-outage-{module_id}-{}",
10393 std::process::id()
10394 )));
10395 let module = supervisor
10396 .supervise_configured(
10397 ModuleSpec {
10398 module_id: module_id.to_string(),
10399 program: fake_aft_stub_path(),
10400 args: Vec::new(),
10401 env: Vec::new(),
10402 reserved: false,
10403 reserved_prefixes: Vec::new(),
10404 protocol: ModuleProtocol::Subc,
10405 overlap: Default::default(),
10406 },
10407 enabled,
10408 )
10409 .unwrap();
10410 (supervisor_handle, module)
10411 }
10412
10413 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
10414 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
10415 module_id: module_id.to_string(),
10416 drain_timeout_ms: Some(50),
10417 })
10418 .unwrap();
10419 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
10420 }
10421
10422 #[test]
10426 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
10427 let registry = Arc::new(Registry::default());
10428 let forwarding = Arc::new(ForwardingTable::default());
10429 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10430 let second = ControlHandler::with_forwarding(registry, forwarding);
10431 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
10432 }
10433
10434 #[tokio::test(flavor = "current_thread")]
10438 async fn route_open_unknown_module_refusals_add_no_outage_state() {
10439 let handler = ControlHandler::new(Arc::new(Registry::default()));
10440 let capture = EventCapture::default();
10441 let _subscriber =
10442 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10443 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
10444 for corr in 0..8 {
10445 let response = handler
10446 .handle_control_frame(
10447 &ctx,
10448 route_open_frame(
10449 380 + corr,
10450 &format!("nobody-{corr}"),
10451 unique_project_root("outage-unknown"),
10452 ),
10453 )
10454 .await
10455 .unwrap();
10456 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10457 }
10458
10459 assert_eq!(handler.route_outages.tracked_module_count(), 0);
10460 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
10461 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
10462 }
10463
10464 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10469 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
10470 let registry = Arc::new(Registry::default());
10471 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
10472 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10473 let capture = EventCapture::default();
10474 let _subscriber =
10475 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10476 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
10477 handler.route_outages.record_accepted("outage-restart");
10480
10481 let response = handler
10482 .handle_control_frame(
10483 &ctx,
10484 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
10485 )
10486 .await
10487 .unwrap();
10488 assert_eq!(response[0].header.ty, FrameType::Error);
10489 let starts = outage_lines(&capture, OUTAGE_START);
10490 assert_eq!(starts.len(), 1, "{starts:?}");
10491 assert_eq!(starts[0].level, tracing::Level::WARN);
10492 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
10493 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
10494 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
10495 handler.route_outages.record_accepted("outage-restart");
10496 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
10497
10498 let restart = handler
10499 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
10500 .await
10501 .unwrap();
10502 assert_eq!(
10503 restart[0].header.ty,
10504 FrameType::Response,
10505 "{:?}",
10506 parse_error(&restart[0])
10507 );
10508 handler
10509 .handle_control_frame(
10510 &ctx,
10511 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
10512 )
10513 .await
10514 .unwrap();
10515 module.stop().await.unwrap();
10516
10517 let starts = outage_lines(&capture, OUTAGE_START);
10518 assert_eq!(starts.len(), 2, "{starts:?}");
10519 assert_eq!(starts[1].level, tracing::Level::INFO);
10520 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
10521 }
10522
10523 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10527 async fn failed_operator_restart_leaves_no_operator_mark() {
10528 let registry = Arc::new(Registry::default());
10529 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
10530 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10531 let capture = EventCapture::default();
10532 let _subscriber =
10533 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10534 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
10535 handler.route_outages.record_accepted("outage-disabled");
10536
10537 let restart = handler
10538 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
10539 .await
10540 .unwrap();
10541 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
10542 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
10543
10544 handler
10545 .handle_control_frame(
10546 &ctx,
10547 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
10548 )
10549 .await
10550 .unwrap();
10551 let starts = outage_lines(&capture, OUTAGE_START);
10552 assert_eq!(starts.len(), 1, "{starts:?}");
10553 assert_eq!(starts[0].level, tracing::Level::WARN);
10554 }
10555
10556 #[tokio::test(flavor = "current_thread")]
10557 async fn route_open_unknown_module_escapes_target_module_id() {
10558 let handler = ControlHandler::new(Arc::new(Registry::default()));
10559 let capture = EventCapture::default();
10560 let _subscriber =
10561 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10562 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
10563 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10564 let response = handler
10565 .handle_control_frame(
10566 &ctx,
10567 route_open_frame(
10568 395,
10569 hostile_module_id,
10570 unique_project_root("hostile-target-module-id"),
10571 ),
10572 )
10573 .await
10574 .unwrap();
10575
10576 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10577 let event = capture
10578 .events()
10579 .into_iter()
10580 .find(|event| {
10581 event.target == "control"
10582 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10583 })
10584 .expect("route.open unknown-module refusal event");
10585 let logged = event.fields.get("module_id").expect("module_id field");
10586 assert!(!logged.bytes().any(|byte| byte < 0x20));
10587 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10588 }
10589
10590 #[tokio::test(flavor = "current_thread")]
10591 async fn route_open_module_rejection_uses_daemon_counter_key() {
10592 let registry = Arc::new(Registry::default());
10593 let forwarding = Arc::new(ForwardingTable::default());
10594 let handler =
10595 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10596 let module_connection = ConnectionId::new(95);
10597 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10598 hello_via_sink(
10599 &handler,
10600 &module_ctx,
10601 &mut module_rx,
10602 hello_frame("aft", PROTOCOL_VERSION, 395),
10603 )
10604 .await;
10605
10606 let client_connection = ConnectionId::new(96);
10607 let (client_ctx, _client_rx) = route_ctx(client_connection);
10608 let capture = EventCapture::default();
10609 let _subscriber =
10610 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10611 let (route_task, bind) = relay_route_open(
10612 &handler,
10613 client_connection,
10614 &client_ctx.egress,
10615 &mut module_rx,
10616 396,
10617 "aft",
10618 "hostile-module-code",
10619 )
10620 .await;
10621 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
10622 let rejection = Frame::build(
10623 FrameType::Error,
10624 control_flags(),
10625 0,
10626 0,
10627 bind.header.corr,
10628 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
10629 )
10630 .unwrap();
10631 handler
10632 .handle_control_frame(&module_ctx, rejection)
10633 .await
10634 .unwrap();
10635
10636 let response = route_task.await.unwrap();
10637 assert_eq!(parse_error(&response[0])["code"], hostile_code);
10638 let counters = handler.counters().snapshot();
10639 assert_eq!(
10640 counters["route_open_refused_by_code"],
10641 json!({ "module_rejected": 1 })
10642 );
10643 assert!(counters["route_open_refused_by_code"]
10644 .get(hostile_code)
10645 .is_none());
10646
10647 let event = capture
10648 .events()
10649 .into_iter()
10650 .find(|event| {
10651 event.target == "control"
10652 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
10653 })
10654 .expect("route.open module-rejection refusal event");
10655 let logged = event.fields.get("module_code").expect("module_code field");
10656 assert!(!logged.bytes().any(|byte| byte < 0x20));
10657 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10658 }
10659
10660 #[tokio::test]
10661 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
10662 let registry = Arc::new(Registry::default());
10663 let supervisor_handle = SupervisorHandle::new();
10664 let missing_program = std::env::temp_dir().join(format!(
10665 "subc-route-open-missing-program-{}",
10666 std::process::id()
10667 ));
10668 let supervisor =
10669 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10670 .with_handle(supervisor_handle.clone());
10671 let module = supervisor
10672 .supervise_configured(
10673 ModuleSpec {
10674 module_id: "failed".to_string(),
10675 program: missing_program,
10676 args: Vec::new(),
10677 env: Vec::new(),
10678 reserved: false,
10679 reserved_prefixes: Vec::new(),
10680 protocol: ModuleProtocol::Subc,
10681 overlap: Default::default(),
10682 },
10683 true,
10684 )
10685 .unwrap();
10686 assert_eq!(module.state().unwrap(), ModuleState::Failed);
10687
10688 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10689 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
10690 let response = handler
10691 .handle_control_frame(
10692 &ctx,
10693 route_open_frame(305, "failed", unique_project_root("failed")),
10694 )
10695 .await
10696 .unwrap();
10697
10698 assert_eq!(response[0].header.ty, FrameType::Error);
10699 let error = parse_error(&response[0]);
10700 assert_eq!(error["code"], "target_unavailable");
10701 assert!(error["message"]
10702 .as_str()
10703 .unwrap()
10704 .contains("state=failed, enabled=true, live=false"));
10705 }
10706
10707 #[tokio::test]
10708 async fn route_open_role_mismatch_remains_target_unavailable() {
10709 let registry = Arc::new(Registry::default());
10710 let handler = ControlHandler::new(Arc::clone(®istry));
10711 handler
10712 .handle_control(
10713 ConnectionId::new(41),
10714 non_routable_hello_frame_with_control_ops("health-only", 306, None),
10715 )
10716 .unwrap();
10717
10718 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
10719 let response = handler
10720 .handle_control_frame(
10721 &ctx,
10722 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
10723 )
10724 .await
10725 .unwrap();
10726
10727 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10728 assert!(parse_error(&response[0])["message"]
10729 .as_str()
10730 .unwrap()
10731 .contains("does not provide the requested target"));
10732 }
10733
10734 #[tokio::test]
10735 async fn route_open_inactive_registration_remains_target_unavailable() {
10736 let registry = Arc::new(Registry::default());
10737 let handler = ControlHandler::new(Arc::clone(®istry));
10738 handler
10739 .handle_control(
10740 ConnectionId::new(43),
10741 hello_frame("inactive", PROTOCOL_VERSION, 308),
10742 )
10743 .unwrap();
10744 assert!(registry
10745 .set_module_state_for_test("inactive", ChannelState::Closed)
10746 .unwrap());
10747
10748 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
10749 let response = handler
10750 .handle_control_frame(
10751 &ctx,
10752 route_open_frame(309, "inactive", unique_project_root("inactive")),
10753 )
10754 .await
10755 .unwrap();
10756
10757 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10758 assert!(parse_error(&response[0])["message"]
10759 .as_str()
10760 .unwrap()
10761 .contains("is not active"));
10762 }
10763
10764 #[tokio::test]
10765 async fn late_health_reply_is_recorded_through_the_module_response_path() {
10766 let registry = Arc::new(Registry::default());
10767 let forwarding = Arc::new(ForwardingTable::default());
10768 let supervisor_handle = SupervisorHandle::new();
10769 let supervisor =
10770 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
10771 .with_forwarding(Arc::clone(&forwarding))
10772 .with_handle(supervisor_handle.clone());
10773 let module = supervisor
10774 .supervise_configured(
10775 crate::ModuleSpec {
10776 module_id: "late-health-response".to_string(),
10777 program: PathBuf::from("disabled-module"),
10778 args: Vec::new(),
10779 env: Vec::new(),
10780 reserved: false,
10781 reserved_prefixes: Vec::new(),
10782 protocol: ModuleProtocol::Subc,
10783 overlap: Default::default(),
10784 },
10785 false,
10786 )
10787 .unwrap();
10788 let handler =
10789 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10790 .with_supervisor(supervisor_handle);
10791 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
10792 handler
10793 .handle_control_frame(
10794 &module_ctx,
10795 hello_frame_with_control_ops(
10796 "late-health-response",
10797 PROTOCOL_VERSION,
10798 7,
10799 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10800 ),
10801 )
10802 .await
10803 .unwrap();
10804 let probe_started_at = Instant::now() - Duration::from_millis(80);
10805 let pending = forwarding
10806 .begin_health_probe_rpc_for(
10807 "late-health-response",
10808 MODULE_CONTROL_OP_HEALTH_CHECK,
10809 probe_started_at,
10810 Instant::now() - Duration::from_millis(1),
10811 )
10812 .unwrap();
10813 assert!(forwarding
10814 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
10815 .unwrap());
10816
10817 let responses = handler
10818 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
10819 .await
10820 .unwrap();
10821
10822 assert!(responses.is_empty());
10823 let health = module.status().unwrap().health;
10824 assert_eq!(health.late_answer_count, 1);
10825 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
10826 }
10827
10828 #[tokio::test]
10829 async fn health_probe_timeout_and_module_death_are_typed() {
10830 let registry = Arc::new(Registry::default());
10831 let forwarding = Arc::new(ForwardingTable::default());
10832 let handler =
10833 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10834 .with_health_probe_timeout(Duration::from_millis(50));
10835 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10836 hello_via_sink(
10837 &handler,
10838 &module_ctx,
10839 &mut module_rx,
10840 hello_frame_with_control_ops(
10841 "aft",
10842 PROTOCOL_VERSION,
10843 7,
10844 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10845 ),
10846 )
10847 .await;
10848
10849 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10850 let responses = handler
10851 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
10852 .await
10853 .unwrap();
10854 assert_eq!(responses[0].header.ty, FrameType::Error);
10855 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
10856 let _ = module_rx.try_recv();
10857
10858 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
10859 let health_handler = handler.clone();
10860 let death_task = tokio::spawn(async move {
10861 health_handler
10862 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
10863 .await
10864 .unwrap()
10865 });
10866 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10867 .await
10868 .unwrap()
10869 .unwrap();
10870 handler
10871 .cleanup_connection(module_ctx.connection_id)
10872 .unwrap();
10873 let responses = death_task.await.unwrap();
10874 assert_eq!(responses[0].header.ty, FrameType::Error);
10875 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
10876 }
10877
10878 #[test]
10879 fn hello_requires_exact_protocol_version() {
10880 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
10881 let registry = Arc::new(Registry::default());
10882 let handler = ControlHandler::new(Arc::clone(®istry));
10883 let responses = handler
10884 .handle_control(
10885 ConnectionId::new(connection),
10886 hello_frame("aft", offered, 9),
10887 )
10888 .unwrap();
10889
10890 assert_eq!(responses.len(), 1);
10891 assert_eq!(responses[0].header.ty, FrameType::Error);
10892 let error = parse_error(&responses[0]);
10893 assert_eq!(error["code"], "version_unsupported");
10894 assert!(registry.get_module("aft").unwrap().is_none());
10895 assert_eq!(registry.active_registration_count().unwrap(), 0);
10896 }
10897 }
10898
10899 #[test]
10900 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
10901 let registry = Arc::new(Registry::default());
10902 let forwarding = Arc::new(ForwardingTable::default());
10903 let handler =
10904 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10905 let module_connection = ConnectionId::new(301);
10906 let registration = registry
10907 .register_with_control_ops(
10908 manifest("aft-push", PROTOCOL_VERSION),
10909 PROTOCOL_VERSION,
10910 module_connection,
10911 module_baseline_control_ops(),
10912 )
10913 .unwrap();
10914 let (module_tx, _module_rx) = mpsc::channel(8);
10915 let endpoint = forwarding
10916 .register_module_connection(
10917 module_connection,
10918 "aft-push".to_string(),
10919 PROTOCOL_VERSION,
10920 manifest_concurrency(®istration.manifest),
10921 FrameSink::new(module_tx),
10922 )
10923 .unwrap();
10924
10925 let unknown = Frame::build(
10927 FrameType::Push,
10928 control_flags(),
10929 0,
10930 0,
10931 5,
10932 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
10933 )
10934 .unwrap();
10935 let out = handler.handle_status_update(endpoint, unknown).unwrap();
10936 assert!(
10937 out.is_empty(),
10938 "unknown push op must be ignored, got {out:?}"
10939 );
10940
10941 let malformed = Frame::build(
10943 FrameType::Push,
10944 control_flags(),
10945 0,
10946 0,
10947 6,
10948 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
10949 )
10950 .unwrap();
10951 let out = handler.handle_status_update(endpoint, malformed).unwrap();
10952 assert_eq!(out.len(), 1);
10953 assert_eq!(out[0].header.ty, FrameType::Error);
10954 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
10955 }
10956
10957 #[test]
10958 fn hello_rejected_when_connection_already_owns_client_routes() {
10959 let registry = Arc::new(Registry::default());
10960 let forwarding = Arc::new(ForwardingTable::default());
10961 let handler =
10962 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10963 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
10965 let client_connection = ConnectionId::new(202);
10966
10967 let responses = handler
10970 .handle_control(
10971 client_connection,
10972 hello_frame("aft-second", PROTOCOL_VERSION, 9),
10973 )
10974 .unwrap();
10975 assert_eq!(responses[0].header.ty, FrameType::Error);
10976 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
10977 assert!(registry.get_module("aft-second").unwrap().is_none());
10978 }
10979
10980 #[tokio::test]
10981 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
10982 let registry = Arc::new(Registry::default());
10983 let forwarding = Arc::new(ForwardingTable::default());
10984 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
10985 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
10986 hello_via_sink(
10987 &handler,
10988 &module_ctx,
10989 &mut module_rx,
10990 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
10991 )
10992 .await;
10993 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
10994 let pending = forwarding
10995 .begin_route_bind_relay_for_test(
10996 client_ctx.connection_id,
10997 client_ctx.egress.clone(),
10998 2,
10999 "alpha",
11000 )
11001 .unwrap();
11002 forwarding
11003 .complete_pending_relay(
11004 module_ctx.connection_id,
11005 pending.corr,
11006 RouteBindRelayOutcome::Accepted,
11007 )
11008 .unwrap();
11009 client_rx.try_recv().unwrap();
11010 for module_id in ["beta", "alpha"] {
11011 let replies = handler
11012 .handle_control_frame(
11013 &module_ctx,
11014 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
11015 )
11016 .await
11017 .unwrap();
11018 assert_eq!(replies.len(), 1, "second HELLO must be refused");
11019 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
11020 }
11021 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
11022 assert!(registry.get_module("beta").unwrap().is_none());
11023 assert!(matches!(
11024 forwarding
11025 .lookup_data_route(
11026 client_ctx.connection_id,
11027 pending.client_channel,
11028 pending.client_epoch,
11029 )
11030 .unwrap(),
11031 DataRoute::Client(DataRouteState::Bound(_))
11032 ));
11033 assert!(handler
11034 .hello_launch_nonces
11035 .lock()
11036 .unwrap()
11037 .presented(module_ctx.connection_id, Some("alpha-nonce")));
11038 assert!(module_rx.try_recv().is_err());
11039 }
11040
11041 #[test]
11042 fn reserved_module_hello_requires_matching_launch_nonce() {
11043 let registry = Arc::new(Registry::default());
11044 let supervisor = SupervisorHandle::new();
11045 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
11048 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11049
11050 let no_nonce = handler
11052 .handle_control(
11053 ConnectionId::new(1),
11054 hello_frame("vault", PROTOCOL_VERSION, 1),
11055 )
11056 .unwrap();
11057 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
11058 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
11059 assert!(registry.get_module("vault").unwrap().is_none());
11060
11061 let wrong = handler
11063 .handle_control(
11064 ConnectionId::new(2),
11065 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
11066 )
11067 .unwrap();
11068 assert_eq!(wrong[0].header.ty, FrameType::Error);
11069 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
11070 assert!(registry.get_module("vault").unwrap().is_none());
11071
11072 let ok = handler
11074 .handle_control(
11075 ConnectionId::new(3),
11076 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
11077 )
11078 .unwrap();
11079 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
11080 assert!(registry.get_module("vault").unwrap().is_some());
11081 }
11082
11083 #[test]
11084 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
11085 let registry = Arc::new(Registry::default());
11086 let supervisor = SupervisorHandle::new();
11087 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11088 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11089 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11090
11091 let squat = handler
11092 .handle_control(
11093 ConnectionId::new(1),
11094 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
11095 )
11096 .unwrap();
11097 assert_eq!(squat[0].header.ty, FrameType::Error);
11098 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
11099 assert!(parse_error(&squat[0])["message"]
11100 .as_str()
11101 .unwrap()
11102 .contains("fed:"));
11103
11104 let accepted_peer = handler
11105 .handle_control(
11106 ConnectionId::new(2),
11107 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
11108 )
11109 .unwrap();
11110 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
11111
11112 let accepted_short = handler
11113 .handle_control(
11114 ConnectionId::new(3),
11115 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
11116 )
11117 .unwrap();
11118 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
11119
11120 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
11121 let response = handler
11122 .handle_control(
11123 ConnectionId::new(conn),
11124 hello_frame(module_id, PROTOCOL_VERSION, conn),
11125 )
11126 .unwrap();
11127 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
11128 }
11129 }
11130
11131 #[test]
11132 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
11133 let registry = Arc::new(Registry::default());
11134 let supervisor = SupervisorHandle::new();
11135 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11136 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11137 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
11138 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11139
11140 let owner_nonce = handler
11141 .handle_control(
11142 ConnectionId::new(1),
11143 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
11144 )
11145 .unwrap();
11146 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
11147 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
11148 assert!(registry.get_module("fed:special").unwrap().is_none());
11149
11150 let exact_nonce = handler
11151 .handle_control(
11152 ConnectionId::new(2),
11153 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
11154 )
11155 .unwrap();
11156 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
11157 assert!(registry.get_module("fed:special").unwrap().is_some());
11158 }
11159
11160 #[test]
11161 fn non_reserved_module_ignores_launch_nonce() {
11162 let registry = Arc::new(Registry::default());
11163 let handler = ControlHandler::new(Arc::clone(®istry));
11166 let no_nonce = handler
11167 .handle_control(
11168 ConnectionId::new(1),
11169 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
11170 )
11171 .unwrap();
11172 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
11173 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
11174
11175 let echoed_nonce = handler
11176 .handle_control(
11177 ConnectionId::new(2),
11178 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
11179 )
11180 .unwrap();
11181 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
11182 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
11183 }
11184
11185 #[test]
11186 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
11187 let handler = ControlHandler::default();
11188 let conn = ConnectionId::new(1);
11189 let malformed = Frame::build(
11190 FrameType::Hello,
11191 control_flags(),
11192 0,
11193 0,
11194 3,
11195 b"{not json".to_vec(),
11196 )
11197 .unwrap();
11198
11199 let error = handler.handle_control(conn, malformed).unwrap();
11200 assert_eq!(error[0].header.ty, FrameType::Error);
11201 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
11202
11203 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11204 let pong = handler.handle_control(conn, ping).unwrap();
11205 assert_eq!(pong[0].header.ty, FrameType::Pong);
11206 assert_eq!(pong[0].header.corr, 4);
11207 }
11208
11209 #[test]
11210 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
11211 let registry = Arc::new(Registry::default());
11212 let handler = ControlHandler::new(Arc::clone(®istry));
11213
11214 handler
11215 .handle_control(
11216 ConnectionId::new(1),
11217 hello_frame("aft", PROTOCOL_VERSION, 1),
11218 )
11219 .unwrap();
11220 let duplicate = handler
11221 .handle_control(
11222 ConnectionId::new(2),
11223 hello_frame("aft", PROTOCOL_VERSION, 2),
11224 )
11225 .unwrap();
11226
11227 assert_eq!(duplicate[0].header.ty, FrameType::Error);
11228 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11229 let registration = registry.get_module("aft").unwrap().unwrap();
11230 assert_eq!(registration.connection_id, ConnectionId::new(1));
11231 }
11232
11233 #[test]
11234 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
11235 let registry = Arc::new(Registry::default());
11236 let forwarding = Arc::new(ForwardingTable::default());
11237 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
11238 let handler =
11239 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11240 .with_process_liveness(process_liveness);
11241 let (ctx, route_channel, route_epoch) =
11242 bind_liveness_route(®istry, &forwarding, "aft-dead");
11243 let responses = handler
11244 .handle_route_poll(
11245 &ctx,
11246 route_poll_frame(41, PollKind::Liveness, route_channel),
11247 route_channel,
11248 route_epoch,
11249 PollKind::Liveness,
11250 )
11251 .unwrap();
11252
11253 assert_eq!(responses.len(), 1);
11254 assert_eq!(responses[0].header.ty, FrameType::Response);
11255 assert_route_poll_liveness(&responses[0], false);
11256 }
11257
11258 #[test]
11259 fn liveness_poll_without_process_source_uses_bound_route() {
11260 let registry = Arc::new(Registry::default());
11261 let forwarding = Arc::new(ForwardingTable::default());
11262 let handler =
11263 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11264 let (ctx, route_channel, route_epoch) =
11265 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
11266 let responses = handler
11267 .handle_route_poll(
11268 &ctx,
11269 route_poll_frame(42, PollKind::Liveness, route_channel),
11270 route_channel,
11271 route_epoch,
11272 PollKind::Liveness,
11273 )
11274 .unwrap();
11275
11276 assert_route_poll_liveness(&responses[0], true);
11277 }
11278
11279 #[test]
11280 fn liveness_poll_untracked_process_source_uses_bound_route() {
11281 let registry = Arc::new(Registry::default());
11282 let forwarding = Arc::new(ForwardingTable::default());
11283 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
11284 let handler =
11285 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11286 .with_process_liveness(process_liveness);
11287 let (ctx, route_channel, route_epoch) =
11288 bind_liveness_route(®istry, &forwarding, "aft-untracked");
11289 let responses = handler
11290 .handle_route_poll(
11291 &ctx,
11292 route_poll_frame(43, PollKind::Liveness, route_channel),
11293 route_channel,
11294 route_epoch,
11295 PollKind::Liveness,
11296 )
11297 .unwrap();
11298
11299 assert_route_poll_liveness(&responses[0], true);
11300 }
11301
11302 #[tokio::test]
11303 async fn unknown_op_returns_unknown_control_op() {
11304 let handler = ControlHandler::default();
11305 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
11306 let request = Frame::build(
11307 FrameType::Request,
11308 control_flags(),
11309 0,
11310 0,
11311 55,
11312 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
11313 )
11314 .unwrap();
11315
11316 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11317
11318 assert_eq!(response.len(), 1);
11319 assert_eq!(response[0].header.ty, FrameType::Error);
11320 assert_eq!(response[0].header.corr, 55);
11321 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
11322 }
11323
11324 #[tokio::test]
11325 async fn supervisor_provenance_rejects_unknown_exact_module() {
11326 let handler = ControlHandler::default();
11327 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
11328 let request = Frame::build(
11329 FrameType::Request,
11330 control_flags(),
11331 0,
11332 0,
11333 57,
11334 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
11335 )
11336 .unwrap();
11337
11338 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11339
11340 assert_eq!(response.len(), 1);
11341 assert_eq!(response[0].header.ty, FrameType::Error);
11342 assert_eq!(response[0].header.corr, 57);
11343 let error = parse_error(&response[0]);
11344 assert_eq!(error["code"], "unknown_module");
11345 assert_eq!(error["message"], "module_id 'missing' is not supervised");
11346 }
11347
11348 #[test]
11349 fn provenance_probe_override_keeps_handler_tests_deterministic() {
11350 let expected = subc_control::RunningImageAgreement::Unavailable {
11351 reason: subc_control::RunningImageUnavailableReason::HashFailed,
11352 };
11353 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
11354 assert_eq!(handler.provenance_probe_override, Some(expected));
11355 }
11356
11357 #[test]
11358 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
11359 let verdict = reload_verdict(
11360 std::path::Path::new("/bin/new"),
11361 Some(std::path::Path::new("/bin/old")),
11362 subc_control::RunningImageAgreement::Unavailable {
11363 reason: subc_control::RunningImageUnavailableReason::HashFailed,
11364 },
11365 );
11366 assert!(matches!(
11367 verdict.path,
11368 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
11369 if configured == std::path::Path::new("/bin/new")
11370 && spawned_from == std::path::Path::new("/bin/old")
11371 ));
11372 }
11373
11374 #[test]
11375 fn reload_verdict_detects_replaced_image_at_same_path() {
11376 let image = subc_control::RunningImageAgreement::Mismatch {
11377 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
11378 digest: "old".into(),
11379 },
11380 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
11381 digest: "new".into(),
11382 },
11383 };
11384 let verdict = reload_verdict(
11385 std::path::Path::new("/bin/same"),
11386 Some(std::path::Path::new("/bin/same")),
11387 image.clone(),
11388 );
11389 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11390 assert_eq!(verdict.image, image);
11391 }
11392
11393 #[test]
11394 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
11395 let image = subc_control::RunningImageAgreement::Unavailable {
11396 reason: subc_control::RunningImageUnavailableReason::NotRunning,
11397 };
11398 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
11399 assert_eq!(
11400 verdict.path,
11401 subc_control::ReloadPathAgreement::Unavailable {
11402 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
11403 }
11404 );
11405 assert_eq!(verdict.image, image);
11406
11407 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
11408 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11409 };
11410 let verdict = reload_verdict(
11411 std::path::Path::new("/bin/same"),
11412 Some(std::path::Path::new("/bin/same")),
11413 unconfirmed.clone(),
11414 );
11415 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11416 assert_eq!(verdict.image, unconfirmed);
11417 }
11418
11419 #[test]
11420 fn reload_verdict_preserves_each_image_unavailability_reason() {
11421 use subc_control::RunningImageUnavailableReason as Reason;
11422
11423 for reason in [
11424 Reason::NotRunning,
11425 Reason::UnsupportedPlatform,
11426 Reason::RunningExecutableUnreadable,
11427 Reason::SpawnedPathUnreadable,
11428 Reason::HashFailed,
11429 Reason::ProcessIdentityUnconfirmed,
11430 Reason::Unknown("future_probe_reason".to_string()),
11431 ] {
11432 let image = subc_control::RunningImageAgreement::Unavailable {
11433 reason: reason.clone(),
11434 };
11435 let verdict = reload_verdict(
11436 std::path::Path::new("/bin/same"),
11437 Some(std::path::Path::new("/bin/same")),
11438 image.clone(),
11439 );
11440 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11441 assert_eq!(verdict.image, image, "{reason:?}");
11442 }
11443 }
11444
11445 #[tokio::test]
11446 async fn malformed_control_bodies_return_invalid_control_body() {
11447 let handler = ControlHandler::default();
11448 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
11449
11450 for (corr, body) in [
11451 (56, br#"{"route_channel":1}"#.as_slice()),
11452 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
11453 (
11454 58,
11455 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
11456 ),
11457 ] {
11458 let request = Frame::build(
11459 FrameType::Request,
11460 control_flags(),
11461 0,
11462 0,
11463 corr,
11464 body.to_vec(),
11465 )
11466 .unwrap();
11467 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11468
11469 assert_eq!(response.len(), 1);
11470 assert_eq!(response[0].header.ty, FrameType::Error);
11471 assert_eq!(response[0].header.corr, corr);
11472 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
11473 }
11474 }
11475
11476 #[tokio::test]
11477 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
11478 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11479 let registry = Arc::new(Registry::default());
11480 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11481 let router = Router::with_control_handler(Arc::clone(&control));
11482 let connection = router.begin_connection();
11483 let (ctx, mut rx) = route_ctx(connection.id());
11484
11485 router
11486 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
11487 .await
11488 .unwrap();
11489 let response = rx.recv().await.unwrap();
11490 let ack = parse_ack(&response);
11491 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11492 let channel = 1;
11493
11494 let goodbye =
11495 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
11496 router.route_for_connection(&ctx, goodbye).await.unwrap();
11497 assert!(rx.try_recv().is_err());
11498 assert!(registry.get_module("aft").unwrap().is_none());
11499
11500 router
11501 .route_for_connection(&ctx, channel_request(channel, 13))
11502 .await
11503 .unwrap();
11504 let error_frame = rx.recv().await.unwrap();
11505 assert_eq!(error_frame.header.ty, FrameType::Error);
11506 assert_eq!(error_frame.header.channel, channel);
11507 let captured = crate::router::test_log::captured_logs(&logs);
11508 assert_eq!(
11509 captured
11510 .lines()
11511 .filter(|line| {
11512 line.contains("module_id=aft")
11513 && line.contains("reason=explicit_goodbye")
11514 && line.contains("module registration ended")
11515 })
11516 .count(),
11517 1,
11518 "unexpected GOODBYE registry log: {captured}"
11519 );
11520 }
11521
11522 #[tokio::test]
11523 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
11524 let registry = Arc::new(Registry::default());
11525 let handler = ControlHandler::new(registry).with_capability_config(
11526 [("prov".to_string(), true), ("cons".to_string(), true)],
11527 BTreeMap::new(),
11528 );
11529 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
11530 register_capability_manifest(
11531 &handler,
11532 &provider_ctx,
11533 &mut provider_rx,
11534 capability_manifest("prov", &["thing/v1"], &[]),
11535 1,
11536 )
11537 .await;
11538 let mut consumer = capability_manifest("cons", &[], &[]);
11539 consumer.capabilities.as_mut().unwrap().requires.push(
11540 subc_protocol::manifest::CapabilityRequirement {
11541 capability: "thing/v1".to_string(),
11542 need: subc_protocol::manifest::CapabilityNeed::Required,
11543 },
11544 );
11545 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
11546 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
11547 assert_eq!(
11548 handler.capability_evaluator.verdict("cons", "thing/v1"),
11549 Some(CapabilityVerdict::Provided)
11550 );
11551 let (mut client_rx, _) = open_route_for_capability_test(
11552 &handler,
11553 &provider_ctx,
11554 &mut provider_rx,
11555 703,
11556 3,
11557 "prov",
11558 None,
11559 )
11560 .await;
11561 let goodbye =
11562 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11563 handler
11564 .handle_control_frame(&provider_ctx, goodbye)
11565 .await
11566 .unwrap();
11567 assert_eq!(
11568 handler.capability_evaluator.verdict("cons", "thing/v1"),
11569 Some(CapabilityVerdict::NeverProvided)
11570 );
11571 let closed = client_rx
11572 .try_recv()
11573 .expect("GOODBYE pushes route.closed before route GOODBYE");
11574 assert!(
11575 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
11576 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
11577 );
11578 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
11579 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11580 }
11581
11582 #[tokio::test]
11583 async fn dropping_router_connection_releases_registration() {
11584 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11585 let registry = Arc::new(Registry::default());
11586 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11587 let router = Router::with_control_handler(Arc::clone(&control));
11588 let connection = router.begin_connection();
11589 let connection_id = connection.id();
11590 let (ctx, mut rx) = route_ctx(connection_id);
11591
11592 router
11593 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
11594 .await
11595 .unwrap();
11596 let response = rx.recv().await.unwrap();
11597 let ack = parse_ack(&response);
11598 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11599 assert!(registry.get_module("aft").unwrap().is_some());
11600
11601 drop(connection);
11602
11603 assert!(registry.get_module("aft").unwrap().is_none());
11604 assert_eq!(registry.active_registration_count().unwrap(), 0);
11605
11606 control
11607 .cleanup_connection(ConnectionId::new(u64::MAX))
11608 .unwrap();
11609 let captured = crate::router::test_log::captured_logs(&logs);
11610 println!("captured registry lifecycle logs:\n{captured}");
11611 let events: Vec<_> = captured
11612 .lines()
11613 .filter(|line| {
11614 line.contains("module registered module_id=aft ")
11615 || line.contains("module registration ended")
11616 })
11617 .collect();
11618 assert_eq!(
11619 events.len(),
11620 2,
11621 "unexpected registry lifecycle logs: {captured}"
11622 );
11623 assert!(events[0].contains("module registered module_id=aft "));
11624 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
11625 assert!(events[1].contains(&format!(
11626 "module_id=aft connection_id={}",
11627 connection_id.get()
11628 )));
11629 assert!(events[1].contains("reason=connection_closed"));
11630 assert!(events[1].contains("module registration ended"));
11631 assert_eq!(
11632 captured
11633 .lines()
11634 .filter(|line| line.contains("module registered module_id=aft "))
11635 .count(),
11636 1,
11637 "legacy registration admission line must appear once: {captured}"
11638 );
11639 }
11640
11641 #[tokio::test]
11642 async fn hello_registration_keeps_legacy_module_registered_line_once() {
11643 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11644 let registry = Arc::new(Registry::default());
11645 let control = ControlHandler::new(Arc::clone(®istry));
11646 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
11647 hello_via_sink(
11648 &control,
11649 &ctx,
11650 &mut rx,
11651 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
11652 )
11653 .await;
11654
11655 let captured = crate::router::test_log::captured_logs(&logs);
11656 let admissions: Vec<_> = captured
11657 .lines()
11658 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
11659 .collect();
11660 assert_eq!(
11661 admissions.len(),
11662 1,
11663 "legacy admission line must remain exactly once: {captured}"
11664 );
11665 assert!(admissions[0].contains("routable_provider=true"));
11666 assert!(admissions[0].contains("connection_id=777"));
11667 }
11668
11669 fn capability_manifest(
11670 module_id: &str,
11671 provides: &[&str],
11672 must_never_reach: &[&str],
11673 ) -> ModuleManifest {
11674 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11675 manifest.capabilities = Some(CapabilityDeclarations {
11676 provides: provides
11677 .iter()
11678 .map(|capability| (*capability).to_string())
11679 .collect(),
11680 requires: Vec::new(),
11681 must_never_reach: must_never_reach
11682 .iter()
11683 .map(|capability| (*capability).to_string())
11684 .collect(),
11685 });
11686 manifest
11687 }
11688
11689 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
11690 Frame::build(
11691 FrameType::Hello,
11692 control_flags(),
11693 0,
11694 0,
11695 corr,
11696 serde_json::to_vec(&ModuleHelloBody {
11697 protocol_ver: manifest.protocol_ver,
11698 manifest,
11699 control_ops: None,
11700 launch_nonce: None,
11701 })
11702 .expect("capability test HELLO serializes"),
11703 )
11704 .expect("capability test HELLO frame builds")
11705 }
11706
11707 fn catalog_update_with_capabilities_frame(
11708 corr: u64,
11709 capabilities: CapabilityDeclarations,
11710 ) -> Frame {
11711 Frame::build(
11712 FrameType::Request,
11713 control_flags(),
11714 0,
11715 0,
11716 corr,
11717 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11718 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
11719 capabilities: Some(capabilities),
11720 ready: None,
11721 })
11722 .expect("capability catalog.update serializes"),
11723 )
11724 .expect("capability catalog.update frame builds")
11725 }
11726
11727 async fn register_capability_manifest(
11728 handler: &ControlHandler,
11729 ctx: &RouteCtx,
11730 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11731 manifest: ModuleManifest,
11732 corr: u64,
11733 ) {
11734 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
11735 }
11736
11737 async fn open_route_for_capability_test(
11738 handler: &ControlHandler,
11739 target_ctx: &RouteCtx,
11740 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11741 client_connection_id: u64,
11742 corr: u64,
11743 target_module_id: &str,
11744 consumer_identity: Option<ConsumerIdentity>,
11745 ) -> (
11746 mpsc::Receiver<crate::router::OutboundFrame>,
11747 ModuleControlRequest,
11748 ) {
11749 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
11750 let route_handler = handler.clone();
11751 let target_module_id = target_module_id.to_string();
11752 let route_task = tokio::spawn(async move {
11753 route_handler
11754 .handle_control_frame(
11755 &client_ctx,
11756 route_open_frame_with_admission_facts(
11757 corr,
11758 &target_module_id,
11759 unique_project_root("admission-facts"),
11760 consumer_identity,
11761 None,
11762 ),
11763 )
11764 .await
11765 .expect("capability test route.open succeeds")
11766 });
11767 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
11768 .await
11769 .expect("capability test route.open must reach route.bind")
11770 .expect("target control receiver stays open");
11771 let bind_request: ModuleControlRequest =
11772 serde_json::from_slice(&bind.body).expect("route.bind decodes");
11773 handler
11774 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
11775 .await
11776 .expect("capability test route.bind ACK succeeds");
11777 assert!(route_task.await.expect("route.open task joins").is_empty());
11778 let opened = client_rx
11779 .recv()
11780 .await
11781 .expect("successful route.open publishes a response");
11782 assert!(matches!(
11783 serde_json::from_slice::<ClientControlResponse>(&opened.body),
11784 Ok(ClientControlResponse::RouteOpen { .. })
11785 ));
11786 (client_rx, bind_request)
11787 }
11788
11789 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
11790 assert_eq!(frame.header.ty, FrameType::Push);
11791 assert_eq!(frame.header.channel, 0);
11792 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
11793 .expect("route.closed control push decodes");
11794 let ClientControlPush::RouteClosed { channels, .. } = &push else {
11795 panic!("expected route.closed");
11796 };
11797 assert_eq!(channels.len(), 1, "exactly one violating route closed");
11798 let channels = channels.clone();
11799 assert_eq!(
11800 push,
11801 ClientControlPush::RouteClosed {
11802 module_id: target_module_id.to_string(),
11803 channels,
11804 reason: RouteCloseReason::CapabilityDenied,
11805 drained: false,
11806 abandoned: 0,
11807 excluded_subscriptions: 0,
11808 terminal: Some(false),
11809 }
11810 );
11811 }
11812
11813 #[tokio::test]
11814 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
11815 let registry = Arc::new(Registry::default());
11816 let forwarding = Arc::new(ForwardingTable::default());
11817 let supervisor = SupervisorHandle::new();
11818 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11819 let handler =
11820 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11821 .with_supervisor(supervisor);
11822 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
11823 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
11824 register_capability_manifest(
11825 &handler,
11826 &target_ctx,
11827 &mut target_rx,
11828 capability_manifest("target", &["credentials-provider/v1"], &[]),
11829 1,
11830 )
11831 .await;
11832 register_capability_manifest(
11833 &handler,
11834 &opener_ctx,
11835 &mut opener_rx,
11836 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11837 2,
11838 )
11839 .await;
11840
11841 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
11842 let replies = handler
11843 .handle_control_frame(
11844 &client_ctx,
11845 route_open_frame_with_admission_facts(
11846 3,
11847 "target",
11848 unique_project_root("admission-facts"),
11849 Some(ConsumerIdentity {
11850 module_id: "opener".to_string(),
11851 launch_nonce: "opener-nonce".to_string(),
11852 }),
11853 None,
11854 ),
11855 )
11856 .await
11857 .expect("denied route.open returns a typed frame");
11858 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
11859 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11860 assert!(
11861 target_rx.try_recv().is_err(),
11862 "forbidden route.open must not relay route.bind"
11863 );
11864 }
11865
11866 #[tokio::test]
11867 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
11868 let registry = Arc::new(Registry::default());
11869 let forwarding = Arc::new(ForwardingTable::default());
11870 let supervisor = SupervisorHandle::new();
11871 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11872 let handler =
11873 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11874 .with_supervisor(supervisor);
11875 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
11876 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
11877 register_capability_manifest(
11878 &handler,
11879 &target_ctx,
11880 &mut target_rx,
11881 capability_manifest("target", &["credentials-provider/v1"], &[]),
11882 1,
11883 )
11884 .await;
11885 register_capability_manifest(
11886 &handler,
11887 &old_opener_ctx,
11888 &mut old_opener_rx,
11889 capability_manifest("opener", &[], &[]),
11890 2,
11891 )
11892 .await;
11893 let (mut client_rx, _) = open_route_for_capability_test(
11894 &handler,
11895 &target_ctx,
11896 &mut target_rx,
11897 712,
11898 3,
11899 "target",
11900 Some(ConsumerIdentity {
11901 module_id: "opener".to_string(),
11902 launch_nonce: "opener-nonce".to_string(),
11903 }),
11904 )
11905 .await;
11906 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11907
11908 handler
11909 .cleanup_connection(old_opener_ctx.connection_id)
11910 .expect("old opener registration cleans up");
11911 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
11912 register_capability_manifest(
11913 &handler,
11914 &new_opener_ctx,
11915 &mut new_opener_rx,
11916 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11917 4,
11918 )
11919 .await;
11920
11921 assert_capability_denied_push(
11922 client_rx
11923 .try_recv()
11924 .expect("HELLO deny addition must emit route.closed")
11925 .frame,
11926 "target",
11927 );
11928 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
11929 assert!(matches!(
11930 target_rx.try_recv(),
11931 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
11932 ));
11933 }
11934
11935 #[tokio::test]
11936 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
11937 let registry = Arc::new(Registry::default());
11938 let forwarding = Arc::new(ForwardingTable::default());
11939 let supervisor = SupervisorHandle::new();
11940 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
11941 let handler =
11942 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11943 .with_supervisor(supervisor);
11944 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
11945 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
11946 register_capability_manifest(
11947 &handler,
11948 &target_ctx,
11949 &mut target_rx,
11950 capability_manifest("target", &[], &[]),
11951 1,
11952 )
11953 .await;
11954 register_capability_manifest(
11955 &handler,
11956 &opener_ctx,
11957 &mut opener_rx,
11958 capability_manifest("opener", &[], &["credentials-provider/v1"]),
11959 2,
11960 )
11961 .await;
11962 let (mut client_rx, _) = open_route_for_capability_test(
11963 &handler,
11964 &target_ctx,
11965 &mut target_rx,
11966 722,
11967 3,
11968 "target",
11969 Some(ConsumerIdentity {
11970 module_id: "opener".to_string(),
11971 launch_nonce: "opener-nonce".to_string(),
11972 }),
11973 )
11974 .await;
11975 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
11976
11977 let replies = handler
11978 .handle_control_frame(
11979 &target_ctx,
11980 catalog_update_with_capabilities_frame(
11981 4,
11982 CapabilityDeclarations {
11983 provides: vec!["credentials-provider/v1".to_string()],
11984 requires: Vec::new(),
11985 must_never_reach: Vec::new(),
11986 },
11987 ),
11988 )
11989 .await
11990 .expect("claim catalog.update succeeds");
11991 assert!(matches!(
11992 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
11993 Ok(ModuleControlResponseToModule::CatalogUpdate {})
11994 ));
11995 assert_capability_denied_push(
11996 client_rx
11997 .try_recv()
11998 .expect("claim addition must emit route.closed")
11999 .frame,
12000 "target",
12001 );
12002 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12003 assert!(matches!(
12004 target_rx.try_recv(),
12005 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
12006 ));
12007 }
12008
12009 #[tokio::test]
12010 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
12011 let registry = Arc::new(Registry::default());
12012 let forwarding = Arc::new(ForwardingTable::default());
12013 let supervisor = SupervisorHandle::new();
12014 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12015 let handler =
12016 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12017 .with_supervisor(supervisor);
12018 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
12019 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
12020 register_capability_manifest(
12021 &handler,
12022 &target_ctx,
12023 &mut target_rx,
12024 capability_manifest("target", &["credentials-provider/v1"], &[]),
12025 1,
12026 )
12027 .await;
12028 register_capability_manifest(
12029 &handler,
12030 &opener_ctx,
12031 &mut opener_rx,
12032 capability_manifest("opener", &[], &[]),
12033 2,
12034 )
12035 .await;
12036 let (mut client_rx, _) = open_route_for_capability_test(
12037 &handler,
12038 &target_ctx,
12039 &mut target_rx,
12040 732,
12041 3,
12042 "target",
12043 Some(ConsumerIdentity {
12044 module_id: "opener".to_string(),
12045 launch_nonce: "opener-nonce".to_string(),
12046 }),
12047 )
12048 .await;
12049 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12050
12051 handler
12052 .handle_control_frame(
12053 &target_ctx,
12054 catalog_update_with_capabilities_frame(
12055 4,
12056 CapabilityDeclarations {
12057 provides: Vec::new(),
12058 requires: Vec::new(),
12059 must_never_reach: Vec::new(),
12060 },
12061 ),
12062 )
12063 .await
12064 .expect("claim removal catalog.update succeeds");
12065 assert_eq!(
12066 forwarding.active_binding_count().unwrap(),
12067 1,
12068 "removing an attested target claim must leave the route census unchanged"
12069 );
12070 assert!(
12071 client_rx.try_recv().is_err(),
12072 "claim removal must not emit route.closed capability_denied"
12073 );
12074 assert!(
12075 target_rx.try_recv().is_err(),
12076 "claim removal must not send the target a route GOODBYE"
12077 );
12078 }
12079
12080 #[tokio::test]
12083 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
12084 let registry = Arc::new(Registry::default());
12085 let forwarding = Arc::new(ForwardingTable::default());
12086 let supervisor = SupervisorHandle::new();
12087 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12088 let handler =
12089 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12090 .with_supervisor(supervisor);
12091 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
12092 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
12093 register_capability_manifest(
12094 &handler,
12095 &target_ctx,
12096 &mut target_rx,
12097 capability_manifest("target", &["credentials-provider/v1"], &[]),
12098 1,
12099 )
12100 .await;
12101 register_capability_manifest(
12102 &handler,
12103 &opener_ctx,
12104 &mut opener_rx,
12105 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12106 2,
12107 )
12108 .await;
12109
12110 let (_client_rx, bind) = open_route_for_capability_test(
12111 &handler,
12112 &target_ctx,
12113 &mut target_rx,
12114 742,
12115 3,
12116 "target",
12117 None,
12118 )
12119 .await;
12120 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12121 panic!("direct scope-honesty route must bind");
12122 };
12123 assert_eq!(principal, Some(Principal::Direct));
12124 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12125 }
12126
12127 #[tokio::test]
12130 async fn must_never_reach_self_route_is_capability_forbidden() {
12131 let registry = Arc::new(Registry::default());
12132 let forwarding = Arc::new(ForwardingTable::default());
12133 let supervisor = SupervisorHandle::new();
12134 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
12135 let handler =
12136 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12137 .with_supervisor(supervisor);
12138 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
12139 register_capability_manifest(
12140 &handler,
12141 &self_ctx,
12142 &mut self_rx,
12143 capability_manifest(
12144 "self-provider",
12145 &["credentials-provider/v1"],
12146 &["credentials-provider/v1"],
12147 ),
12148 1,
12149 )
12150 .await;
12151
12152 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
12153 let replies = handler
12154 .handle_control_frame(
12155 &client_ctx,
12156 route_open_frame_with_admission_facts(
12157 2,
12158 "self-provider",
12159 unique_project_root("admission-facts"),
12160 Some(ConsumerIdentity {
12161 module_id: "self-provider".to_string(),
12162 launch_nonce: "self-nonce".to_string(),
12163 }),
12164 None,
12165 ),
12166 )
12167 .await
12168 .expect("self-route refusal returns a typed frame");
12169 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
12170 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12171 assert!(
12172 self_rx.try_recv().is_err(),
12173 "self denial must not relay route.bind"
12174 );
12175 }
12176
12177 #[test]
12178 fn unsupported_channel_zero_frame_returns_error() {
12179 let handler = ControlHandler::default();
12180 let request = Frame::build(
12181 FrameType::Request,
12182 control_flags(),
12183 0,
12184 0,
12185 21,
12186 b"opaque".to_vec(),
12187 )
12188 .unwrap();
12189
12190 let response = handler
12191 .handle_control(ConnectionId::new(1), request)
12192 .unwrap();
12193
12194 assert_eq!(response[0].header.ty, FrameType::Error);
12195 assert_eq!(
12196 parse_error(&response[0])["code"],
12197 "unsupported_control_frame"
12198 );
12199 }
12200
12201 mod swap {
12206 use super::*;
12207
12208 const INCUMBENT: ConnectionId = ConnectionId::new(30);
12209 const CANDIDATE: ConnectionId = ConnectionId::new(40);
12210
12211 struct Swap {
12212 registry: Arc<Registry>,
12213 forwarding: Arc<ForwardingTable>,
12214 handler: ControlHandler,
12215 incumbent_ctx: RouteCtx,
12216 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12217 candidate_ctx: RouteCtx,
12218 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12219 }
12220
12221 async fn swap_with_incumbent() -> Swap {
12222 let registry = Arc::new(Registry::default());
12223 let forwarding = Arc::new(ForwardingTable::default());
12224 let handler =
12225 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12226 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
12227 hello_via_sink(
12228 &handler,
12229 &incumbent_ctx,
12230 &mut incumbent_rx,
12231 hello_frame("aft", PROTOCOL_VERSION, 7),
12232 )
12233 .await;
12234 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
12235 Swap {
12236 registry,
12237 forwarding,
12238 handler,
12239 incumbent_ctx,
12240 incumbent_rx,
12241 candidate_ctx,
12242 candidate_rx,
12243 }
12244 }
12245
12246 fn register_candidate(swap: &Swap, ready: Option<bool>) {
12247 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
12248 candidate_manifest.ready = ready;
12249 let registration = swap
12250 .registry
12251 .register_candidate_with_control_ops(
12252 candidate_manifest,
12253 PROTOCOL_VERSION,
12254 CANDIDATE,
12255 module_baseline_control_ops(),
12256 )
12257 .unwrap();
12258 swap.forwarding
12259 .register_candidate_module_connection(
12260 CANDIDATE,
12261 "aft".to_string(),
12262 PROTOCOL_VERSION,
12263 manifest_concurrency(®istration.manifest),
12264 swap.candidate_ctx.egress.clone(),
12265 )
12266 .unwrap();
12267 }
12268
12269 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
12270 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
12271 swap.registry.promote_candidate("aft").unwrap().unwrap();
12272 cutover.incumbent.unwrap()
12273 }
12274
12275 fn keyed_total(counters: &Value, key: &str) -> u64 {
12276 counters[key]
12277 .as_object()
12278 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
12279 .unwrap_or(0)
12280 }
12281
12282 #[tokio::test]
12283 async fn replacement_logs_old_end_and_new_admission_once() {
12284 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12285 let mut swap = swap_with_incumbent().await;
12286 swap.handler
12287 .supervisor
12288 .open_swap("aft", "candidate-nonce".to_string());
12289 hello_via_sink(
12290 &swap.handler,
12291 &swap.candidate_ctx,
12292 &mut swap.candidate_rx,
12293 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
12294 )
12295 .await;
12296 cutover(&swap);
12297 swap.handler.cleanup_connection(INCUMBENT).unwrap();
12298
12299 let captured = crate::router::test_log::captured_logs(&logs);
12300 println!("captured replacement registry logs:\n{captured}");
12301 let new_admissions: Vec<_> = captured
12302 .lines()
12303 .filter(|line| {
12304 line.contains("connection_id=40")
12305 && line.contains("swap candidate registered; not routable until cutover")
12306 })
12307 .collect();
12308 assert_eq!(
12309 new_admissions.len(),
12310 1,
12311 "unexpected admission logs: {captured}"
12312 );
12313 assert!(new_admissions[0].contains("module_id=aft"));
12314 assert!(new_admissions[0].contains("connection_id=40"));
12315 assert!(
12316 new_admissions[0].contains("swap candidate registered; not routable until cutover")
12317 );
12318 assert!(new_admissions[0].contains("ready=true"));
12319
12320 let old_admissions: Vec<_> = captured
12321 .lines()
12322 .filter(|line| {
12323 line.contains("module registered module_id=aft ")
12324 && line.contains("connection_id=30")
12325 })
12326 .collect();
12327 assert_eq!(
12328 old_admissions.len(),
12329 1,
12330 "unexpected incumbent admission: {captured}"
12331 );
12332
12333 let old_ends: Vec<_> = captured
12334 .lines()
12335 .filter(|line| {
12336 line.contains("connection_id=30") && line.contains("module registration ended")
12337 })
12338 .collect();
12339 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
12340 assert!(old_ends[0].contains("module_id=aft"));
12341 assert!(old_ends[0].contains("connection_id=30"));
12342 assert!(old_ends[0].contains("reason=replaced"));
12343 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
12344 assert_eq!(
12345 captured
12346 .lines()
12347 .filter(|line| line.contains("module registration promoted"))
12348 .count(),
12349 1,
12350 "unexpected promotion logs: {captured}"
12351 );
12352 }
12353
12354 #[tokio::test]
12360 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
12361 let mut swap = swap_with_incumbent().await;
12362 let handler = swap.handler.clone();
12363
12364 let cotenant = ConnectionId::new(31);
12366 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
12367 let (cotenant_task, cotenant_bind) = relay_route_open(
12368 &handler,
12369 cotenant,
12370 &cotenant_ctx.egress,
12371 &mut swap.incumbent_rx,
12372 100,
12373 "aft",
12374 "swap-cotenant",
12375 )
12376 .await;
12377 handler
12378 .handle_control_frame(
12379 &swap.incumbent_ctx,
12380 route_bind_ack(cotenant_bind.header.corr),
12381 )
12382 .await
12383 .unwrap();
12384 assert!(cotenant_task.await.unwrap().is_empty());
12385 let (cotenant_channel, cotenant_epoch) =
12386 published_route(&cotenant_rx.recv().await.unwrap());
12387
12388 let caller = ConnectionId::new(32);
12390 let (caller_ctx, mut caller_rx) = route_ctx(caller);
12391 let (caller_task, caller_bind) = relay_route_open(
12392 &handler,
12393 caller,
12394 &caller_ctx.egress,
12395 &mut swap.incumbent_rx,
12396 101,
12397 "aft",
12398 "swap-caller",
12399 )
12400 .await;
12401 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
12402
12403 register_candidate(&swap, None);
12404 cutover(&swap);
12405
12406 let ack = handler
12408 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
12409 .await;
12410 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
12411 if module_loop_error.is_some() {
12412 handler.cleanup_connection(INCUMBENT).unwrap();
12415 }
12416
12417 assert!(
12419 cotenant_rx.try_recv().is_err(),
12420 "the co-tenant route on the incumbent was torn down by one late ack: \
12421 {module_loop_error:?}"
12422 );
12423 assert!(matches!(
12424 swap.forwarding
12425 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
12426 .unwrap(),
12427 DataRoute::Client(DataRouteState::Bound(_))
12428 ));
12429 assert_eq!(module_loop_error, None);
12430 assert!(swap
12431 .registry
12432 .get_module_by_connection(INCUMBENT)
12433 .unwrap()
12434 .is_some());
12435
12436 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
12438 .await
12439 .expect("the incumbent is told to drop the abandoned binding")
12440 .unwrap()
12441 .frame;
12442 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12443 assert_eq!(goodbye.header.channel, abandoned_channel);
12444 assert_eq!(goodbye.header.epoch, abandoned_epoch);
12445 assert!(swap.incumbent_rx.try_recv().is_err());
12446
12447 let response = caller_task.await.unwrap();
12449 assert_eq!(response.len(), 1);
12450 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
12451 assert!(caller_rx.try_recv().is_err());
12452
12453 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
12457 let counters = handler.counters().snapshot();
12458 assert_eq!(
12459 keyed_total(&counters, "route_open_accepted_by_principal"),
12460 1
12461 );
12462 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
12463 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
12464 }
12465
12466 #[tokio::test]
12470 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
12471 let mut swap = swap_with_incumbent().await;
12472 register_candidate(&swap, None);
12473 let incumbent = cutover(&swap);
12474 swap.forwarding
12475 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
12476 .unwrap()
12477 .expect("the incumbent is still registered");
12478
12479 let client = ConnectionId::new(33);
12480 let (client_ctx, mut client_rx) = route_ctx(client);
12481 let route_handler = swap.handler.clone();
12482 let open_ctx = RouteCtx {
12483 connection_id: client,
12484 egress: client_ctx.egress.clone(),
12485 };
12486 let mut route_task = tokio::spawn(async move {
12487 route_handler
12488 .handle_control_frame(
12489 &open_ctx,
12490 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
12491 )
12492 .await
12493 .unwrap()
12494 });
12495 let bind = tokio::select! {
12496 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
12497 response = &mut route_task => {
12498 let response = response.unwrap();
12499 panic!(
12500 "post-cutover route.open was refused instead of relayed to the candidate: {}",
12501 parse_error(&response[0])["code"]
12502 );
12503 }
12504 };
12505 swap.handler
12506 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
12507 .await
12508 .unwrap();
12509 assert!(route_task.await.unwrap().is_empty());
12510 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
12511 match swap
12512 .forwarding
12513 .lookup_data_route(client, channel, epoch)
12514 .unwrap()
12515 {
12516 DataRoute::Client(DataRouteState::Bound(route)) => {
12517 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
12518 }
12519 other => panic!("expected a bound route on the candidate, got {other:?}"),
12520 }
12521 assert!(swap.incumbent_rx.try_recv().is_err());
12522 }
12523
12524 #[tokio::test]
12529 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
12530 let swap = swap_with_incumbent().await;
12531 register_candidate(&swap, Some(false));
12532 let update = Frame::build(
12533 FrameType::Request,
12534 control_flags(),
12535 0,
12536 0,
12537 55,
12538 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
12539 provides: manifest("aft", PROTOCOL_VERSION).provides,
12540 capabilities: None,
12541 ready: Some(true),
12542 })
12543 .unwrap(),
12544 )
12545 .unwrap();
12546
12547 let replies = swap
12548 .handler
12549 .handle_control_frame(&swap.candidate_ctx, update)
12550 .await
12551 .unwrap();
12552
12553 assert_eq!(replies.len(), 1);
12554 assert_eq!(
12555 replies[0].header.ty,
12556 FrameType::Response,
12557 "candidate catalog.update was refused: {:?}",
12558 serde_json::from_slice::<Value>(&replies[0].body).ok()
12559 );
12560 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
12561 assert_eq!(
12562 swap.registry
12563 .get_module("aft")
12564 .unwrap()
12565 .unwrap()
12566 .connection_id,
12567 INCUMBENT
12568 );
12569 }
12570 }
12571
12572 mod swap_admission {
12576 use super::*;
12577
12578 const INCUMBENT_NONCE: &str = "incumbent-nonce";
12579 const CANDIDATE_NONCE: &str = "candidate-nonce";
12580
12581 fn handler_with_incumbent(
12582 module_id: &str,
12583 reserved: bool,
12584 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
12585 let registry = Arc::new(Registry::default());
12586 let supervisor = SupervisorHandle::new();
12587 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
12588 if reserved {
12589 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
12590 }
12591 let handler =
12592 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
12593 let incumbent = handler
12594 .handle_control(
12595 ConnectionId::new(1),
12596 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
12597 )
12598 .unwrap();
12599 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
12600 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
12601 (registry, supervisor, handler)
12602 }
12603
12604 #[test]
12610 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
12611 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
12612
12613 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
12614 let replies = handler
12615 .handle_control(
12616 ConnectionId::new(connection),
12617 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
12618 )
12619 .unwrap();
12620 assert_eq!(replies[0].header.ty, FrameType::Error);
12621 assert_eq!(
12622 parse_error(&replies[0])["code"],
12623 "swap_token_invalid",
12624 "nonce {nonce:?}"
12625 );
12626 }
12627 assert!(registry.get_candidate("aft").unwrap().is_none());
12628 assert_eq!(
12629 registry.get_module("aft").unwrap().unwrap().connection_id,
12630 ConnectionId::new(1)
12631 );
12632
12633 let admitted = handler
12636 .handle_control(
12637 ConnectionId::new(4),
12638 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
12639 )
12640 .unwrap();
12641 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
12642 assert_eq!(
12643 registry
12644 .get_candidate("aft")
12645 .unwrap()
12646 .unwrap()
12647 .connection_id,
12648 ConnectionId::new(4)
12649 );
12650 assert_eq!(
12651 registry.get_module("aft").unwrap().unwrap().connection_id,
12652 ConnectionId::new(1),
12653 "the candidate must not take the active slot"
12654 );
12655 let replayed = handler
12656 .handle_control(
12657 ConnectionId::new(5),
12658 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
12659 )
12660 .unwrap();
12661 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
12662
12663 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
12667 let squatter = handler
12668 .handle_control(
12669 ConnectionId::new(6),
12670 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
12671 )
12672 .unwrap();
12673 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
12674 assert!(
12675 registry.get_module("aft").unwrap().is_none(),
12676 "a squatter took the active slot of an id being swapped"
12677 );
12678 }
12679
12680 #[test]
12685 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
12686 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
12687
12688 let replies = handler
12689 .handle_control(
12690 ConnectionId::new(2),
12691 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12692 )
12693 .unwrap();
12694
12695 assert_eq!(
12696 replies[0].header.ty,
12697 FrameType::HelloAck,
12698 "reserved candidate refused: {:?}",
12699 serde_json::from_slice::<Value>(&replies[0].body).ok()
12700 );
12701 assert_eq!(
12702 registry
12703 .get_candidate("vault")
12704 .unwrap()
12705 .unwrap()
12706 .connection_id,
12707 ConnectionId::new(2)
12708 );
12709 }
12710
12711 #[test]
12714 fn without_an_open_swap_the_ordinary_gates_decide() {
12715 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
12716 supervisor.close_swap("vault");
12717
12718 let candidate = handler
12719 .handle_control(
12720 ConnectionId::new(2),
12721 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12722 )
12723 .unwrap();
12724 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
12725 let duplicate = handler
12726 .handle_control(
12727 ConnectionId::new(3),
12728 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
12729 )
12730 .unwrap();
12731 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12732 assert!(registry.get_candidate("vault").unwrap().is_none());
12733 }
12734 }
12735
12736 mod scopes {
12740 mod delta {
12741 include!("scope_handler_delta_tests.rs");
12742 }
12743
12744 use subc_protocol::scope::{
12745 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
12746 ScopeStatus,
12747 };
12748
12749 use super::*;
12750
12751 const OWNER: &str = "prefrontal-core";
12752
12753 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
12754 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
12755 }
12756
12757 async fn call(
12758 handler: &ControlHandler,
12759 ctx: &RouteCtx,
12760 request: &ModuleControlRequestFromModule,
12761 ) -> Frame {
12762 let body = serde_json::to_vec(request).unwrap();
12763 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
12764 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
12765 assert_eq!(replies.len(), 1, "{replies:?}");
12766 replies.pop().unwrap()
12767 }
12768
12769 async fn sync(
12770 handler: &ControlHandler,
12771 ctx: &RouteCtx,
12772 generation: u64,
12773 scopes: Vec<ScopeRecord>,
12774 ) -> Result<ModuleControlResponseToModule, String> {
12775 let reply = call(
12776 handler,
12777 ctx,
12778 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
12779 )
12780 .await;
12781 match reply.header.ty {
12782 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
12783 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
12784 }
12785 }
12786
12787 async fn describe(
12788 handler: &ControlHandler,
12789 ctx: &RouteCtx,
12790 owner: &str,
12791 scope_ref: &str,
12792 ) -> ModuleControlResponseToModule {
12793 let reply = call(
12794 handler,
12795 ctx,
12796 &ModuleControlRequestFromModule::ScopeDescribe {
12797 owner: Principal::Reserved {
12798 module_id: owner.to_string(),
12799 },
12800 scope_ref: scope_ref.to_string(),
12801 },
12802 )
12803 .await;
12804 assert_eq!(
12805 reply.header.ty,
12806 FrameType::Response,
12807 "{:?}",
12808 parse_error(&reply)
12809 );
12810 serde_json::from_slice(&reply.body).unwrap()
12811 }
12812
12813 async fn module(
12815 handler: &ControlHandler,
12816 connection: u64,
12817 module_id: &str,
12818 nonce: Option<&str>,
12819 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
12820 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
12821 hello_via_sink(
12822 handler,
12823 &ctx,
12824 &mut rx,
12825 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
12826 )
12827 .await;
12828 (ctx, rx)
12829 }
12830
12831 #[tokio::test]
12834 async fn a_client_connection_cannot_sync_or_describe() {
12835 let handler = ControlHandler::new(Arc::new(Registry::default()));
12836 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
12837 for request in [
12838 ModuleControlRequestFromModule::ScopeSync {
12839 generation: 1,
12840 scopes: vec![head("s", 1)],
12841 },
12842 ModuleControlRequestFromModule::ScopeDescribe {
12843 owner: Principal::Direct,
12844 scope_ref: "s".to_string(),
12845 },
12846 ] {
12847 let reply = call(&handler, &ctx, &request).await;
12848 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
12849 }
12850 assert!(
12851 !handler
12852 .scopes
12853 .read()
12854 .unwrap()
12855 .describe(
12856 &Principal::Reserved {
12857 module_id: OWNER.to_string()
12858 },
12859 "s"
12860 )
12861 .owner_synced
12862 );
12863 }
12864
12865 #[tokio::test]
12868 async fn a_module_without_a_supervised_launch_cannot_sync() {
12869 let handler = ControlHandler::new(Arc::new(Registry::default()));
12870 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
12871 assert_eq!(
12872 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
12873 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12874 );
12875 }
12876
12877 #[tokio::test]
12878 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
12879 let supervisor = SupervisorHandle::new();
12880 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12881 let handler = ControlHandler::new(Arc::new(Registry::default()))
12882 .with_supervisor(supervisor.clone());
12883 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12884 sync(&handler, &incumbent, 1, vec![head("s", 1)])
12885 .await
12886 .expect("the current launch syncs");
12887
12888 supervisor.open_swap(OWNER, "n2".to_string());
12891 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
12892 assert_eq!(
12893 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12894 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12895 );
12896 sync(&handler, &incumbent, 2, vec![head("s", 1)])
12897 .await
12898 .expect("the serving owner syncs during the swap");
12899
12900 supervisor.close_swap(OWNER);
12903 assert_eq!(
12904 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
12905 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12906 );
12907 sync(&handler, &incumbent, 3, vec![head("s", 1)])
12908 .await
12909 .expect("the serving owner syncs after the rollback");
12910 handler.cleanup_connection(candidate.connection_id).unwrap();
12911
12912 supervisor.open_swap(OWNER, "n3".to_string());
12917 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
12918 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
12919 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
12920 .await
12921 .expect("the promoted launch takes authority");
12922 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
12923 panic!("unexpected reply {reply:?}");
12924 };
12925 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
12926 assert_eq!(
12927 sync(&handler, &incumbent, 4, Vec::new()).await,
12928 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
12929 );
12930 }
12931
12932 #[tokio::test]
12935 async fn closing_the_authority_connection_frees_sync_authority() {
12936 let supervisor = SupervisorHandle::new();
12937 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12938 let handler = ControlHandler::new(Arc::new(Registry::default()))
12939 .with_supervisor(supervisor.clone());
12940 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
12941 sync(&handler, &first, 10, vec![head("s", 1)])
12942 .await
12943 .unwrap();
12944 handler.cleanup_connection(first.connection_id).unwrap();
12945
12946 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
12947 sync(&handler, &second, 1, vec![head("s", 1)])
12948 .await
12949 .expect("the next connection takes the released authority");
12950 }
12951
12952 #[tokio::test]
12953 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
12954 let supervisor = SupervisorHandle::new();
12955 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
12956 let handler =
12957 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
12958 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
12959 sync(&handler, &first, 10, vec![head("s", 1)])
12960 .await
12961 .unwrap();
12962 handler
12963 .handle_control_frame(
12964 &first,
12965 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
12966 )
12967 .await
12968 .unwrap();
12969 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
12970 sync(&handler, &second, 1, vec![head("s", 1)])
12971 .await
12972 .expect("GOODBYE releases authority even if the old socket remains open");
12973 }
12974
12975 #[tokio::test]
12976 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
12977 let registry = Arc::new(Registry::default());
12978 let supervisor_handle = SupervisorHandle::new();
12979 let supervisor =
12980 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
12981 .with_handle(supervisor_handle.clone())
12982 .with_daemon_incarnation("incarnation-7".to_string());
12983 supervisor
12986 .supervise_configured(
12987 ModuleSpec {
12988 module_id: OWNER.to_string(),
12989 program: PathBuf::from("/nonexistent/prefrontal-core"),
12990 args: Vec::new(),
12991 env: Vec::new(),
12992 reserved: false,
12993 reserved_prefixes: Vec::new(),
12994 protocol: ModuleProtocol::Subc,
12995 overlap: Default::default(),
12996 },
12997 false,
12998 )
12999 .unwrap();
13000 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
13001 let handler =
13002 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13003 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
13004
13005 let ModuleControlResponseToModule::ScopeDescribe {
13007 status,
13008 daemon_incarnation,
13009 owner_synced,
13010 owner_configured,
13011 scope,
13012 ..
13013 } = describe(&handler, &reader, OWNER, "s").await
13014 else {
13015 panic!("not a describe reply");
13016 };
13017 assert_eq!(status, ScopeStatus::NotLive);
13018 assert_eq!(daemon_incarnation, "incarnation-7");
13019 assert!(!owner_synced);
13020 assert!(owner_configured);
13021 assert!(scope.is_none());
13022
13023 let ModuleControlResponseToModule::ScopeDescribe {
13026 status,
13027 owner_configured,
13028 ..
13029 } = describe(&handler, &reader, "ghost", "s").await
13030 else {
13031 panic!("not a describe reply");
13032 };
13033 assert_eq!(status, ScopeStatus::NotLive);
13034 assert!(!owner_configured);
13035
13036 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
13038 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
13039 let ModuleControlResponseToModule::ScopeDescribe {
13040 status,
13041 scope_epoch,
13042 owner_synced,
13043 scope,
13044 ..
13045 } = describe(&handler, &reader, OWNER, "s").await
13046 else {
13047 panic!("not a describe reply");
13048 };
13049 assert_eq!(status, ScopeStatus::Live);
13050 assert_eq!(scope_epoch, Some(4));
13051 assert!(owner_synced);
13052 let stamp = scope.expect("a live scope carries its stamp");
13053 assert!(
13054 stamp.owner_authorized,
13055 "prefrontal-core is the default authority"
13056 );
13057 assert_eq!(stamp.kind, ScopeKind::Head);
13058 }
13059
13060 #[tokio::test]
13061 async fn scope_authority_owners_decides_owner_authorized() {
13062 let supervisor = SupervisorHandle::new();
13063 supervisor.set_spawn_nonce("broca", "b1".to_string());
13064 let handler = ControlHandler::new(Arc::new(Registry::default()))
13065 .with_supervisor(supervisor)
13066 .with_scope_authority_owners(vec!["broca".to_string()]);
13067 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
13068 let mut gated = head("s", 1);
13069 gated.attributes.agent_id = Some("agent".to_string());
13070 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
13071 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
13072 describe(&handler, &broca, "broca", "s").await
13073 else {
13074 panic!("not a describe reply");
13075 };
13076 assert!(scope.unwrap().owner_authorized);
13077 }
13078
13079 #[tokio::test]
13083 async fn scope_ops_and_the_scopes_capability_are_advertised() {
13084 let handler = ControlHandler::new(Arc::new(Registry::default()));
13085 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
13086 let ack = hello_via_sink(
13087 &handler,
13088 &ctx,
13089 &mut rx,
13090 hello_frame("m", PROTOCOL_VERSION, 1),
13091 )
13092 .await;
13093 let ack = parse_ack(&ack);
13094 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
13095 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
13096 }
13097 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
13098
13099 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
13100 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13101 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13102 let reply = handler
13103 .handle_control_frame(&client, frame)
13104 .await
13105 .unwrap()
13106 .pop()
13107 .unwrap();
13108 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13109 serde_json::from_slice(&reply.body).unwrap()
13110 else {
13111 panic!("not a server.describe reply");
13112 };
13113 assert!(
13114 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
13115 "{capabilities:?}"
13116 );
13117 }
13118
13119 const PLEXUS: &str = "plexus";
13122 const OTHER: &str = "other";
13123 const AFT: &str = "aft";
13124 const BROCA: &str = "broca";
13125 const MAGIC: &str = "magic-context";
13126
13127 fn nonce(module_id: &str) -> String {
13128 format!("nonce-{module_id}")
13129 }
13130
13131 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
13132 let (tx, rx) = mpsc::channel(64);
13133 (
13134 RouteCtx {
13135 connection_id: ConnectionId::new(connection),
13136 egress: FrameSink::new(tx),
13137 },
13138 rx,
13139 )
13140 }
13141
13142 struct Rig {
13146 handler: ControlHandler,
13147 forwarding: Arc<ForwardingTable>,
13148 owner: RouteCtx,
13149 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13150 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
13151 generation: u64,
13152 next_connection: u64,
13153 _supervisor: Supervisor,
13154 }
13155
13156 async fn rig() -> Rig {
13157 rig_with_flow_support(true).await
13158 }
13159
13160 async fn rig_with_flow_support(flow_support: bool) -> Rig {
13161 rig_with_scope_capabilities(flow_support, false).await
13162 }
13163
13164 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
13165 let registry = Arc::new(Registry::default());
13166 let forwarding = Arc::new(ForwardingTable::default());
13167 let supervisor_handle = SupervisorHandle::new();
13168 let supervisor =
13169 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
13170 .with_handle(supervisor_handle.clone());
13171 supervisor
13172 .supervise_configured(
13173 ModuleSpec {
13174 module_id: OWNER.to_string(),
13175 program: PathBuf::from("/nonexistent/prefrontal-core"),
13176 args: Vec::new(),
13177 env: Vec::new(),
13178 reserved: false,
13179 reserved_prefixes: Vec::new(),
13180 protocol: ModuleProtocol::Subc,
13181 overlap: Default::default(),
13182 },
13183 false,
13184 )
13185 .unwrap();
13186 for module_id in [OWNER, AFT, BROCA, MAGIC] {
13187 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
13188 }
13189 let handler =
13190 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13191 .with_supervisor(supervisor_handle);
13192 let (owner, mut owner_rx) = wide_ctx(1);
13193 hello_via_sink(
13194 &handler,
13195 &owner,
13196 &mut owner_rx,
13197 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
13198 )
13199 .await;
13200 let mut modules = BTreeMap::new();
13201 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
13202 let (ctx, mut rx) = wide_ctx(connection);
13203 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
13204 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
13205 body["manifest"]["provenance"] =
13209 serde_json::json!({"wire_crate_version": "0.29.0"});
13210 if flow_support || run_support {
13211 let mut provides = Vec::new();
13212 if flow_support {
13213 provides.push("flow-scopes/v1");
13214 }
13215 if run_support {
13216 provides.push("agent-run-scopes/v1");
13217 }
13218 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
13219 }
13220 let hello = Frame::build(
13221 FrameType::Hello,
13222 control_flags(),
13223 0,
13224 0,
13225 connection,
13226 serde_json::to_vec(&body).unwrap(),
13227 )
13228 .unwrap();
13229 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
13230 modules.insert(module_id.to_string(), (ctx, rx));
13231 }
13232 Rig {
13233 handler,
13234 forwarding,
13235 owner,
13236 _owner_rx: owner_rx,
13237 modules,
13238 generation: 0,
13239 next_connection: 100,
13240 _supervisor: supervisor,
13241 }
13242 }
13243
13244 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
13245 ScopeCarrier::new(Principal::Reserved {
13246 module_id: module_id.to_string(),
13247 })
13248 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
13249 }
13250
13251 fn session(scope_epoch: u64) -> ScopeRecord {
13254 let mut record = head("s", scope_epoch);
13255 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
13256 record.attributes.agent_id = Some("agent-1".to_string());
13257 record.attributes.delegates = true;
13258 record
13259 }
13260
13261 impl Rig {
13262 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
13263 self.generation += 1;
13264 sync(&self.handler, &self.owner, self.generation, scopes)
13265 .await
13266 .expect("the owner's sync is accepted");
13267 }
13268
13269 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
13270 ScopeSelector {
13271 owner: Principal::Reserved {
13272 module_id: OWNER.to_string(),
13273 },
13274 scope_ref: scope_ref.to_string(),
13275 scope_epoch,
13276 }
13277 }
13278
13279 fn open_frame(
13280 &mut self,
13281 opener: Option<&str>,
13282 target: &str,
13283 scope: Option<ScopeSelector>,
13284 ) -> (
13285 RouteCtx,
13286 mpsc::Receiver<crate::router::OutboundFrame>,
13287 Frame,
13288 ) {
13289 self.next_connection += 1;
13290 let (ctx, rx) = wide_ctx(self.next_connection);
13291 let root = unique_project_root("scoped-open");
13292 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
13293 target: RouteTarget::ToolProvider {
13294 module_id: target.to_string(),
13295 },
13296 identity: BindIdentity::new(
13297 root.path().to_path_buf(),
13298 "unit".to_string(),
13299 "session".to_string(),
13300 ),
13301 consumer_identity: opener.map(|module_id| ConsumerIdentity {
13302 module_id: module_id.to_string(),
13303 launch_nonce: nonce(module_id),
13304 }),
13305 consumer_capabilities: None,
13306 role_versions: None,
13307 admission_facts: None,
13308 scope,
13309 })
13310 .unwrap();
13311 let frame = Frame::build(
13312 FrameType::Request,
13313 control_flags(),
13314 0,
13315 0,
13316 self.next_connection,
13317 body,
13318 )
13319 .unwrap();
13320 (ctx, rx, frame)
13321 }
13322
13323 async fn refused(
13325 &mut self,
13326 opener: Option<&str>,
13327 target: &str,
13328 scope: Option<ScopeSelector>,
13329 ) -> String {
13330 self.refusal_body(opener, target, scope).await["code"]
13331 .as_str()
13332 .unwrap()
13333 .to_string()
13334 }
13335
13336 async fn refusal_body(
13337 &mut self,
13338 opener: Option<&str>,
13339 target: &str,
13340 scope: Option<ScopeSelector>,
13341 ) -> Value {
13342 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
13343 let replies = tokio::time::timeout(
13344 Duration::from_secs(2),
13345 self.handler.handle_control_frame(&ctx, frame),
13346 )
13347 .await
13348 .expect("the open must be refused before waiting for a bind ack")
13349 .unwrap();
13350 assert_eq!(replies.len(), 1, "{replies:?}");
13351 assert_eq!(replies[0].header.ty, FrameType::Error);
13352 let (_, module_rx) = self.modules.get_mut(target).unwrap();
13353 assert!(
13354 module_rx.try_recv().is_err(),
13355 "a refused open relays nothing"
13356 );
13357 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
13358 parse_error(&replies[0])
13359 }
13360
13361 async fn relayed(
13363 &mut self,
13364 opener: Option<&str>,
13365 target: &str,
13366 scope: Option<ScopeSelector>,
13367 ) -> Relayed {
13368 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
13369 let handler = self.handler.clone();
13370 let task_ctx = ctx.clone();
13371 let task = tokio::spawn(async move {
13372 handler
13373 .handle_control_frame(&task_ctx, frame)
13374 .await
13375 .unwrap()
13376 });
13377 let (_, module_rx) = self.modules.get_mut(target).unwrap();
13378 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
13379 .await
13380 .expect("the target receives the relayed route.bind")
13381 .unwrap()
13382 .frame;
13383 Relayed {
13384 target: target.to_string(),
13385 client: ctx,
13386 client_rx: rx,
13387 task,
13388 bind,
13389 }
13390 }
13391
13392 async fn ack(&self, relayed: &Relayed) {
13393 let (module, _) = &self.modules[&relayed.target];
13394 self.handler
13395 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
13396 .await
13397 .unwrap();
13398 }
13399
13400 async fn bound(
13402 &mut self,
13403 opener: Option<&str>,
13404 target: &str,
13405 scope: Option<ScopeSelector>,
13406 ) -> Bound {
13407 let relayed = self.relayed(opener, target, scope).await;
13408 self.ack(&relayed).await;
13409 let Relayed {
13410 target,
13411 client,
13412 mut client_rx,
13413 task,
13414 bind,
13415 } = relayed;
13416 assert!(
13417 task.await.unwrap().is_empty(),
13418 "the open is answered by commit"
13419 );
13420 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
13421 Bound {
13422 target,
13423 client,
13424 client_rx,
13425 channel,
13426 epoch,
13427 bind,
13428 }
13429 }
13430
13431 fn live(&self, route: &Bound) -> bool {
13432 matches!(
13433 self.forwarding
13434 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
13435 .unwrap(),
13436 DataRoute::Client(DataRouteState::Bound(_))
13437 )
13438 }
13439 }
13440
13441 struct Relayed {
13442 target: String,
13443 client: RouteCtx,
13444 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13445 task: tokio::task::JoinHandle<Vec<Frame>>,
13446 bind: Frame,
13447 }
13448
13449 struct Bound {
13450 target: String,
13451 client: RouteCtx,
13452 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13453 channel: u16,
13454 epoch: u32,
13455 bind: Frame,
13456 }
13457
13458 impl Bound {
13459 fn closed_reason(&mut self) -> RouteCloseReason {
13462 let mut reason = None;
13463 let mut goodbye = false;
13464 while let Ok(outbound) = self.client_rx.try_recv() {
13465 let frame = outbound.frame;
13466 match frame.header.ty {
13467 FrameType::Goodbye => {
13468 assert_eq!(
13469 (frame.header.channel, frame.header.epoch),
13470 (self.channel, self.epoch)
13471 );
13472 goodbye = true;
13473 }
13474 FrameType::Push => {
13475 let ClientControlPush::RouteClosed {
13476 reason: r,
13477 module_id,
13478 ..
13479 } = serde_json::from_slice(&frame.body).unwrap()
13480 else {
13481 panic!("unexpected push");
13482 };
13483 assert_eq!(module_id, self.target);
13484 reason = Some(r);
13485 }
13486 other => panic!("unexpected frame {other:?}"),
13487 }
13488 }
13489 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
13490 reason.expect("the client is told why the route closed")
13491 }
13492
13493 fn untouched(&mut self) -> bool {
13494 self.client_rx.try_recv().is_err()
13495 }
13496
13497 fn stamp(&self) -> Option<ScopeStamp> {
13498 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
13499 ModuleControlRequest::RouteBind { scope, .. } => scope,
13500 other => panic!("expected a route.bind, got {other:?}"),
13501 }
13502 }
13503 }
13504
13505 #[tokio::test]
13506 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
13507 ) {
13508 let mut rig = rig().await;
13509 let mut record = session(1);
13510 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
13511 record.child_owners = vec![Principal::Reserved {
13512 module_id: MAGIC.to_string(),
13513 }];
13514 rig.sync(vec![record]).await;
13515 let scope = || Some(rig_selector("s", Some(1)));
13516
13517 rig.bound(Some(OWNER), PLEXUS, scope()).await;
13520 rig.bound(Some(AFT), OTHER, scope()).await;
13521 rig.bound(Some(BROCA), PLEXUS, scope()).await;
13522
13523 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
13527 assert_eq!(
13528 rig.refused(opener, target, scope()).await,
13529 error_codes::SCOPE_NOT_CARRIER,
13530 "{opener:?} -> {target}"
13531 );
13532 }
13533 }
13534
13535 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
13536 ScopeSelector {
13537 owner: Principal::Reserved {
13538 module_id: OWNER.to_string(),
13539 },
13540 scope_ref: scope_ref.to_string(),
13541 scope_epoch,
13542 }
13543 }
13544
13545 #[tokio::test]
13546 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
13547 let mut rig = rig().await;
13548 rig.sync(vec![session(1)]).await;
13549 for opener in [OWNER, AFT] {
13550 assert_eq!(
13551 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
13552 .await,
13553 error_codes::SCOPE_EPOCH_REQUIRED,
13554 "{opener}"
13555 );
13556 }
13557 }
13558
13559 #[tokio::test]
13560 async fn admission_separates_not_synced_not_live_and_ended() {
13561 let mut rig = rig().await;
13562 let code = rig
13564 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13565 .await;
13566 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
13567 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
13568
13569 let ghost = ScopeSelector {
13571 owner: Principal::Reserved {
13572 module_id: "ghost".to_string(),
13573 },
13574 scope_ref: "s".to_string(),
13575 scope_epoch: Some(1),
13576 };
13577 assert_eq!(
13578 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
13579 error_codes::SCOPE_NOT_LIVE
13580 );
13581
13582 rig.sync(vec![session(2)]).await;
13583 assert_eq!(
13584 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
13585 .await,
13586 error_codes::SCOPE_NOT_LIVE
13587 );
13588 for epoch in [1, 3] {
13589 assert_eq!(
13590 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
13591 .await,
13592 error_codes::SCOPE_ENDED,
13593 "epoch {epoch}"
13594 );
13595 }
13596 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
13598 .await;
13599 }
13600
13601 #[tokio::test]
13602 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
13603 let mut rig = rig().await;
13604 rig.sync(vec![session(1)]).await;
13605 let route = rig
13606 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13607 .await;
13608 let stamp = route.stamp().expect("a scoped bind carries the stamp");
13609 assert_eq!(stamp.scope_ref, "s");
13610 assert_eq!(stamp.scope_epoch, 1);
13611 assert_eq!(stamp.kind, ScopeKind::Head);
13612 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
13613 assert!(stamp.attributes.delegates);
13614 assert!(stamp.owner_authorized);
13615 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13616 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
13617
13618 let (broca, mut broca_rx) = wide_ctx(50);
13621 hello_via_sink(
13622 &rig.handler,
13623 &broca,
13624 &mut broca_rx,
13625 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
13626 )
13627 .await;
13628 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
13629 .await
13630 .unwrap();
13631 let own = ScopeSelector {
13632 owner: Principal::Reserved {
13633 module_id: BROCA.to_string(),
13634 },
13635 scope_ref: "b".to_string(),
13636 scope_epoch: Some(1),
13637 };
13638 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
13639 assert!(!route.stamp().unwrap().owner_authorized);
13640 }
13641
13642 #[tokio::test]
13643 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
13644 let mut rig = rig().await;
13645 let mut record = head("s", 1);
13646 record.carriers = vec![carrier(AFT, None)];
13647 let flow_id = "Flow:run-7/step_2!~";
13648 record.attributes.flow_id = Some(flow_id.to_string());
13649 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
13650 .await
13651 .unwrap();
13652 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
13653 panic!("not a sync reply");
13654 };
13655 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
13656 let route = rig
13657 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13658 .await;
13659 assert!(rig.live(&route), "the stamped bind committed");
13660 let stamp = route.stamp().expect("a flow scope carries a stamp");
13661 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
13662 assert_eq!(stamp.attributes.agent_id, None);
13663 assert!(!stamp.attributes.delegates);
13664 assert!(stamp.owner_authorized);
13665 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13666 assert_eq!(unscoped.stamp(), None);
13667 }
13668
13669 #[tokio::test]
13670 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
13671 let mut rig = rig_with_flow_support(false).await;
13672 let mut record = session(1);
13673 record.attributes.flow_id = Some("flow:7".to_string());
13674 rig.sync(vec![record]).await;
13675 for opener in [OWNER, AFT] {
13676 let body = rig
13677 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13678 .await;
13679 assert_eq!(body["code"], "target_flow_unsupported");
13680 let message = body["message"].as_str().unwrap();
13681 for required in [PLEXUS, "flow-scopes/v1"] {
13682 assert!(message.contains(required), "{message}");
13683 }
13684 }
13685 }
13686
13687 #[tokio::test]
13688 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
13689 let mut rig = rig_with_flow_support(true).await;
13690 let mut record = session(1);
13691 record.attributes.flow_id = Some("flow:7".to_string());
13692 rig.sync(vec![record]).await;
13693 for opener in [OWNER, AFT] {
13694 let route = rig
13695 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13696 .await;
13697 assert!(rig.live(&route));
13698 assert_eq!(
13699 route.stamp().unwrap().attributes.flow_id.as_deref(),
13700 Some("flow:7")
13701 );
13702 }
13703 }
13704
13705 #[tokio::test]
13706 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
13707 use std::future::Future;
13708
13709 let mut rig = rig_with_flow_support(true).await;
13710 let mut record = session(1);
13711 record.attributes.flow_id = Some("flow:7".to_string());
13712 rig.sync(vec![record]).await;
13713 let (client, mut client_rx, frame) =
13714 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
13715 for _ in 0..64 {
13718 client.egress.try_send(route_bind_ack(1)).unwrap();
13719 }
13720 let handler = rig.handler.clone();
13721 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
13722 std::future::poll_fn(|cx| {
13723 assert!(open.as_mut().poll(cx).is_pending());
13724 std::task::Poll::Ready(())
13725 })
13726 .await;
13727 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13728
13729 let old_connection = rig.modules[PLEXUS].0.connection_id;
13730 rig.handler.cleanup_connection(old_connection).unwrap();
13731 let (replacement, mut replacement_rx) = wide_ctx(200);
13732 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
13733 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
13734 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
13735 let hello = Frame::build(
13736 FrameType::Hello,
13737 control_flags(),
13738 0,
13739 0,
13740 200,
13741 serde_json::to_vec(&body).unwrap(),
13742 )
13743 .unwrap();
13744 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
13745
13746 client_rx.try_recv().unwrap();
13747 let replies = tokio::time::timeout(Duration::from_secs(2), open)
13748 .await
13749 .expect("the replacement is refused without waiting for a bind ack")
13750 .unwrap();
13751 assert_eq!(replies.len(), 1);
13752 let body = parse_error(&replies[0]);
13753 assert_eq!(body["code"], "target_flow_unsupported");
13754 for required in [PLEXUS, "flow-scopes/v1"] {
13755 assert!(body["message"].as_str().unwrap().contains(required));
13756 }
13757 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
13758 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
13759 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13760 assert!(rig
13761 .handler
13762 .registry
13763 .get_module_by_connection(replacement.connection_id)
13764 .unwrap()
13765 .is_some());
13766 }
13767
13768 #[tokio::test]
13769 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
13770 {
13771 let mut rig = rig_with_flow_support(false).await;
13772 rig.sync(vec![session(1)]).await;
13773 let route = rig
13774 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13775 .await;
13776 assert!(rig.live(&route));
13777 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
13778 let mut record = session(1);
13779 record.attributes.flow_id = Some("flow:7".to_string());
13780 rig.sync(vec![record]).await;
13781 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
13782 assert!(rig.live(&unscoped));
13783 assert_eq!(unscoped.stamp(), None);
13784 }
13785
13786 #[tokio::test]
13787 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
13788 let mut rig = rig().await;
13789 let mut record = session(1);
13790 record.attributes.flow_id = Some("flow:7".to_string());
13791 rig.sync(vec![record.clone()]).await;
13792 let before = rig.forwarding.published_scope_tag(OWNER, "s");
13793 let mut owner_route = rig
13794 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
13795 .await;
13796 let mut carrier_route = rig
13797 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
13798 .await;
13799 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13800 rig.sync(vec![record.clone()]).await;
13801 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
13802 assert!(rig.live(&owner_route) && owner_route.untouched());
13803 assert!(rig.live(&carrier_route) && carrier_route.untouched());
13804
13805 record.attributes.flow_id = Some("flow:8".to_string());
13806 rig.sync(vec![record]).await;
13807 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
13808 let before = before.unwrap();
13809 assert_eq!(after.scope_epoch, before.scope_epoch);
13810 assert!(after.version > before.version);
13811 for route in [&mut owner_route, &mut carrier_route] {
13812 assert!(!rig.live(route));
13813 assert_eq!(
13814 route.closed_reason(),
13815 RouteCloseReason::ScopeDelegationChanged
13816 );
13817 }
13818 assert!(rig.live(&unscoped) && unscoped.untouched());
13819 for target in [PLEXUS, OTHER] {
13822 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
13823 let goodbye = module_rx
13824 .try_recv()
13825 .expect("the provider sees the drain")
13826 .frame;
13827 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13828 assert!(module_rx.try_recv().is_err());
13829 }
13830 let rebound = rig
13831 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13832 .await;
13833 assert_eq!(
13834 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
13835 Some("flow:8")
13836 );
13837 }
13838
13839 #[tokio::test]
13844 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
13845 let mut rig = rig().await;
13846 rig.sync(vec![session(1)]).await;
13847 let mut cotenant = rig
13848 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
13849 .await;
13850
13851 let mut changed = session(1);
13852 changed.child_owners.push(Principal::Reserved {
13853 module_id: MAGIC.to_string(),
13854 });
13855 let mut ended = None;
13856 for (code, next) in [
13857 (error_codes::SCOPE_CHANGED, vec![changed]),
13858 (error_codes::SCOPE_ENDED, Vec::new()),
13859 ] {
13860 let relayed = rig
13861 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13862 .await;
13863 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
13864 ended = Some(next.is_empty());
13865 rig.sync(next).await;
13866 rig.ack(&relayed).await;
13867 let replies = relayed.task.await.unwrap();
13868 assert_eq!(replies.len(), 1, "{replies:?}");
13869 assert_eq!(parse_error(&replies[0])["code"], code);
13870 assert_eq!(
13871 subc_protocol::error_codes::is_retryable_route_open(code),
13872 code == error_codes::SCOPE_CHANGED
13873 );
13874 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13875 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
13877 let mut goodbyes = Vec::new();
13880 while let Ok(outbound) = plexus_rx.try_recv() {
13881 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
13882 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
13883 }
13884 assert!(
13885 goodbyes.contains(&(bind_channel, bind_epoch)),
13886 "{goodbyes:?}"
13887 );
13888 assert!(rig
13889 .handler
13890 .registry
13891 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
13892 .unwrap()
13893 .is_some());
13894 }
13895 assert_eq!(ended, Some(true));
13896 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
13899 }
13900
13901 #[tokio::test]
13904 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
13905 struct Case {
13906 name: &'static str,
13907 change: fn(&mut ScopeRecord),
13908 closed: [Option<RouteCloseReason>; 4],
13911 }
13912 use RouteCloseReason::*;
13913 let cases = [
13914 Case {
13915 name: "a carrier entry removed",
13916 change: |r| {
13917 r.carriers.retain(|c| {
13918 c.principal
13919 != Principal::Reserved {
13920 module_id: AFT.to_string(),
13921 }
13922 })
13923 },
13924 closed: [None, Some(ScopeCarrierRemoved), None, None],
13925 },
13926 Case {
13927 name: "a target removed from a carrier",
13928 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
13929 closed: [None, None, None, Some(ScopeCarrierRemoved)],
13930 },
13931 Case {
13932 name: "a bare carrier narrowed to targets",
13933 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
13934 closed: [None, Some(ScopeCarrierRemoved), None, None],
13935 },
13936 Case {
13937 name: "delegates turned off",
13938 change: |r| r.attributes.delegates = false,
13939 closed: [Some(ScopeDelegationChanged); 4],
13940 },
13941 Case {
13942 name: "agent_id changed",
13943 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
13944 closed: [Some(ScopeDelegationChanged); 4],
13945 },
13946 Case {
13947 name: "a carrier added, child owners changed, the record re-sent",
13948 change: |r| {
13949 r.carriers.push(carrier(MAGIC, None));
13950 r.child_owners.push(Principal::Reserved {
13951 module_id: MAGIC.to_string(),
13952 });
13953 },
13954 closed: [None; 4],
13955 },
13956 Case {
13957 name: "a target added",
13958 change: |r| {
13959 r.carriers[1]
13960 .targets
13961 .as_mut()
13962 .unwrap()
13963 .push("third".to_string())
13964 },
13965 closed: [None; 4],
13966 },
13967 Case {
13968 name: "delegates turned on",
13969 change: |r| r.attributes.delegates = true,
13970 closed: [None; 4],
13971 },
13972 ];
13973 for case in cases {
13974 let mut rig = rig().await;
13975 rig.sync(vec![session(1)]).await;
13976 let scope = || Some(rig_selector("s", Some(1)));
13977 let mut routes = [
13978 rig.bound(Some(OWNER), PLEXUS, scope()).await,
13979 rig.bound(Some(AFT), PLEXUS, scope()).await,
13980 rig.bound(Some(BROCA), PLEXUS, scope()).await,
13981 rig.bound(Some(BROCA), OTHER, scope()).await,
13982 ];
13983 let mut record = session(1);
13984 (case.change)(&mut record);
13985 rig.sync(vec![record]).await;
13986 for (index, expected) in case.closed.iter().enumerate() {
13987 let route = &mut routes[index];
13988 match expected {
13989 Some(reason) => {
13990 assert!(!rig.live(route), "{}: route {index} still live", case.name);
13991 assert_eq!(
13992 route.closed_reason(),
13993 *reason,
13994 "{}: route {index}",
13995 case.name
13996 );
13997 }
13998 None => {
13999 assert!(rig.live(route), "{}: route {index} closed", case.name);
14000 assert!(
14001 route.untouched(),
14002 "{}: route {index} was told something",
14003 case.name
14004 );
14005 }
14006 }
14007 }
14008 }
14009 }
14010
14011 #[tokio::test]
14012 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
14013 for next in [Vec::new(), vec![session(2)]] {
14015 let mut rig = rig().await;
14016 rig.sync(vec![session(1)]).await;
14017 let mut route = rig
14018 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14019 .await;
14020 rig.sync(next).await;
14021 assert!(!rig.live(&route));
14022 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
14023 }
14024
14025 let mut rig = rig().await;
14028 let mut child = session(1);
14029 child.scope_ref = "child".to_string();
14030 child.kind = ScopeKind::Worker;
14031 child.parent = Some(ScopeParent::new(
14032 Principal::Reserved {
14033 module_id: OWNER.to_string(),
14034 },
14035 "s".to_string(),
14036 1,
14037 ));
14038 rig.sync(vec![session(1), child.clone()]).await;
14039 let mut child_route = rig
14040 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
14041 .await;
14042 assert_eq!(
14043 child_route.stamp().unwrap().parent_state,
14044 Some(ParentState::Linked)
14045 );
14046 rig.sync(vec![child]).await;
14047 assert!(!rig.live(&child_route));
14048 assert_eq!(
14049 child_route.closed_reason(),
14050 RouteCloseReason::ScopeParentEnded
14051 );
14052 }
14053
14054 #[tokio::test]
14055 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
14056 ) {
14057 let mut rig = rig().await;
14058 rig.sync(vec![session(1)]).await;
14059 let mut route = rig
14060 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14061 .await;
14062 let before = rig.forwarding.published_scope_tag(OWNER, "s");
14063 rig.sync(vec![session(1)]).await;
14064 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
14065 assert!(rig.live(&route) && route.untouched());
14066
14067 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
14071 .forwarding
14072 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
14073 .unwrap()
14074 else {
14075 panic!("the route is bound");
14076 };
14077 binding.flow.acquire_tagged(9, false).await.unwrap();
14078 let mut widened = session(1);
14079 widened.carriers.push(carrier(MAGIC, None));
14080 rig.sync(vec![widened]).await;
14081 assert!(rig.live(&route) && route.untouched());
14082 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14083 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
14084 assert_eq!(binding.flow.in_flight(), 1);
14087 binding
14088 .flow
14089 .acquire_tagged(10, false)
14090 .await
14091 .expect("the flow is still open");
14092 }
14093
14094 #[tokio::test]
14097 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
14098 let mut rig = rig().await;
14099 rig.sync(vec![session(1)]).await;
14100 let mut on_incumbent = rig
14101 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14102 .await;
14103
14104 let (candidate, _candidate_rx) = wide_ctx(9);
14107 let registration = rig
14108 .handler
14109 .registry
14110 .register_candidate_with_control_ops(
14111 manifest(PLEXUS, PROTOCOL_VERSION),
14112 PROTOCOL_VERSION,
14113 candidate.connection_id,
14114 module_baseline_control_ops(),
14115 )
14116 .unwrap();
14117 rig.forwarding
14118 .register_candidate_module_connection(
14119 candidate.connection_id,
14120 PLEXUS.to_string(),
14121 PROTOCOL_VERSION,
14122 manifest_concurrency(®istration.manifest),
14123 candidate.egress.clone(),
14124 )
14125 .unwrap();
14126 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
14127 rig.handler
14128 .registry
14129 .promote_candidate(PLEXUS)
14130 .unwrap()
14131 .unwrap();
14132 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
14133
14134 rig.sync(Vec::new()).await;
14135 assert!(!rig.live(&on_incumbent));
14136 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
14137 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14138 let goodbye = incumbent_rx
14139 .try_recv()
14140 .expect("the superseded endpoint is told")
14141 .frame;
14142 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14143 }
14144 }
14145}
14146
14147#[cfg(test)]
14148mod concurrency_default_exposure_tests {
14149 use super::*;
14150
14151 fn hello_body(role_json: &str) -> Vec<u8> {
14152 format!(
14153 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
14154 )
14155 .into_bytes()
14156 }
14157
14158 fn manifest_from(body: &[u8]) -> ModuleManifest {
14159 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
14160 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
14161 .expect("manifest parses")
14162 }
14163
14164 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
14165
14166 #[test]
14167 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
14168 let body = hello_body(&format!(
14169 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
14170 ));
14171 let manifest = manifest_from(&body);
14172 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
14175 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
14176 }
14177
14178 #[test]
14179 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
14180 let body = hello_body(&format!(
14181 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
14182 ));
14183 let manifest = manifest_from(&body);
14184 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
14185 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
14186 }
14187
14188 #[test]
14189 fn non_management_roles_are_never_reported() {
14190 let body = hello_body(
14191 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
14192 );
14193 let manifest = manifest_from(&body);
14194 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
14195 }
14196}