1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_event_declarations,
23 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
24 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
25 },
26 scope::{
27 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
29 SCOPE_SYNC_OP,
30 },
31 session::{
32 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
33 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
34 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
35 ROLE_VERSIONS_FIELD,
36 },
37 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
38 Priority, RouteTarget, PROTOCOL_VERSION,
39};
40use tokio::time::{timeout_at, Instant};
41use tracing::{debug, info, warn};
42
43use crate::{
44 capability_requirements::{
45 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
46 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
47 RuntimeModule,
48 },
49 daemon_config::RestartRequiredSection,
50 forwarding::{
51 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
52 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
53 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
54 },
55 observability::{
56 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
57 },
58 provenance::{
59 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
60 },
61 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
62 router::{RouteCtx, RouterError},
63 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
64 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
65 stderr_tail::{CaptureState, TailEntry},
66 supervise::{
67 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
68 SupervisorHandle, SwapHelloAdmission,
69 },
70 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
71};
72
73pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
79
80const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
81const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
82const CAP_PING_PONG: &str = "ping_pong_v1";
83const CAP_SESSION_ATTACH: &str = "session_attach_v1";
84const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
85
86const SUBC_CONTROL_OPS: &[&str] = &[
87 ops::SERVER_DESCRIBE,
88 ops::CATALOG_LIST,
89 ops::ROUTE_OPEN,
90 ops::ROUTE_POLL,
91 ops::ROUTE_CLOSING,
92 ops::ROUTE_CLOSED,
93 ops::SUPERVISOR_LIST,
94 ops::SUPERVISOR_RESTART,
95 ops::SUPERVISOR_SWAP,
96 ops::SUPERVISOR_RELOAD,
97 ops::SUPERVISOR_RESCAN,
98 ops::SUPERVISOR_RELEASE_RESERVED,
99 ops::SUPERVISOR_SET_ENABLED,
100 ops::SUPERVISOR_HEALTH_PROBE,
101 ops::SUPERVISOR_HEALTH,
102 ops::SUPERVISOR_STDERR_TAIL,
103 ops::SUPERVISOR_TERMINALS,
104 ops::SUPERVISOR_ROUTES,
105 ops::SUPERVISOR_PROVENANCE,
106 ops::SUPERVISOR_SPAWN_SNAPSHOT,
107 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
108];
109
110const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
111 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
112 "supervisor.live_roots",
113 SCOPE_SYNC_OP,
114 SCOPE_APPLY_OP,
115 SCOPE_DESCRIBE_OP,
116 "operator.confirm",
117];
118
119const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
123
124const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
125
126pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
134
135pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
150
151pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
169
170const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
171const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
172
173fn reload_verdict(
174 configured: &Path,
175 spawned_from: Option<&Path>,
176 image: subc_control::RunningImageAgreement,
177) -> PendingReloadVerdict {
178 let path = match spawned_from {
179 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
180 Some(spawned_from) => ReloadPathAgreement::Mismatch {
181 configured: configured.to_path_buf(),
182 spawned_from: spawned_from.to_path_buf(),
183 },
184 None => ReloadPathAgreement::Unavailable {
185 reason: if matches!(
186 image,
187 subc_control::RunningImageAgreement::Unavailable {
188 reason: subc_control::RunningImageUnavailableReason::NotRunning
189 }
190 ) {
191 ReloadPathUnavailableReason::NotRunning
192 } else {
193 ReloadPathUnavailableReason::SpawnedPathUnavailable
194 },
195 },
196 };
197 PendingReloadVerdict { path, image }
198}
199
200#[derive(Clone)]
201struct DaemonProvenanceFacts {
202 build: DaemonBuildProvenance,
203 pid: Option<u32>,
204 started_at_ms: Option<u64>,
205 start_clock: Option<crate::clock::StartClock>,
206 executable_path: Option<PathBuf>,
207 executable_identity: Option<SpawnedFileIdentity>,
208 process_start_time: Option<u64>,
209 probe: ExecutableIdentityProbe,
210}
211
212impl Default for DaemonProvenanceFacts {
213 fn default() -> Self {
214 Self {
215 build: DaemonBuildProvenance {
216 build_git_sha: None,
217 build_lock_digest: None,
218 },
219 pid: None,
220 started_at_ms: None,
221 start_clock: None,
222 executable_path: None,
223 executable_identity: None,
224 process_start_time: None,
225 probe: ExecutableIdentityProbe::default(),
226 }
227 }
228}
229
230#[derive(Debug, Clone)]
231struct SupervisorRescanContext {
232 supervisor: Supervisor,
233 config_path: PathBuf,
234 configured_port: Option<u16>,
235 storage_config: Option<crate::daemon_config::StorageConfig>,
236 admission_facts_carrier_module_id: Option<String>,
237 admission_facts_targets: Option<Vec<String>>,
238 scope_authority_owners: Vec<String>,
239}
240
241const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
262 "reloading",
263 "supervisor_not_live",
264 "registration_not_active",
265 "no_forwarding_connection",
266 "relay_send_failed",
267];
268
269#[derive(Clone)]
271pub struct ControlHandler {
272 registry: Arc<Registry>,
273 forwarding: Arc<ForwardingTable>,
274 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
275 supervisor: SupervisorHandle,
276 subc_capabilities: Arc<[String]>,
277 route_bind_relay_timeout: Duration,
281 route_bind_relay_timeouts: BTreeMap<String, Duration>,
285 route_bind_breakers: RouteBindBreakers,
288 route_bind_concurrency: RouteBindConcurrency,
291 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
296 route_bind_breaker_threshold: u32,
298 route_bind_breaker_cooldown: Duration,
300 health_probe_timeout: Duration,
301 storage_config: Option<crate::daemon_config::StorageConfig>,
304 machine_id: Option<crate::machine_id::MachineId>,
308 admission_facts_carrier_module_id: Option<String>,
309 admission_facts_targets: Option<Vec<String>>,
310 scopes: Arc<RwLock<ScopeTable>>,
314 scope_deadline_added: Arc<tokio::sync::Notify>,
318 scope_authority_owners: Vec<String>,
321 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
324 rescan: Option<SupervisorRescanContext>,
325 connected_clients: ConnectedClients,
326 counters: DaemonCounters,
327 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
328 daemon_provenance: DaemonProvenanceFacts,
329 #[cfg(test)]
330 control_dispatch_delay: Option<Duration>,
331 #[cfg(test)]
332 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
333 #[cfg(test)]
337 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
338}
339
340impl fmt::Debug for ControlHandler {
341 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
342 f.debug_struct("ControlHandler")
343 .field("registry", &self.registry)
344 .field("forwarding", &self.forwarding)
345 .field("process_liveness", &self.process_liveness.is_some())
346 .field("supervisor", &self.supervisor)
347 .field("subc_capabilities", &self.subc_capabilities)
348 .finish()
349 }
350}
351
352struct RouteOpenRequest {
353 target: RouteTarget,
354 identity: BindIdentity,
355 consumer_identity: Option<ConsumerIdentity>,
356 consumer_capabilities: Option<Vec<String>>,
357 role_versions: Option<BTreeMap<String, String>>,
358 admission_facts: Option<serde_json::Value>,
359 scope: Option<ScopeSelector>,
360}
361
362struct RouteBindReservationGuard {
363 forwarding: Arc<ForwardingTable>,
364 endpoint: ModuleEndpointId,
365 relay_corr: u64,
366 armed: bool,
367}
368
369struct ModuleControlRpcGuard {
370 forwarding: Arc<ForwardingTable>,
371 endpoint: ModuleEndpointId,
372 corr: u64,
373 armed: bool,
374}
375
376impl ModuleControlRpcGuard {
377 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
378 Self {
379 forwarding,
380 endpoint,
381 corr,
382 armed: true,
383 }
384 }
385
386 fn disarm(&mut self) {
387 self.armed = false;
388 }
389}
390
391impl Drop for ModuleControlRpcGuard {
392 fn drop(&mut self) {
393 if self.armed {
394 let _ = self
395 .forwarding
396 .cancel_module_control_rpc(self.endpoint, self.corr);
397 }
398 }
399}
400
401impl RouteBindReservationGuard {
402 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
403 Self {
404 forwarding,
405 endpoint,
406 relay_corr,
407 armed: true,
408 }
409 }
410
411 fn release_and_disarm(&mut self) {
412 if !self.armed {
413 return;
414 }
415 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
416 self.endpoint,
417 self.relay_corr,
418 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
419 ) {
420 send_goodbye_target_best_effort(
421 &self.forwarding.counters(),
422 &target,
423 "canceled route.bind",
424 );
425 }
426 self.armed = false;
427 }
428
429 fn disarm(&mut self) {
430 self.armed = false;
431 }
432}
433
434impl Drop for RouteBindReservationGuard {
435 fn drop(&mut self) {
436 self.release_and_disarm();
437 }
438}
439
440#[derive(Debug, Clone, Default)]
470pub(crate) struct RouteBindBreakers {
471 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
472}
473
474#[derive(Debug, Clone, Default)]
475pub(crate) struct RouteBindConcurrency {
476 modules: Arc<Mutex<HashMap<String, usize>>>,
477}
478
479struct RouteBindConcurrencyGuard {
480 concurrency: RouteBindConcurrency,
481 module_id: String,
482}
483
484impl RouteBindConcurrency {
485 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
488 let mut modules = self
489 .modules
490 .lock()
491 .expect("route.bind concurrency mutex poisoned");
492 let in_flight = modules.entry(module_id.to_string()).or_default();
493 if *in_flight >= limit {
494 return Err(*in_flight);
495 }
496 *in_flight += 1;
497 Ok(RouteBindConcurrencyGuard {
498 concurrency: self.clone(),
499 module_id: module_id.to_string(),
500 })
501 }
502}
503
504impl Drop for RouteBindConcurrencyGuard {
505 fn drop(&mut self) {
506 let mut modules = self
507 .concurrency
508 .modules
509 .lock()
510 .expect("route.bind concurrency mutex poisoned");
511 let remove = {
512 let in_flight = modules
513 .get_mut(&self.module_id)
514 .expect("admitted route.bind has a concurrency entry");
515 *in_flight -= 1;
516 *in_flight == 0
517 };
518 if remove {
519 modules.remove(&self.module_id);
520 }
521 }
522}
523
524#[derive(Debug, Default)]
525struct ModuleBreakerState {
526 consecutive_timeouts: u32,
528 cooldown_until: Option<Instant>,
531 probe_in_flight: Option<Arc<()>>,
536}
537
538enum RouteBindAdmission<'a> {
540 Admitted {
541 guard: RouteBindBreakerGuard<'a>,
542 probe: bool,
545 },
546 Refused {
547 consecutive_timeouts: u32,
548 retry_in: Duration,
552 probe_in_flight: bool,
553 },
554}
555
556struct RouteBindBreakerGuard<'a> {
563 breakers: RouteBindBreakers,
564 module_id: &'a str,
565 probe_token: Option<Arc<()>>,
566 settled: bool,
567}
568
569impl RouteBindBreakerGuard<'_> {
570 fn record_accepted(&mut self) -> bool {
574 self.settled = true;
575 self.breakers.record_accepted(self.module_id)
576 }
577
578 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
581 self.settled = true;
582 self.breakers.record_timeout(
583 self.module_id,
584 self.probe_token.as_ref(),
585 threshold,
586 cooldown,
587 )
588 }
589
590 fn record_inconclusive(&mut self) {
599 self.settled = true;
600 self.breakers
601 .record_inconclusive(self.module_id, self.probe_token.as_ref());
602 }
603}
604
605impl Drop for RouteBindBreakerGuard<'_> {
606 fn drop(&mut self) {
607 if !self.settled {
608 self.breakers
609 .record_inconclusive(self.module_id, self.probe_token.as_ref());
610 }
611 }
612}
613
614struct BreakerOpened {
618 consecutive_timeouts: u32,
619 reopened_after_probe: bool,
622}
623
624impl RouteBindBreakers {
625 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
626 self.modules
627 .lock()
628 .expect("route.bind breaker mutex poisoned")
629 }
630
631 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
634 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
635 probe: probe_token.is_some(),
636 guard: RouteBindBreakerGuard {
637 breakers: self.clone(),
638 module_id,
639 probe_token,
640 settled: false,
641 },
642 };
643
644 let mut modules = self.lock();
645 let Some(state) = modules.get_mut(module_id) else {
646 return admitted(None);
647 };
648 let Some(cooldown_until) = state.cooldown_until else {
649 return admitted(None);
650 };
651 if state.probe_in_flight.is_some() {
652 return RouteBindAdmission::Refused {
653 consecutive_timeouts: state.consecutive_timeouts,
654 retry_in: Duration::ZERO,
655 probe_in_flight: true,
656 };
657 }
658 let now = Instant::now();
659 if now < cooldown_until {
660 return RouteBindAdmission::Refused {
661 consecutive_timeouts: state.consecutive_timeouts,
662 retry_in: cooldown_until - now,
663 probe_in_flight: false,
664 };
665 }
666 let token = Arc::new(());
667 state.probe_in_flight = Some(Arc::clone(&token));
668 admitted(Some(token))
669 }
670
671 fn record_accepted(&self, module_id: &str) -> bool {
672 self.lock()
673 .remove(module_id)
674 .is_some_and(|state| state.cooldown_until.is_some())
675 }
676
677 fn record_timeout(
678 &self,
679 module_id: &str,
680 probe_token: Option<&Arc<()>>,
681 threshold: u32,
682 cooldown: Duration,
683 ) -> Option<BreakerOpened> {
684 let mut modules = self.lock();
685 let state = modules.entry(module_id.to_string()).or_default();
686 let was_open = state.cooldown_until.is_some();
687 let was_probe = Self::owns_probe(state, probe_token);
688 if was_probe {
689 state.probe_in_flight = None;
690 }
691 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
692 if state.consecutive_timeouts < threshold {
693 return None;
694 }
695 state.cooldown_until = Some(Instant::now() + cooldown);
696 Some(BreakerOpened {
697 consecutive_timeouts: state.consecutive_timeouts,
698 reopened_after_probe: was_open && was_probe,
699 })
700 }
701
702 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
703 state
707 .probe_in_flight
708 .as_ref()
709 .zip(token)
710 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
711 }
712
713 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
714 if let Some(state) = self.lock().get_mut(module_id) {
715 if Self::owns_probe(state, probe_token) {
716 state.probe_in_flight = None;
717 }
718 }
719 }
720
721 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
736 self.lock()
737 .remove(module_id)
738 .map(|state| state.consecutive_timeouts)
739 .filter(|discarded| *discarded > 0)
740 }
741
742 fn open_snapshot(&self) -> Option<serde_json::Value> {
749 let now = Instant::now();
750 let modules = self.lock();
751 let open = modules
752 .iter()
753 .filter_map(|(module_id, state)| {
754 let cooldown_until = state.cooldown_until?;
755 Some((
756 module_id.clone(),
757 serde_json::json!({
758 "consecutive_timeouts": state.consecutive_timeouts,
759 "cooldown_remaining_ms":
760 cooldown_until.saturating_duration_since(now).as_millis() as u64,
761 "probe_in_flight": state.probe_in_flight.is_some(),
762 }),
763 ))
764 })
765 .collect::<serde_json::Map<String, serde_json::Value>>();
766 (!open.is_empty()).then_some(serde_json::Value::Object(open))
767 }
768}
769
770impl ControlHandler {
771 pub fn new(registry: Arc<Registry>) -> Self {
772 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
773 }
774
775 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
776 let counters = forwarding.counters();
777 let route_bind_breakers = forwarding.route_bind_breakers();
781 let route_bind_concurrency = forwarding.route_bind_concurrency();
782 let route_outages = forwarding.route_outages();
783 Self {
784 registry,
785 forwarding,
786 process_liveness: None,
787 supervisor: SupervisorHandle::new(),
788 subc_capabilities: Arc::from([
789 CAP_MANIFEST_REGISTRATION.to_string(),
790 CAP_CHANNEL_LIFECYCLE.to_string(),
791 CAP_PING_PONG.to_string(),
792 CAP_SESSION_ATTACH.to_string(),
793 CAP_ADMISSION_FACTS_RELAY.to_string(),
794 CAP_SCOPES_V1.to_string(),
795 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
796 ]),
797 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
798 route_bind_relay_timeouts: BTreeMap::new(),
799 route_bind_breakers,
800 route_bind_concurrency,
801 route_outages,
802 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
803 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
804 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
805 storage_config: None,
806 machine_id: None,
807 admission_facts_carrier_module_id: None,
808 admission_facts_targets: None,
809 scopes: Arc::new(RwLock::new(ScopeTable::new(
810 crate::daemon_config::default_scope_authority_owners(),
811 ))),
812 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
813 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
814 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
815 rescan: None,
816 connected_clients: ConnectedClients::new(),
817 counters,
818 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
819 daemon_provenance: DaemonProvenanceFacts::default(),
820 #[cfg(test)]
821 control_dispatch_delay: None,
822 #[cfg(test)]
823 provenance_probe_override: None,
824 #[cfg(test)]
825 route_bind_relay_sent_hold: None,
826 }
827 }
828
829 pub fn with_storage_config(
832 mut self,
833 storage_config: Option<crate::daemon_config::StorageConfig>,
834 ) -> Self {
835 self.storage_config = storage_config;
836 self
837 }
838
839 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
842 self.machine_id = machine_id;
843 self
844 }
845
846 pub fn with_admission_facts_config(
850 mut self,
851 carrier_module_id: Option<String>,
852 targets: Option<Vec<String>>,
853 ) -> Self {
854 self.admission_facts_carrier_module_id = carrier_module_id;
855 self.admission_facts_targets = targets;
856 self
857 }
858
859 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
863 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
864 self.scope_authority_owners = owners;
865 self
866 }
867
868 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
871 self.route_bind_relay_timeout = timeout;
872 self
873 }
874
875 pub fn with_route_bind_relay_timeouts(
881 mut self,
882 timeouts: impl IntoIterator<Item = (String, Duration)>,
883 ) -> Self {
884 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
885 self
886 }
887
888 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
894 self.route_bind_relay_timeouts
895 .get(module_id)
896 .copied()
897 .unwrap_or(self.route_bind_relay_timeout)
898 }
899
900 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
908 self.route_bind_breaker_threshold = threshold.max(1);
909 self.route_bind_breaker_cooldown = cooldown;
910 self
911 }
912
913 #[cfg(test)]
914 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
915 self.health_probe_timeout = timeout;
916 self
917 }
918
919 #[cfg(test)]
920 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
921 self.control_dispatch_delay = Some(delay);
922 self
923 }
924
925 #[cfg(test)]
929 pub(crate) fn with_route_bind_relay_sent_hold(
930 mut self,
931 hold: Arc<tokio::sync::Barrier>,
932 ) -> Self {
933 self.route_bind_relay_sent_hold = Some(hold);
934 self
935 }
936
937 pub fn with_process_liveness(
938 mut self,
939 process_liveness: Arc<dyn ModuleProcessLiveness>,
940 ) -> Self {
941 self.process_liveness = Some(process_liveness);
942 self
943 }
944
945 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
946 self.supervisor = supervisor;
947 self
948 }
949
950 pub fn with_daemon_provenance(
951 mut self,
952 pid: u32,
953 started_at_ms: u64,
954 executable_path: Option<PathBuf>,
955 build_git_sha: Option<String>,
956 build_lock_digest: Option<String>,
957 ) -> Self {
958 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
959 let process_start_time = process_start_time(pid);
960 self.daemon_provenance = DaemonProvenanceFacts {
961 build: DaemonBuildProvenance {
962 build_git_sha,
963 build_lock_digest,
964 },
965 pid: Some(pid),
966 started_at_ms: Some(started_at_ms),
967 start_clock: None,
968 executable_path,
969 executable_identity,
970 process_start_time,
971 probe: ExecutableIdentityProbe::default(),
972 };
973 self
974 }
975
976 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
977 self.daemon_provenance.start_clock = Some(clock);
978 self
979 }
980
981 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
983 self.scopes
984 .write()
985 .unwrap_or_else(|poisoned| poisoned.into_inner())
986 .set_wall_clock(Arc::new(clock));
987 self
988 }
989
990 #[cfg(test)]
991 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
992 self.provenance_probe_override = Some(result);
993 self
994 }
995
996 pub fn with_capability_config(
1000 self,
1001 modules: impl IntoIterator<Item = (String, bool)>,
1002 reserved_capabilities: BTreeMap<String, String>,
1003 ) -> Self {
1004 self.capability_evaluator
1005 .configure(modules, reserved_capabilities);
1006 self
1007 }
1008
1009 pub fn with_supervisor_rescan(
1010 mut self,
1011 supervisor: Supervisor,
1012 config_path: impl Into<PathBuf>,
1013 configured_port: Option<u16>,
1014 ) -> Self {
1015 self.rescan = Some(SupervisorRescanContext {
1016 supervisor,
1017 config_path: config_path.into(),
1018 configured_port,
1019 storage_config: self.storage_config.clone(),
1020 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1021 admission_facts_targets: self.admission_facts_targets.clone(),
1022 scope_authority_owners: self.scope_authority_owners.clone(),
1023 });
1024 self
1025 }
1026
1027 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1028 self.connected_clients = connected_clients;
1029 self
1030 }
1031
1032 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1033 Arc::clone(&self.forwarding)
1034 }
1035
1036 pub(crate) fn counters(&self) -> DaemonCounters {
1037 self.counters.clone()
1038 }
1039
1040 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1043 tokio::spawn(async move {
1044 loop {
1045 self.capability_evaluator
1046 .wait_for_change_or_deadline()
1047 .await;
1048 self.refresh_capability_requirements();
1049 }
1050 });
1051 }
1052
1053 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1059 tokio::spawn(async move {
1060 loop {
1061 let has_deadlines = self
1062 .scopes
1063 .read()
1064 .unwrap_or_else(|poisoned| poisoned.into_inner())
1065 .has_deadlines();
1066 if !has_deadlines {
1067 self.scope_deadline_added.notified().await;
1068 continue;
1069 }
1070 tokio::time::sleep(Duration::from_secs(1)).await;
1071 if let Err(error) = self.sweep_expired_scopes() {
1072 warn!(%error, "scope expiry sweep failed");
1073 }
1074 }
1075 });
1076 }
1077
1078 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1079 let mut table = self
1080 .scopes
1081 .write()
1082 .unwrap_or_else(|poisoned| poisoned.into_inner());
1083 let swept = table.sweep_expired();
1084 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1085 drop(table);
1086 self.close_scope_drained_routes(drained);
1087 Ok(())
1088 }
1089
1090 fn publish_scope_changes(
1091 &self,
1092 changes: &[crate::scopes::ScopeTagChange],
1093 expired: &[crate::scopes::ScopeExpired],
1094 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1095 if changes.is_empty() {
1096 return Ok(Vec::new());
1097 }
1098 let drained = self
1099 .forwarding
1100 .publish_scope_changes(changes)
1101 .map_err(RouterError::Forwarding)?;
1102 let mut counts = HashMap::new();
1103 for route in &drained {
1104 *counts
1105 .entry((
1106 route.scope.owner.as_str(),
1107 route.scope.scope_ref.as_str(),
1108 route.scope.tag.scope_epoch,
1109 ))
1110 .or_insert(0usize) += 1;
1111 }
1112 for scope in expired {
1113 let routes_closed = counts
1114 .get(&(
1115 scope.owner.as_str(),
1116 scope.scope_ref.as_str(),
1117 scope.scope_epoch,
1118 ))
1119 .copied()
1120 .unwrap_or(0);
1121 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1122 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1123 routes_closed, "scope expired");
1124 }
1125 Ok(drained)
1126 }
1127
1128 fn runtime_capability_snapshot(
1129 &self,
1130 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1131 let runtime = self
1132 .supervisor
1133 .list()
1134 .into_iter()
1135 .map(|module| {
1136 let status = module.status().map_err(|err| {
1137 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1138 })?;
1139 Ok(RuntimeModule {
1140 module_id: status.module_id,
1141 state: status.state,
1142 enabled: status.enabled,
1143 })
1144 })
1145 .collect::<Result<Vec<_>, RouterError>>()?;
1146 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1147 RouterError::backend(
1148 0,
1149 0,
1150 format!("failed to list capability registrations: {err}"),
1151 )
1152 })?;
1153 let registrations = registrations
1154 .into_iter()
1155 .map(|registration| RegisteredModule {
1156 module_id: registration.manifest.module_id,
1157 module_version: registration.manifest.module_version,
1158 capabilities: registration.manifest.capabilities,
1159 })
1160 .collect();
1161 Ok((runtime, registrations))
1162 }
1163
1164 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1171 let cached_registration = RegisteredModule {
1172 module_id: registration.manifest.module_id.clone(),
1173 module_version: registration.manifest.module_version.clone(),
1174 capabilities: registration.manifest.capabilities.clone(),
1175 };
1176 if self.capability_evaluator.record_hello(&cached_registration) {
1177 warn!(
1178 module_id = %cached_registration.module_id,
1179 "capability claims drifted from the cached manifest"
1180 );
1181 }
1182 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1183 self.enforce_capability_denies();
1184 }
1185 self.refresh_capability_requirements();
1186 }
1187
1188 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1192 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1193 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1194 self.supervisor.set_swap_promotion_observer(observer);
1195 }
1196
1197 pub fn refresh_capability_requirements(&self) {
1198 match self.runtime_capability_snapshot() {
1199 Ok((runtime, registrations)) => {
1200 log_requirement_events(
1201 self.capability_evaluator
1202 .evaluate_now(&runtime, ®istrations),
1203 );
1204 }
1205 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1206 }
1207 }
1208
1209 fn enforce_capability_denies(&self) {
1213 let (_, registrations) = match self.registry.list_modules() {
1214 Ok(snapshot) => snapshot,
1215 Err(err) => {
1216 warn!(error = %err, "failed to read registrations for capability deny census");
1217 return;
1218 }
1219 };
1220 let manifests = registrations
1221 .into_iter()
1222 .map(|registration| {
1223 (
1224 registration.manifest.module_id.clone(),
1225 registration.manifest,
1226 )
1227 })
1228 .collect::<BTreeMap<_, _>>();
1229 let census = match self.forwarding.route_census(None) {
1230 Ok(census) => census,
1231 Err(err) => {
1232 warn!(error = %err, "failed to read route census for capability deny enforcement");
1233 return;
1234 }
1235 };
1236
1237 for (target_module_id, routes) in census {
1238 let Some(target_manifest) = manifests.get(&target_module_id) else {
1239 continue;
1240 };
1241 let mut closed_routes = Vec::new();
1242 let mut module_goodbyes = Vec::new();
1243 for route in routes {
1244 let Principal::Reserved {
1245 module_id: opening_module_id,
1246 } = &route.principal
1247 else {
1248 continue;
1249 };
1250 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1251 continue;
1252 };
1253 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1254 continue;
1255 };
1256
1257 match self.forwarding.release_client_route(
1258 route.goodbye_target.connection_id,
1259 route.goodbye_target.channel,
1260 route.goodbye_target.epoch,
1261 ) {
1262 Ok(RouteRelease::Removed(module_goodbye)) => {
1263 warn!(
1264 opening_module_id,
1265 target_module_id,
1266 capability,
1267 "force-closing route because an attested capability deny edge now matches"
1268 );
1269 closed_routes.push(route);
1270 module_goodbyes.push(module_goodbye);
1271 }
1272 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1273 Err(err) => warn!(
1274 opening_module_id,
1275 target_module_id,
1276 capability,
1277 error = %err,
1278 "failed to force-close capability-denied route"
1279 ),
1280 }
1281 }
1282
1283 if closed_routes.is_empty() {
1284 continue;
1285 }
1286 send_route_control_pushes(
1287 &self.forwarding,
1288 closed_routes,
1289 ClientControlPush::RouteClosed {
1290 module_id: target_module_id,
1291 channels: Vec::new(),
1292 reason: RouteCloseReason::CapabilityDenied,
1293 drained: false,
1294 abandoned: 0,
1295 excluded_subscriptions: 0,
1296 terminal: Some(false),
1297 },
1298 );
1299 self.emit_route_goodbyes(module_goodbyes);
1300 }
1301 }
1302
1303 fn not_ready_reason(
1309 &self,
1310 registration: &crate::registry::ModuleRegistration,
1311 ) -> Option<NotReadyReason> {
1312 if !registration.ready {
1313 return Some(NotReadyReason {
1314 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1315 capability: None,
1316 });
1317 }
1318 self.first_unprovided_required_capability(registration)
1319 .map(|capability| NotReadyReason {
1320 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1321 capability: Some(capability),
1322 })
1323 }
1324
1325 fn first_unprovided_required_capability(
1343 &self,
1344 registration: &crate::registry::ModuleRegistration,
1345 ) -> Option<String> {
1346 let required = registration
1347 .manifest
1348 .capabilities
1349 .iter()
1350 .flat_map(|declarations| declarations.requires.iter())
1351 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1352 .map(|requirement| requirement.capability.as_str())
1353 .collect::<BTreeSet<_>>();
1354 if required.is_empty() {
1355 return None;
1356 }
1357 let module_id = registration.manifest.module_id.as_str();
1358 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1359 if required
1360 .iter()
1361 .any(|capability| verdict(capability).is_none())
1362 {
1363 self.refresh_capability_requirements();
1364 }
1365 required
1366 .into_iter()
1367 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1368 .map(str::to_string)
1369 }
1370
1371 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1372 self.capability_evaluator
1373 .statuses()
1374 .into_iter()
1375 .map(capability_requirement_status)
1376 .collect()
1377 }
1378
1379 fn deregister_connection(
1386 &self,
1387 connection_id: ConnectionId,
1388 reason: RegistrationEndReason,
1389 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1390 self.registry
1391 .deregister_connection_with_reason(connection_id, reason)
1392 }
1393
1394 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1395 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1396 return None;
1397 }
1398 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1399 parse_client_control_request(&frame.body)
1400 else {
1401 return None;
1402 };
1403 Some(target_module_id(&target).to_string())
1404 }
1405
1406 pub(crate) fn route_open_capacity_refusal(
1407 &self,
1408 ctx: &RouteCtx,
1409 frame: &Frame,
1410 target_module_id: &str,
1411 in_flight: usize,
1412 limit: usize,
1413 ) -> Result<Frame, RouterError> {
1414 self.route_open_admission_refusal_frame(
1415 ctx,
1416 frame,
1417 target_module_id,
1418 "open_admission_full",
1419 (in_flight, limit),
1420 format!(
1421 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1422 ),
1423 )
1424 }
1425
1426 fn route_open_target_capacity_refusal(
1427 &self,
1428 ctx: &RouteCtx,
1429 frame: &Frame,
1430 target_module_id: &str,
1431 in_flight: usize,
1432 ) -> Result<Frame, RouterError> {
1433 self.route_open_admission_refusal_frame(
1434 ctx,
1435 frame,
1436 target_module_id,
1437 "target_binds_full",
1438 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1439 format!(
1440 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1441 ),
1442 )
1443 }
1444
1445 fn route_open_admission_refusal_frame(
1453 &self,
1454 ctx: &RouteCtx,
1455 frame: &Frame,
1456 target_module_id: &str,
1457 reason: &'static str,
1458 (in_flight, limit): (usize, usize),
1459 message: impl Into<String>,
1460 ) -> Result<Frame, RouterError> {
1461 let code = error_codes::TARGET_UNAVAILABLE;
1462 self.counters.increment_route_open_refused(code);
1463 info!(
1464 target: "control",
1465 code,
1466 reason,
1467 module_id = ?target_module_id,
1468 connection_id = ctx.connection_id.get(),
1469 in_flight,
1470 limit,
1471 "route.open refused"
1472 );
1473 control_error_frame(frame, code, message.into())
1474 }
1475
1476 #[cfg(test)]
1482 pub fn handle_control(
1483 &self,
1484 connection_id: ConnectionId,
1485 frame: Frame,
1486 ) -> Result<Vec<Frame>, RouterError> {
1487 match frame.header.ty {
1488 FrameType::Ping => Ok(vec![pong(&frame)?]),
1489 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1490 FrameType::Goodbye => self.handle_goodbye(connection_id),
1491 ty => Ok(vec![control_error_frame(
1492 &frame,
1493 "unsupported_control_frame",
1494 format!("unsupported channel-0 frame {ty:?}"),
1495 )?]),
1496 }
1497 }
1498
1499 pub async fn handle_control_frame(
1500 &self,
1501 ctx: &RouteCtx,
1502 frame: Frame,
1503 ) -> Result<Vec<Frame>, RouterError> {
1504 self.handle_control_frame_timed(ctx, frame, None).await
1505 }
1506
1507 pub(crate) async fn handle_control_frame_timed(
1508 &self,
1509 ctx: &RouteCtx,
1510 frame: Frame,
1511 dispatch_started_at: Option<StdInstant>,
1512 ) -> Result<Vec<Frame>, RouterError> {
1513 match frame.header.ty {
1514 FrameType::Ping => Ok(vec![pong(&frame)?]),
1515 FrameType::Hello => {
1516 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1517 }
1518 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1519 FrameType::Cancel => {
1520 if self
1523 .forwarding
1524 .operator_confirms()
1525 .cancel(ctx.connection_id, frame.header.corr)
1526 || self
1527 .supervisor
1528 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1529 {
1530 Ok(Vec::new())
1531 } else {
1532 Ok(vec![control_error_frame(
1533 &frame,
1534 "unknown_subscription",
1535 "no supervisor spawn subscription has this correlation id",
1536 )?])
1537 }
1538 }
1539 FrameType::Request => {
1540 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1543 if op
1544 .as_ref()
1545 .is_some_and(|probe| probe.op == "operator.confirm")
1546 {
1547 return self.handle_operator_confirm(ctx, frame);
1548 }
1549 if self
1550 .forwarding
1551 .module_endpoint_for_connection(ctx.connection_id)
1552 .map_err(RouterError::Forwarding)?
1553 .is_some()
1554 {
1555 if !is_known_module_request_op(&frame.body) {
1556 return Ok(vec![control_error_frame(
1557 &frame,
1558 "unsupported_control_frame",
1559 "module-originated channel-0 REQUEST is not supported",
1560 )?]);
1561 }
1562 let request = match parse_module_control_request_from_module(&frame.body) {
1563 Ok(request) => request,
1564 Err((err, ControlRequestBodyError::UnknownOp)) => {
1565 return Ok(vec![control_error_frame(
1566 &frame,
1567 "unsupported_control_frame",
1568 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1569 )?])
1570 }
1571 Err((err, ControlRequestBodyError::InvalidBody)) => {
1572 return Ok(vec![control_error_frame(
1573 &frame,
1574 "invalid_control_body",
1575 format!("malformed module control body: {err}"),
1576 )?])
1577 }
1578 };
1579 let op = module_control_request_op(&request);
1580 let corr = frame.header.corr;
1581 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1582 let result =
1583 self.handle_module_control_request(ctx.connection_id, frame, request);
1584 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1585 return result;
1586 }
1587
1588 if is_known_module_request_op(&frame.body) {
1589 return Ok(vec![control_error_frame(
1590 &frame,
1591 "not_registered",
1592 "catalog.update requires an active module registration owned by this connection",
1593 )?]);
1594 }
1595
1596 let request = match parse_client_control_request(&frame.body) {
1597 Ok(request) => request,
1598 Err((err, ControlRequestBodyError::UnknownOp)) => {
1599 return Ok(vec![control_error_frame(
1600 &frame,
1601 "unknown_control_op",
1602 format!("unknown client control op: {err}"),
1603 )?])
1604 }
1605 Err((err, ControlRequestBodyError::InvalidBody)) => {
1606 return Ok(vec![control_error_frame(
1607 &frame,
1608 "invalid_control_body",
1609 format!("malformed client control body: {err}"),
1610 )?])
1611 }
1612 };
1613 let op = client_control_request_op(&request);
1614 let corr = frame.header.corr;
1615 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1616 #[cfg(test)]
1617 if let Some(delay) = self.control_dispatch_delay {
1618 tokio::time::sleep(delay).await;
1619 }
1620 let result = self
1621 .handle_client_control_request(ctx, frame, request)
1622 .await;
1623 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1624 result
1625 }
1626 FrameType::Push => {
1627 let Some(endpoint) = self
1628 .forwarding
1629 .module_endpoint_for_connection(ctx.connection_id)
1630 .map_err(RouterError::Forwarding)?
1631 else {
1632 return Ok(vec![control_error_frame(
1633 &frame,
1634 "unsupported_control_frame",
1635 "client-originated channel-0 PUSH is not supported",
1636 )?]);
1637 };
1638 self.handle_status_update(endpoint, frame)
1639 }
1640 FrameType::Response | FrameType::Error
1641 if self
1642 .forwarding
1643 .module_endpoint_for_connection(ctx.connection_id)
1644 .map_err(RouterError::Forwarding)?
1645 .is_some() =>
1646 {
1647 self.handle_module_relay_response(ctx.connection_id, frame)
1648 }
1649 ty => Ok(vec![control_error_frame(
1650 &frame,
1651 "unsupported_control_frame",
1652 format!("unsupported channel-0 frame {ty:?}"),
1653 )?]),
1654 }
1655 }
1656
1657 pub fn cleanup_connection(
1658 &self,
1659 connection_id: ConnectionId,
1660 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1661 self.cleanup_connection_with_end_reason(
1662 connection_id,
1663 RegistrationEndReason::ConnectionClosed,
1664 )
1665 }
1666
1667 fn cleanup_connection_with_end_reason(
1668 &self,
1669 connection_id: ConnectionId,
1670 requested_reason: RegistrationEndReason,
1671 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1672 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1673 self.registry
1674 .get_module_by_connection(connection_id)?
1675 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1676 .and_then(|module| module.registration_end_reason().ok().flatten())
1677 .unwrap_or(requested_reason)
1678 } else {
1679 requested_reason
1680 };
1681 let crash_closed = self
1682 .registry
1683 .get_module_by_connection(connection_id)?
1684 .and_then(|registration| {
1685 self.forwarding
1686 .module_endpoint_for_connection(connection_id)
1687 .ok()
1688 .flatten()
1689 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1690 .map(|routes| (registration.manifest.module_id, routes))
1691 });
1692 let crash_closed = crash_closed.map(|(module_id, routes)| {
1693 let terminal = match self.supervisor.get(&module_id) {
1694 None => false,
1695 Some(module) => match module.will_recover_after_connection_loss() {
1696 Ok(will_recover) => !will_recover,
1697 Err(err) => {
1698 warn!(
1699 %module_id,
1700 error = %err,
1701 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1702 );
1703 false
1704 }
1705 },
1706 };
1707 let reason = match self.forwarding.is_daemon_draining() {
1711 Ok(true) => RouteCloseReason::Restart,
1712 Ok(false) => RouteCloseReason::Crash,
1713 Err(err) => {
1714 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1715 RouteCloseReason::Crash
1716 }
1717 };
1718 (module_id, routes, reason, terminal)
1719 });
1720 let registrations = self.deregister_connection(connection_id, end_reason);
1721 let cleanup = if crash_closed.is_some() {
1722 self.forwarding.cleanup_connection_counted(connection_id)
1723 } else {
1724 self.forwarding
1730 .cleanup_connection(connection_id)
1731 .map(|released| crate::forwarding::ConnectionCleanup {
1732 released,
1733 abandoned_relays: 0,
1734 })
1735 };
1736 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1741 let abandoned = cleanup
1742 .as_ref()
1743 .map_or(0, |cleanup| cleanup.abandoned_relays);
1744 send_route_control_pushes(
1745 &self.forwarding,
1746 routes,
1747 ClientControlPush::RouteClosed {
1748 module_id,
1749 channels: Vec::new(),
1750 reason,
1751 drained: false,
1752 abandoned,
1753 excluded_subscriptions: 0,
1754 terminal: Some(terminal),
1755 },
1756 );
1757 }
1758 if let Ok(cleanup) = cleanup {
1759 self.emit_route_goodbyes(cleanup.released);
1760 }
1761 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1765 crate::supervise::notify_registration_release();
1766 self.capability_evaluator.wake_deadline_loop();
1767 self.refresh_capability_requirements();
1768 }
1769 self.supervisor.remove_spawn_subscribers(connection_id);
1770 self.hello_launch_nonces
1773 .lock()
1774 .unwrap_or_else(|poisoned| poisoned.into_inner())
1775 .forget(connection_id);
1776 self.scopes
1777 .write()
1778 .unwrap_or_else(|poisoned| poisoned.into_inner())
1779 .release_connection(connection_id);
1780 registrations
1781 }
1782
1783 pub(crate) fn handle_route_goodbye(
1784 &self,
1785 connection_id: ConnectionId,
1786 route_channel: u16,
1787 route_epoch: u32,
1788 ) -> Result<bool, RouterError> {
1789 debug!(
1790 connection_id = connection_id.get(),
1791 route_channel, route_epoch, "handling route GOODBYE"
1792 );
1793 let RouteRelease::Removed(released_route) = self
1794 .forwarding
1795 .release_client_route(connection_id, route_channel, route_epoch)
1796 .map_err(RouterError::Forwarding)?
1797 else {
1798 return Ok(false);
1799 };
1800 self.emit_route_goodbyes(vec![released_route]);
1801 Ok(true)
1802 }
1803
1804 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1805 for released in released_routes {
1806 let frame = match Frame::build_with_version(
1807 released.negotiated_ver,
1808 FrameType::Goodbye,
1809 control_flags(),
1810 released.channel,
1811 released.epoch,
1812 0,
1813 Vec::new(),
1814 ) {
1815 Ok(frame) => frame,
1816 Err(err) => {
1817 warn!(
1818 route_channel = released.channel,
1819 error = %err,
1820 "failed to build route GOODBYE frame"
1821 );
1822 continue;
1823 }
1824 };
1825 if !released.close_on_delivery_failure() {
1826 crate::forwarding::send_module_route_goodbye(
1827 &self.counters,
1828 &released.sink,
1829 frame,
1830 released.module_id.as_deref(),
1831 "client route released",
1832 );
1833 continue;
1834 }
1835 if let Err(err) = released.sink.try_send(frame) {
1836 warn!(
1837 target_connection_id = released.connection_id.get(),
1838 route_channel = released.channel,
1839 error = %err,
1840 "route GOODBYE was not delivered to client; closing target connection"
1841 );
1842 if self
1843 .forwarding
1844 .escalate_client_delivery_failure(
1845 released.connection_id,
1846 released.channel,
1847 released.epoch,
1848 CloseReason::new(
1849 "route_goodbye_delivery_failed",
1850 format!(
1851 "failed to enqueue route GOODBYE for channel {}: {err}",
1852 released.channel
1853 ),
1854 ),
1855 crate::forwarding::UndeliveredFrame {
1856 module_id: released.module_id.as_deref(),
1857 sink: &released.sink,
1858 },
1859 )
1860 .unwrap_or(false)
1861 {
1862 self.counters.increment_goodbye_relay_client_failed();
1863 }
1864 }
1865 }
1866 }
1867
1868 fn send_abandoned_route_bind_goodbye(
1880 &self,
1881 module_sink: &crate::FrameSink,
1882 negotiated_ver: u8,
1883 module_channel: u16,
1884 module_epoch: u32,
1885 ) {
1886 let frame = match Frame::build_with_version(
1887 negotiated_ver,
1888 FrameType::Goodbye,
1889 control_flags(),
1890 module_channel,
1891 module_epoch,
1892 0,
1893 Vec::new(),
1894 ) {
1895 Ok(frame) => frame,
1896 Err(err) => {
1897 warn!(
1898 route_channel = module_channel,
1899 error = %err,
1900 "failed to build GOODBYE for abandoned route.bind"
1901 );
1902 return;
1903 }
1904 };
1905 crate::forwarding::send_module_route_goodbye(
1906 &self.counters,
1907 module_sink,
1908 frame,
1909 None,
1910 "abandoned route.bind",
1911 );
1912 }
1913
1914 fn handle_hello(
1915 &self,
1916 connection_id: ConnectionId,
1917 sink: Option<crate::FrameSink>,
1918 frame: Frame,
1919 ) -> Result<Vec<Frame>, RouterError> {
1920 debug!(
1921 connection_id = connection_id.get(),
1922 corr = frame.header.corr,
1923 "handling HELLO"
1924 );
1925 if self
1929 .registry
1930 .get_module_by_connection(connection_id)
1931 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
1932 .is_some()
1933 {
1934 return Ok(vec![control_error_frame(
1935 &frame,
1936 "invalid_hello",
1937 "connection is already registered as a module",
1938 )?]);
1939 }
1940 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1941 Ok(value) => value,
1942 Err(err) => {
1943 return Ok(vec![control_error_frame(
1944 &frame,
1945 "invalid_hello",
1946 format!("malformed HELLO body: {err}"),
1947 )?])
1948 }
1949 };
1950 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1951 return Ok(vec![control_error_frame(
1952 &frame,
1953 "invalid_capability_grammar",
1954 err.to_string(),
1955 )?]);
1956 }
1957 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1958 return Ok(vec![control_error_frame(
1959 &frame,
1960 "invalid_manifest",
1961 err.to_string(),
1962 )?]);
1963 }
1964 if let Err(err) = validate_hello_event_declarations(&hello_value) {
1965 return Ok(vec![control_error_frame(
1966 &frame,
1967 "invalid_event_declaration",
1968 err.to_string(),
1969 )?]);
1970 }
1971 if let Some(provenance) = hello_value
1972 .get("manifest")
1973 .and_then(|manifest| manifest.get("provenance"))
1974 {
1975 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1976 return Ok(vec![control_error_frame(
1977 &frame,
1978 "invalid_manifest",
1979 format!("malformed manifest provenance: {err}"),
1980 )?]);
1981 }
1982 }
1983 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1984 Ok(hello) => hello,
1985 Err(err) => {
1986 return Ok(vec![control_error_frame(
1987 &frame,
1988 "invalid_hello",
1989 format!("malformed HELLO body: {err}"),
1990 )?])
1991 }
1992 };
1993
1994 if hello.protocol_ver != hello.manifest.protocol_ver {
1995 return Ok(vec![control_error_frame(
1996 &frame,
1997 "invalid_manifest",
1998 format!(
1999 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2000 hello.protocol_ver, hello.manifest.protocol_ver
2001 ),
2002 )?]);
2003 }
2004
2005 if hello.manifest.module_id.trim().is_empty() {
2006 return Ok(vec![control_error_frame(
2007 &frame,
2008 "invalid_manifest",
2009 "manifest module_id must not be empty",
2010 )?]);
2011 }
2012
2013 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2014 Ok(negotiated_ver) => negotiated_ver,
2015 Err(message) => {
2016 return Ok(vec![control_error_frame(
2017 &frame,
2018 "version_unsupported",
2019 message,
2020 )?])
2021 }
2022 };
2023
2024 let swap_admission = self
2032 .supervisor
2033 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2034 if swap_admission == SwapHelloAdmission::Refused {
2035 warn!(
2036 module_id = %hello.manifest.module_id,
2037 connection_id = connection_id.get(),
2038 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2039 );
2040 return Ok(vec![control_error_frame(
2041 &frame,
2042 "swap_token_invalid",
2043 format!(
2044 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2045 hello.manifest.module_id
2046 ),
2047 )?]);
2048 }
2049 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2050
2051 if let Some(rejection) = (!swap_candidate)
2059 .then(|| {
2060 self.supervisor.reserved_hello_rejection(
2061 &hello.manifest.module_id,
2062 hello.launch_nonce.as_deref(),
2063 )
2064 })
2065 .flatten()
2066 {
2067 let message = match rejection {
2068 ReservedHelloRejection::Exact { module_id } => format!(
2069 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2070 ),
2071 ReservedHelloRejection::Prefix {
2072 prefix,
2073 owner_module_id,
2074 } => format!(
2075 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2076 hello.manifest.module_id
2077 ),
2078 };
2079 return Ok(vec![control_error_frame(
2080 &frame,
2081 "reserved_module",
2082 message,
2083 )?]);
2084 }
2085
2086 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2087 &hello.manifest.module_id,
2088 hello.manifest.capabilities.as_ref(),
2089 );
2090 if let Some(refusal) = reserved_capability_refusals.first() {
2091 let capability = refusal.capability.clone();
2092 let bound_module = refusal.claimants[0].clone();
2093 log_duplicate_claim_events(reserved_capability_refusals);
2094 return Ok(vec![control_error_frame(
2095 &frame,
2096 "reserved_capability",
2097 format!(
2098 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2099 capability, bound_module, hello.manifest.module_id
2100 ),
2101 )?]);
2102 }
2103
2104 if self
2107 .forwarding
2108 .connection_has_client_routes(connection_id)
2109 .map_err(RouterError::Forwarding)?
2110 {
2111 return Ok(vec![control_error_frame(
2112 &frame,
2113 "invalid_hello",
2114 "connection has open client routes and cannot also register as a module",
2115 )?]);
2116 }
2117
2118 self.hello_launch_nonces
2123 .lock()
2124 .unwrap_or_else(|poisoned| poisoned.into_inner())
2125 .record(connection_id, hello.launch_nonce.as_deref());
2126 let control_ops = effective_module_control_ops(hello.control_ops);
2127 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2130 if swap_candidate {
2131 return self.register_swap_candidate(
2132 connection_id,
2133 sink,
2134 &frame,
2135 hello.manifest,
2136 negotiated_ver,
2137 control_ops,
2138 hello_ack,
2139 );
2140 }
2141 let registration = match self.registry.register_with_control_ops(
2142 hello.manifest,
2143 negotiated_ver,
2144 connection_id,
2145 control_ops,
2146 ) {
2147 Ok(registration) => registration,
2148 Err(RegistryError::DuplicateModuleId { module_id }) => {
2149 return Ok(vec![control_error_frame(
2150 &frame,
2151 "duplicate_module_id",
2152 format!(
2153 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2154 ),
2155 )?])
2156 }
2157 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2158 return Ok(vec![control_error_frame(
2159 &frame,
2160 "invalid_module_id",
2161 err.to_string(),
2162 )?])
2163 }
2164 Err(err) => {
2165 return Ok(vec![control_error_frame(
2166 &frame,
2167 "registry_error",
2168 err.to_string(),
2169 )?])
2170 }
2171 };
2172
2173 let reply = if let Some(sink) = sink {
2174 let concurrency = manifest_concurrency(®istration.manifest);
2190 if let Err(err) = self.forwarding.register_module_connection_acked(
2191 connection_id,
2192 registration.manifest.module_id.clone(),
2193 negotiated_ver,
2194 concurrency,
2195 sink,
2196 hello_ack,
2197 ) {
2198 if matches!(
2202 self.deregister_connection(
2203 connection_id,
2204 RegistrationEndReason::RegistrationFailed,
2205 ),
2206 Ok(r) if !r.is_empty()
2207 ) {
2208 crate::supervise::notify_registration_release();
2209 }
2210 return Ok(vec![control_error_frame(
2211 &frame,
2212 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2213 "invalid_hello"
2214 } else {
2215 forwarding_error_code(&err)
2216 },
2217 err.to_string(),
2218 )?]);
2219 }
2220 Vec::new()
2221 } else {
2222 vec![hello_ack]
2225 };
2226
2227 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2236 info!(
2237 module_id = %registration.manifest.module_id,
2238 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2239 );
2240 }
2241
2242 self.apply_registration_capabilities(®istration);
2243
2244 info!(
2245 module_id = %registration.manifest.module_id,
2246 module_version = %registration.manifest.module_version,
2247 negotiated_ver,
2248 routable_provider = manifest_provides_routable_role(®istration.manifest),
2249 connection_id = connection_id.get(),
2250 "module registered"
2251 );
2252
2253 Ok(reply)
2254 }
2255
2256 #[allow(clippy::too_many_arguments)]
2266 fn register_swap_candidate(
2267 &self,
2268 connection_id: ConnectionId,
2269 sink: Option<crate::FrameSink>,
2270 frame: &Frame,
2271 manifest: ModuleManifest,
2272 negotiated_ver: u8,
2273 control_ops: Vec<String>,
2274 hello_ack: Frame,
2275 ) -> Result<Vec<Frame>, RouterError> {
2276 let module_id = manifest.module_id.clone();
2277 let registration = match self.registry.register_candidate_with_control_ops(
2278 manifest,
2279 negotiated_ver,
2280 connection_id,
2281 control_ops,
2282 ) {
2283 Ok(registration) => registration,
2284 Err(RegistryError::DuplicateModuleId { module_id }) => {
2285 return Ok(vec![control_error_frame(
2286 frame,
2287 "duplicate_module_id",
2288 format!(
2289 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2290 ),
2291 )?])
2292 }
2293 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2294 return Ok(vec![control_error_frame(
2295 frame,
2296 "invalid_module_id",
2297 err.to_string(),
2298 )?])
2299 }
2300 Err(err) => {
2301 return Ok(vec![control_error_frame(
2302 frame,
2303 "registry_error",
2304 err.to_string(),
2305 )?])
2306 }
2307 };
2308 let reply = if let Some(sink) = sink {
2309 let concurrency = manifest_concurrency(®istration.manifest);
2313 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2314 connection_id,
2315 module_id.clone(),
2316 negotiated_ver,
2317 concurrency,
2318 sink,
2319 hello_ack,
2320 ) {
2321 if matches!(
2322 self.deregister_connection(
2323 connection_id,
2324 RegistrationEndReason::RegistrationFailed,
2325 ),
2326 Ok(r) if !r.is_empty()
2327 ) {
2328 crate::supervise::notify_registration_release();
2329 }
2330 return Ok(vec![control_error_frame(
2331 frame,
2332 forwarding_error_code(&err),
2333 err.to_string(),
2334 )?]);
2335 }
2336 Vec::new()
2337 } else {
2338 vec![hello_ack]
2339 };
2340 self.supervisor.mark_swap_candidate_admitted(&module_id);
2341 info!(
2342 module_id = %module_id,
2343 module_version = %registration.manifest.module_version,
2344 negotiated_ver,
2345 ready = registration.ready,
2346 connection_id = connection_id.get(),
2347 "swap candidate registered; not routable until cutover"
2348 );
2349 Ok(reply)
2350 }
2351
2352 fn build_hello_ack(
2353 &self,
2354 frame: &Frame,
2355 negotiated_ver: u8,
2356 module_id: &str,
2357 ) -> Result<Frame, RouterError> {
2358 let ack = ModuleHelloAckBody {
2359 negotiated_ver,
2360 subc_ops: module_subc_ops(),
2361 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2362 storage: self
2363 .storage_config
2364 .as_ref()
2365 .map(|cfg| cfg.descriptor_for(module_id)),
2366 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2367 };
2368 let body = serde_json::to_vec(&ack).map_err(|err| {
2369 RouterError::backend(
2370 0,
2371 frame.header.corr,
2372 format!("failed to encode HELLO_ACK: {err}"),
2373 )
2374 })?;
2375
2376 Frame::build_with_version(
2377 negotiated_ver,
2378 FrameType::HelloAck,
2379 control_flags(),
2380 0,
2381 0,
2382 frame.header.corr,
2383 body,
2384 )
2385 .map_err(RouterError::FrameBuild)
2386 }
2387
2388 async fn handle_client_control_request(
2389 &self,
2390 ctx: &RouteCtx,
2391 frame: Frame,
2392 request: ClientControlRequest,
2393 ) -> Result<Vec<Frame>, RouterError> {
2394 match request {
2395 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2396 ClientControlRequest::CatalogList { module_id } => {
2397 self.handle_catalog_list(frame, module_id)
2398 }
2399 ClientControlRequest::RouteOpen {
2400 target,
2401 identity,
2402 consumer_identity,
2403 consumer_capabilities,
2404 role_versions,
2405 admission_facts,
2406 scope,
2407 } => {
2408 self.handle_route_open(
2409 ctx,
2410 frame,
2411 RouteOpenRequest {
2412 target,
2413 identity,
2414 consumer_identity,
2415 consumer_capabilities,
2416 role_versions,
2417 admission_facts,
2418 scope,
2419 },
2420 )
2421 .await
2422 }
2423 ClientControlRequest::RoutePoll {
2424 route_channel,
2425 route_epoch,
2426 kind,
2427 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2428 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2429 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2430 self.handle_supervisor_spawn_snapshot(frame)
2431 }
2432 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2433 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2434 }
2435 ClientControlRequest::SupervisorRestart {
2436 module_id,
2437 drain_timeout_ms,
2438 } => {
2439 self.log_supervisor_request_received(
2440 ctx,
2441 frame.header.corr,
2442 ops::SUPERVISOR_RESTART,
2443 Some(&module_id),
2444 None,
2445 )?;
2446 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2447 .await
2448 }
2449 ClientControlRequest::SupervisorSwap {
2450 module_id,
2451 ready_timeout_ms,
2452 } => {
2453 self.log_supervisor_request_received(
2454 ctx,
2455 frame.header.corr,
2456 ops::SUPERVISOR_SWAP,
2457 Some(&module_id),
2458 None,
2459 )?;
2460 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2461 .await
2462 }
2463 ClientControlRequest::SupervisorReload { module_id } => {
2464 self.log_supervisor_request_received(
2465 ctx,
2466 frame.header.corr,
2467 ops::SUPERVISOR_RELOAD,
2468 Some(&module_id),
2469 None,
2470 )?;
2471 self.handle_supervisor_reload(frame, module_id).await
2472 }
2473 ClientControlRequest::SupervisorRescan { preview } => {
2474 if !preview {
2475 self.log_supervisor_request_received(
2476 ctx,
2477 frame.header.corr,
2478 ops::SUPERVISOR_RESCAN,
2479 None,
2480 None,
2481 )?;
2482 }
2483 self.handle_supervisor_rescan(frame, preview).await
2484 }
2485 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2486 self.log_supervisor_request_received(
2487 ctx,
2488 frame.header.corr,
2489 ops::SUPERVISOR_RELEASE_RESERVED,
2490 Some(&module_id),
2491 None,
2492 )?;
2493 self.handle_supervisor_release_reserved(frame, module_id)
2494 .await
2495 }
2496 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2497 self.log_supervisor_request_received(
2498 ctx,
2499 frame.header.corr,
2500 ops::SUPERVISOR_SET_ENABLED,
2501 Some(&module_id),
2502 Some(enabled),
2503 )?;
2504 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2505 .await
2506 }
2507 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2508 self.handle_supervisor_health_probe(frame, module_id).await
2509 }
2510 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2511 ClientControlRequest::SupervisorRoutes { module_id } => {
2512 self.handle_supervisor_routes(frame, module_id)
2513 }
2514 ClientControlRequest::SupervisorProvenance { module_id } => {
2515 self.handle_supervisor_provenance(frame, module_id).await
2516 }
2517 ClientControlRequest::SupervisorStderrTail {
2518 module_id,
2519 max_lines,
2520 max_bytes,
2521 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2522 ClientControlRequest::SupervisorTerminals { module_id } => {
2523 self.handle_supervisor_terminals(frame, module_id).await
2524 }
2525 }
2526 }
2527
2528 fn log_supervisor_request_received(
2529 &self,
2530 ctx: &RouteCtx,
2531 corr: u64,
2532 op: &'static str,
2533 module_id: Option<&str>,
2534 enabled: Option<bool>,
2535 ) -> Result<(), RouterError> {
2536 let caller = self
2537 .registry
2538 .get_module_by_connection(ctx.connection_id)
2539 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2540 .map(|registration| Principal::Reserved {
2541 module_id: registration.manifest.module_id,
2542 })
2543 .unwrap_or(Principal::Direct);
2544 let caller = principal_label(&caller);
2545 info!(
2548 target: "control",
2549 op,
2550 module_id,
2551 enabled,
2552 connection_id = ctx.connection_id.get(),
2553 caller = %caller,
2554 "supervisor request received"
2555 );
2556 Ok(())
2557 }
2558
2559 fn handle_module_control_request(
2560 &self,
2561 connection_id: ConnectionId,
2562 frame: Frame,
2563 request: ModuleControlRequestFromModule,
2564 ) -> Result<Vec<Frame>, RouterError> {
2565 match request {
2566 ModuleControlRequestFromModule::CatalogUpdate {
2567 provides,
2568 capabilities,
2569 ready,
2570 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2571 ModuleControlRequestFromModule::LiveRoots {} => {
2572 let registered = self
2573 .registry
2574 .get_module_by_connection(connection_id)
2575 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2576 let Some(registration) = registered else {
2577 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2578 };
2579 let response = self
2580 .forwarding
2581 .live_roots(®istration.manifest.module_id)
2582 .map_err(RouterError::Forwarding)?;
2583 Ok(vec![control_response_body_frame(
2584 &frame,
2585 &response,
2586 "ModuleControlResponseToModule::LiveRoots",
2587 )?])
2588 }
2589 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2590 self.handle_scope_sync(connection_id, frame, generation, scopes)
2591 }
2592 ModuleControlRequestFromModule::ScopeApply {
2593 generation,
2594 upsert,
2595 end,
2596 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2597 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2598 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2599 }
2600 }
2601 }
2602
2603 fn handle_operator_confirm(
2604 &self,
2605 ctx: &RouteCtx,
2606 frame: Frame,
2607 ) -> Result<Vec<Frame>, RouterError> {
2608 use crate::operator_confirm::{audit, Outcome};
2609 let registration = self
2610 .registry
2611 .get_module_by_connection(ctx.connection_id)
2612 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2613 let Some(registration) = registration else {
2614 let outcome = Outcome::refusal("not_registered");
2615 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2616 return Ok(vec![outcome.frame(&frame)]);
2617 };
2618 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2619 Ok(request) => request,
2620 Err(_) => {
2621 let outcome = Outcome::refusal("invalid_control_body");
2622 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2623 return Ok(vec![outcome.frame(&frame)]);
2624 }
2625 };
2626 let module_id = registration.manifest.module_id;
2627 let nonce = self
2631 .hello_launch_nonces
2632 .lock()
2633 .unwrap_or_else(|p| p.into_inner())
2634 .nonce(ctx.connection_id)
2635 .map(str::to_owned);
2636 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2637 self.supervisor
2638 .spawned_consumer_authorized(&module_id, nonce)
2639 });
2640 let confirms = self.forwarding.operator_confirms();
2641 self.forwarding
2642 .with_operator_route(
2643 ctx.connection_id,
2644 request.route_channel,
2645 request.route_epoch,
2646 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
2647 )
2648 .map_err(RouterError::Forwarding)
2649 }
2650
2651 fn handle_scope_sync(
2655 &self,
2656 connection_id: ConnectionId,
2657 frame: Frame,
2658 generation: u64,
2659 scopes: Vec<ScopeRecord>,
2660 ) -> Result<Vec<Frame>, RouterError> {
2661 self.handle_scope_change(connection_id, frame, generation, scopes, None)
2662 }
2663
2664 fn handle_scope_change(
2665 &self,
2666 connection_id: ConnectionId,
2667 frame: Frame,
2668 generation: u64,
2669 scopes: Vec<ScopeRecord>,
2670 end: Option<Vec<ScopeEnd>>,
2671 ) -> Result<Vec<Frame>, RouterError> {
2672 let is_apply = end.is_some();
2673 let Some(registration) = self
2674 .registry
2675 .get_module_by_connection(connection_id)
2676 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2677 else {
2678 return Ok(vec![control_error_frame(
2679 &frame,
2680 "not_registered",
2681 "scope.sync and scope.apply require an active module registration owned by this connection",
2682 )?]);
2683 };
2684 let owner = registration.manifest.module_id;
2685 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2686 let is_current_launch = |connection: ConnectionId| {
2687 self.hello_launch_nonces
2688 .lock()
2689 .unwrap_or_else(|poisoned| poisoned.into_inner())
2690 .presented(connection, current_nonce.as_deref())
2691 };
2692 let mut table = self
2697 .scopes
2698 .write()
2699 .unwrap_or_else(|poisoned| poisoned.into_inner());
2700 let outcome = match end {
2701 Some(end) => table.apply(
2702 &owner,
2703 connection_id,
2704 is_current_launch,
2705 generation,
2706 scopes,
2707 end,
2708 ),
2709 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
2710 };
2711 let drained = match &outcome {
2712 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
2713 Err(_) => Vec::new(),
2714 };
2715 let has_deadlines = outcome.is_ok() && table.has_deadlines();
2716 drop(table);
2717 if has_deadlines {
2718 self.scope_deadline_added.notify_one();
2719 }
2720 match outcome {
2721 Ok(applied) => {
2722 let counts = ScopeOutcomeCounts::of(&applied.results);
2723 info!(
2724 owner = %owner,
2725 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2726 generation,
2727 records = applied.results.len(),
2728 created = counts.created,
2729 replaced = counts.replaced,
2730 updated = counts.updated,
2731 unchanged = counts.unchanged,
2732 refused = counts.refused,
2733 ended = applied.ended.len(),
2734 tag_changes = applied.tag_changes.len(),
2735 routes_closed = drained.len(),
2736 "scope change accepted"
2737 );
2738 for refused in applied
2744 .results
2745 .iter()
2746 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2747 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2748 {
2749 warn!(
2750 owner = %owner,
2751 generation,
2752 scope_ref = %refused.scope_ref,
2753 scope_epoch = refused.scope_epoch,
2754 code = refused.code.as_deref().unwrap_or(""),
2755 "scope record refused"
2756 );
2757 }
2758 self.close_scope_drained_routes(drained);
2759 let response = if is_apply {
2760 ModuleControlResponseToModule::ScopeApply {
2761 generation,
2762 results: applied.results,
2763 end_results: applied.end_results,
2764 ended: applied.ended,
2765 }
2766 } else {
2767 ModuleControlResponseToModule::ScopeSync {
2768 generation,
2769 results: applied.results,
2770 ended: applied.ended,
2771 }
2772 };
2773 Ok(vec![control_response_body_frame(
2774 &frame,
2775 &response,
2776 "scope change response",
2777 )?])
2778 }
2779 Err(refusal) => {
2780 info!(
2781 owner = %owner,
2782 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2783 generation,
2784 code = refusal.code,
2785 "scope change refused"
2786 );
2787 Ok(vec![control_error_frame(
2788 &frame,
2789 refusal.code,
2790 refusal.message,
2791 )?])
2792 }
2793 }
2794 }
2795
2796 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2802 if drained.is_empty() {
2803 return;
2804 }
2805 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2806 BTreeMap::new();
2807 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2808 for route in drained {
2809 warn!(
2810 module_id = %route.module_id,
2811 reason = ?route.reason,
2812 client_connection_id = route.client.connection_id.get(),
2813 route_channel = route.client.channel,
2814 "closing route because its scope changed"
2815 );
2816 pushes
2817 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2818 .or_insert_with(|| (route.reason, Vec::new()))
2819 .1
2820 .push(EndpointRoute {
2821 goodbye_target: route.client.clone(),
2822 principal: Principal::Unverified,
2823 bound_at: Instant::now(),
2824 draining: false,
2825 drain_reason: None,
2826 });
2827 goodbyes.push(route.module);
2828 goodbyes.push(route.client);
2829 }
2830 for ((module_id, _), (reason, routes)) in pushes {
2831 send_route_control_pushes(
2832 &self.forwarding,
2833 routes,
2834 ClientControlPush::RouteClosed {
2835 module_id,
2836 channels: Vec::new(),
2837 reason,
2838 drained: false,
2839 abandoned: 0,
2840 excluded_subscriptions: 0,
2841 terminal: Some(false),
2842 },
2843 );
2844 }
2845 self.emit_route_goodbyes(goodbyes);
2846 }
2847
2848 fn handle_scope_describe(
2851 &self,
2852 connection_id: ConnectionId,
2853 frame: Frame,
2854 owner: Principal,
2855 scope_ref: String,
2856 ) -> Result<Vec<Frame>, RouterError> {
2857 let registered = self
2858 .registry
2859 .get_module_by_connection(connection_id)
2860 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2861 if registered.is_none() {
2862 return Ok(vec![control_error_frame(
2863 &frame,
2864 "not_registered",
2865 "scope.describe requires an active module registration owned by this connection",
2866 )?]);
2867 }
2868 let description = self
2869 .scopes
2870 .read()
2871 .unwrap_or_else(|poisoned| poisoned.into_inner())
2872 .describe(&owner, &scope_ref);
2873 let owner_configured = match &owner {
2874 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
2879 _ => false,
2880 };
2881 let response = ModuleControlResponseToModule::ScopeDescribe {
2882 status: description.status,
2883 scope_epoch: description.scope_epoch,
2884 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2885 owner_synced: description.owner_synced,
2886 owner_configured,
2887 scope: description.stamp,
2888 };
2889 Ok(vec![control_response_body_frame(
2890 &frame,
2891 &response,
2892 "ModuleControlResponseToModule::ScopeDescribe",
2893 )?])
2894 }
2895
2896 fn handle_catalog_update(
2897 &self,
2898 connection_id: ConnectionId,
2899 frame: Frame,
2900 provides: Vec<ProviderRole>,
2901 capabilities: Option<CapabilityDeclarations>,
2902 ready: Option<bool>,
2903 ) -> Result<Vec<Frame>, RouterError> {
2904 self.refresh_capability_requirements();
2905 let Some(registration) = self
2906 .registry
2907 .get_module_by_connection(connection_id)
2908 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2909 else {
2910 return Ok(vec![control_error_frame(
2911 &frame,
2912 "not_registered",
2913 "catalog.update requires an active module registration owned by this connection",
2914 )?]);
2915 };
2916
2917 if let Some(message) =
2918 catalog_update_frozen_field_message(®istration.manifest, &provides)
2919 {
2920 return Ok(vec![control_error_frame(
2921 &frame,
2922 "catalog_update_frozen_field",
2923 message,
2924 )?]);
2925 }
2926
2927 let mut candidate = registration.manifest.clone();
2928 candidate.provides = provides.clone();
2929 candidate.capabilities = capabilities
2930 .clone()
2931 .or_else(|| registration.manifest.capabilities.clone());
2932 if let Err(err) = candidate.validate_capability_grammar() {
2933 return Ok(vec![control_error_frame(
2934 &frame,
2935 "invalid_capability_grammar",
2936 err.to_string(),
2937 )?]);
2938 }
2939
2940 let mut conflicts = self
2943 .capability_evaluator
2944 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
2945 if let Some(conflict) = conflicts.first() {
2946 let message = format!(
2947 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2948 conflict.capability, conflict.claimants[0], candidate.module_id
2949 );
2950 for conflict in &mut conflicts {
2951 conflict.source = DuplicateClaimSource::CatalogUpdate;
2952 }
2953 log_duplicate_claim_events(conflicts);
2954 return Ok(vec![control_error_frame(
2955 &frame,
2956 "reserved_capability",
2957 message,
2958 )?]);
2959 }
2960
2961 let updated = self
2962 .registry
2963 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2964 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2965 if updated.is_none() {
2966 return Ok(vec![control_error_frame(
2967 &frame,
2968 "not_registered",
2969 "catalog.update requires an active module registration owned by this connection",
2970 )?]);
2971 }
2972 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2973 log_duplicate_claim_events(
2974 self.capability_evaluator
2975 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2976 );
2977 }
2978 if capability_census_trigger(
2979 registration.manifest.capabilities.as_ref(),
2980 updated
2981 .as_ref()
2982 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2983 ) {
2984 self.enforce_capability_denies();
2985 }
2986 self.refresh_capability_requirements();
2987
2988 let response = ModuleControlResponseToModule::CatalogUpdate {};
2989 control_response_body_frame(
2990 &frame,
2991 &response,
2992 "ModuleControlResponseToModule::CatalogUpdate",
2993 )
2994 .map(|frame| vec![frame])
2995 }
2996
2997 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2998 self.refresh_capability_requirements();
2999 let mut counters = self.counters.snapshot();
3005 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3006 self.forwarding.client_route_concentration(),
3007 counters.as_object_mut(),
3008 ) {
3009 obj.insert(
3010 "client_connections_with_routes".into(),
3011 connections_with_routes.into(),
3012 );
3013 obj.insert("max_routes_on_one_connection".into(), max.into());
3014 }
3015 if let (Some(open_breakers), Some(obj)) = (
3021 self.route_bind_breakers.open_snapshot(),
3022 counters.as_object_mut(),
3023 ) {
3024 obj.insert("route_bind_breakers_open".into(), open_breakers);
3025 }
3026 let response = ClientControlResponse::ServerDescribe {
3027 protocol_ver: PROTOCOL_VERSION,
3028 subc_ops: subc_ops(),
3029 capabilities: self.subc_capabilities.as_ref().to_vec(),
3030 connected_clients: self.connected_clients.count(),
3031 counters: Some(counters),
3032 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3033 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3034 capability_requirements: self.capability_requirement_statuses(),
3035 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3036 };
3037 Ok(vec![control_response_body_frame(
3038 &frame,
3039 &response,
3040 "ClientControlResponse::ServerDescribe",
3041 )?])
3042 }
3043
3044 fn handle_catalog_list(
3045 &self,
3046 frame: Frame,
3047 module_id: Option<String>,
3048 ) -> Result<Vec<Frame>, RouterError> {
3049 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3050 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3051 })?;
3052 let entries = modules
3053 .into_iter()
3054 .filter(|registration| {
3055 module_id
3056 .as_deref()
3057 .map(|wanted| registration.manifest.module_id == wanted)
3058 .unwrap_or(true)
3059 })
3060 .map(|registration| {
3061 let not_ready = self.not_ready_reason(®istration);
3062 let roles = registration.manifest.provides;
3063 CatalogEntry::new(
3064 registration.manifest.module_id,
3065 roles,
3066 registration.control_ops,
3067 )
3068 .with_ready(not_ready.is_none())
3069 .with_not_ready(not_ready)
3070 .with_module_version(Some(registration.manifest.module_version))
3071 .with_capabilities(registration.manifest.capabilities)
3072 .with_self_signals(registration.manifest.self_signals)
3073 .with_events(registration.manifest.events)
3074 })
3075 .collect();
3076 let response = ClientControlResponse::CatalogList {
3077 generation,
3078 modules: entries,
3079 subc_ops: subc_ops(),
3080 };
3081 Ok(vec![control_response_body_frame(
3082 &frame,
3083 &response,
3084 "ClientControlResponse::CatalogList",
3085 )?])
3086 }
3087
3088 fn route_open_principal(
3089 &self,
3090 frame: &Frame,
3091 consumer_identity: Option<ConsumerIdentity>,
3092 ) -> Result<Result<Principal, Frame>, RouterError> {
3093 let Some(consumer_identity) = consumer_identity else {
3094 return Ok(Ok(Principal::Direct));
3095 };
3096
3097 if self.supervisor.spawned_consumer_authorized(
3098 &consumer_identity.module_id,
3099 &consumer_identity.launch_nonce,
3100 ) {
3101 return Ok(Ok(Principal::Reserved {
3102 module_id: consumer_identity.module_id,
3103 }));
3104 }
3105
3106 Ok(Err(control_error_frame(
3107 frame,
3108 "bad_consumer_identity",
3109 format!(
3110 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3111 consumer_identity.module_id
3112 ),
3113 )?))
3114 }
3115
3116 fn route_open_refusal_frame(
3123 &self,
3124 ctx: &RouteCtx,
3125 frame: &Frame,
3126 module_id: &str,
3127 reason: &'static str,
3128 code: &'static str,
3129 message: impl Into<String>,
3130 ) -> Result<Frame, RouterError> {
3131 self.observe_route_open_refusal(ctx, module_id, reason, code);
3132 control_error_frame(frame, code, message.into())
3133 }
3134
3135 fn route_open_breaker_refusal_frame(
3153 &self,
3154 ctx: &RouteCtx,
3155 frame: &Frame,
3156 module_id: &str,
3157 consecutive_timeouts: u32,
3158 retry_in: Duration,
3159 probe_in_flight: bool,
3160 ) -> Result<Frame, RouterError> {
3161 self.counters
3162 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3163 debug!(
3164 target: "control",
3165 code = "module_timeout",
3166 module_id = ?module_id,
3167 connection_id = ctx.connection_id.get(),
3168 consecutive_timeouts,
3169 retry_in_ms = retry_in.as_millis() as u64,
3170 probe_in_flight,
3171 "route.open refused by open bind-relay breaker"
3172 );
3173 let detail = if probe_in_flight {
3180 "one test route is already being tried; retry once it settles".to_string()
3181 } else {
3182 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3183 };
3184 control_error_frame(
3185 frame,
3186 "module_timeout",
3187 format!(
3188 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3189 timed out in a row); {detail}; its established routes are unaffected"
3190 ),
3191 )
3192 }
3193
3194 fn observe_route_open_refusal(
3207 &self,
3208 ctx: &RouteCtx,
3209 module_id: &str,
3210 reason: &'static str,
3211 code: &'static str,
3212 ) {
3213 self.counters.increment_route_open_refused(code);
3214 info!(
3215 target: "control",
3216 code,
3217 reason,
3218 module_id = ?module_id,
3219 connection_id = ctx.connection_id.get(),
3220 "route.open refused"
3221 );
3222 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3223 self.route_outages.record_not_serving(module_id, reason);
3224 }
3225 }
3226
3227 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3272 self.route_outages.record_accepted(module_id);
3273 self.counters.increment_route_open_accepted(principal);
3274 info!(
3275 target: "control",
3276 principal,
3277 module_id,
3278 connection_id = ctx.connection_id.get(),
3279 "route.open accepted"
3280 );
3281 }
3282
3283 fn supervised_absent_route_open_refusal_frame(
3284 &self,
3285 ctx: &RouteCtx,
3286 frame: &Frame,
3287 module_id: &str,
3288 code: &'static str,
3289 status: &crate::supervise::ModuleStatus,
3290 ) -> Result<Frame, RouterError> {
3291 self.counters.increment_route_open_refused(code);
3292 info!(
3293 target: "control",
3294 code,
3295 reason = "supervised_not_registered",
3296 module_id = ?module_id,
3297 connection_id = ctx.connection_id.get(),
3298 state = %status.state,
3299 enabled = status.enabled,
3300 live = status.live,
3301 "route.open refused"
3302 );
3303 self.route_outages
3307 .record_not_serving(module_id, "supervised_not_registered");
3308 control_error_frame(
3309 frame,
3310 code,
3311 format!(
3312 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3313 status.state, status.enabled, status.live
3314 ),
3315 )
3316 }
3317
3318 async fn handle_route_open(
3319 &self,
3320 ctx: &RouteCtx,
3321 frame: Frame,
3322 request: RouteOpenRequest,
3323 ) -> Result<Vec<Frame>, RouterError> {
3324 let RouteOpenRequest {
3325 target,
3326 mut identity,
3327 consumer_identity,
3328 consumer_capabilities,
3329 role_versions,
3330 admission_facts,
3331 scope,
3332 } = request;
3333 let target_module_id = target_module_id(&target).to_string();
3334 if self
3335 .registry
3336 .get_module_by_connection(ctx.connection_id)
3337 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3338 .is_some()
3339 {
3340 return Ok(vec![control_error_frame(
3341 &frame,
3342 "invalid_request",
3343 "module connections cannot open client routes",
3344 )?]);
3345 }
3346 debug!(
3347 connection_id = ctx.connection_id.get(),
3348 corr = frame.header.corr,
3349 module_id = %target_module_id,
3350 "handling route.open"
3351 );
3352
3353 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3359 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3360 self.observe_route_open_refusal(
3361 ctx,
3362 &target_module_id,
3363 "invalid_role_versions",
3364 error_codes::INVALID_REQUEST,
3365 );
3366 return Ok(vec![control_error_body_frame(
3367 &frame,
3368 ErrorBody {
3369 code: error_codes::INVALID_REQUEST.to_string(),
3370 message: error.to_string(),
3371 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3372 },
3373 )?]);
3374 }
3375
3376 let Some(registration) = self
3395 .registry
3396 .get_module(&target_module_id)
3397 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3398 else {
3399 if let Some((status, warming)) =
3400 self.supervisor_status(&target_module_id, frame.header.corr)?
3401 {
3402 if status.protocol == ModuleProtocol::None {
3409 return Ok(vec![self.route_open_refusal_frame(
3410 ctx,
3411 &frame,
3412 &target_module_id,
3413 "protocol_none",
3414 error_codes::MODULE_NO_PROTOCOL,
3415 format!(
3416 "module_id '{target_module_id}' is declared protocol: none; \
3417 it speaks no subc wire and serves no routes"
3418 ),
3419 )?]);
3420 }
3421 let code = if warming {
3422 "module_warming"
3423 } else {
3424 "target_unavailable"
3425 };
3426 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3427 ctx,
3428 &frame,
3429 &target_module_id,
3430 code,
3431 &status,
3432 )?]);
3433 }
3434 if let Some(removed_ago_ms) =
3435 self.supervisor.removal_tombstone_age_ms(&target_module_id)
3436 {
3437 return Ok(vec![self.route_open_refusal_frame(
3438 ctx,
3439 &frame,
3440 &target_module_id,
3441 "removed",
3442 error_codes::MODULE_REMOVED,
3443 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3444 )?]);
3445 }
3446 return Ok(vec![self.route_open_refusal_frame(
3447 ctx,
3448 &frame,
3449 &target_module_id,
3450 "not_registered",
3451 error_codes::UNKNOWN_MODULE,
3452 format!("module_id '{target_module_id}' is not registered"),
3453 )?]);
3454 };
3455
3456 if !registration.ready {
3460 self.counters
3461 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3462 info!(
3463 target: "control",
3464 code = error_codes::MODULE_WARMING,
3465 module_id = ?target_module_id,
3466 connection_id = ctx.connection_id.get(),
3467 reason = "declared_not_ready",
3468 "route.open refused"
3469 );
3470 self.route_outages
3473 .record_not_serving(&target_module_id, "declared_not_ready");
3474 return Ok(vec![control_error_body_frame(
3475 &frame,
3476 ErrorBody {
3477 code: error_codes::MODULE_WARMING.to_string(),
3478 message: format!(
3479 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3480 ),
3481 detail: Some(serde_json::json!({
3482 "reason": "declared_not_ready"
3483 })),
3484 },
3485 )?]);
3486 }
3487
3488 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3507 self.counters
3508 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3509 info!(
3510 target: "control",
3511 code = error_codes::MODULE_WARMING,
3512 module_id = ?target_module_id,
3513 connection_id = ctx.connection_id.get(),
3514 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3515 capability = %capability,
3516 "route.open refused"
3517 );
3518 return Ok(vec![control_error_body_frame(
3519 &frame,
3520 ErrorBody {
3521 code: error_codes::MODULE_WARMING.to_string(),
3522 message: format!(
3523 "module_id '{target_module_id}' requires capability '{capability}', \
3524 which no registered module provides; retry"
3525 ),
3526 detail: Some(serde_json::json!({
3527 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3528 "capability": capability,
3529 })),
3530 },
3531 )?]);
3532 }
3533
3534 if !target_has_required_role(&target, ®istration.manifest.provides) {
3535 return Ok(vec![self.route_open_refusal_frame(
3536 ctx,
3537 &frame,
3538 &target_module_id,
3539 "role_not_provided",
3540 "target_unavailable",
3541 format!("module_id '{target_module_id}' does not provide the requested target"),
3542 )?]);
3543 }
3544
3545 if registration.state != ChannelState::Active {
3546 return Ok(vec![self.route_open_refusal_frame(
3547 ctx,
3548 &frame,
3549 &target_module_id,
3550 "registration_not_active",
3551 "target_unavailable",
3552 format!("module_id '{target_module_id}' is not active"),
3553 )?]);
3554 }
3555
3556 if self
3557 .forwarding
3558 .module_is_draining(&target_module_id)
3559 .map_err(RouterError::Forwarding)?
3560 {
3561 return Ok(vec![self.route_open_refusal_frame(
3562 ctx,
3563 &frame,
3564 &target_module_id,
3565 "reloading",
3566 "module_reloading",
3567 format!("module_id '{target_module_id}' is reloading"),
3568 )?]);
3569 }
3570
3571 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3572 process_liveness.process_live(&target_module_id) == Some(false)
3573 }) {
3574 if process_liveness.process_replacing(&target_module_id) {
3581 return Ok(vec![self.route_open_refusal_frame(
3582 ctx,
3583 &frame,
3584 &target_module_id,
3585 "reloading",
3586 "module_reloading",
3587 format!("module_id '{target_module_id}' is reloading"),
3588 )?]);
3589 }
3590 return Ok(vec![self.route_open_refusal_frame(
3591 ctx,
3592 &frame,
3593 &target_module_id,
3594 "supervisor_not_live",
3595 "target_unavailable",
3596 format!("module_id '{target_module_id}' is not live"),
3597 )?]);
3598 }
3599
3600 if !self
3601 .forwarding
3602 .has_live_module_connection(&target_module_id)
3603 .map_err(RouterError::Forwarding)?
3604 {
3605 return Ok(vec![self.route_open_refusal_frame(
3606 ctx,
3607 &frame,
3608 &target_module_id,
3609 "no_forwarding_connection",
3610 "target_unavailable",
3611 format!("module_id '{target_module_id}' has no live forwarding connection"),
3612 )?]);
3613 }
3614
3615 if let Some(error) =
3616 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3617 {
3618 self.observe_route_open_refusal(
3619 ctx,
3620 &target_module_id,
3621 "op_not_allowed",
3622 "op_not_allowed",
3623 );
3624 return Ok(vec![error]);
3625 }
3626
3627 let principal = match self.route_open_principal(&frame, consumer_identity)? {
3628 Ok(principal) => principal,
3629 Err(error) => {
3630 self.observe_route_open_refusal(
3631 ctx,
3632 &target_module_id,
3633 "bad_consumer_identity",
3634 "bad_consumer_identity",
3635 );
3636 return Ok(vec![error]);
3637 }
3638 };
3639
3640 if let Principal::Reserved {
3644 module_id: opening_module_id,
3645 } = &principal
3646 {
3647 if let Some(opening_registration) = self
3648 .registry
3649 .get_module(opening_module_id)
3650 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3651 {
3652 if let Some(capability) =
3653 denied_capability(&opening_registration.manifest, ®istration.manifest)
3654 {
3655 warn!(
3656 opening_module_id,
3657 target_module_id,
3658 capability,
3659 "refusing route.open because an attested capability deny edge matches"
3660 );
3661 return Ok(vec![self.route_open_refusal_frame(
3662 ctx,
3663 &frame,
3664 &target_module_id,
3665 "capability_deny_edge",
3666 "capability_forbidden",
3667 format!(
3668 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3669 ),
3670 )?]);
3671 }
3672 }
3673 }
3674
3675 if admission_facts.is_some() {
3676 let carrier_matches = matches!(
3677 &principal,
3678 Principal::Reserved { module_id }
3679 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3680 );
3681 if !carrier_matches {
3682 return Ok(vec![self.route_open_refusal_frame(
3683 ctx,
3684 &frame,
3685 &target_module_id,
3686 "admission_facts_carrier_not_permitted",
3687 "admission_facts_not_permitted",
3688 "admission facts may only be carried by the configured reserved module",
3689 )?]);
3690 }
3691
3692 let target_allowed = self
3693 .admission_facts_targets
3694 .as_ref()
3695 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3696 if !target_allowed {
3697 return Ok(vec![self.route_open_refusal_frame(
3698 ctx,
3699 &frame,
3700 &target_module_id,
3701 "admission_facts_target_not_listed",
3702 "admission_facts_target_not_allowed",
3703 format!(
3704 "admission facts are not permitted for target module_id '{target_module_id}'"
3705 ),
3706 )?]);
3707 }
3708
3709 }
3713
3714 let (bound_scope, scope_stamp) = match scope {
3720 None => (None, None),
3721 Some(selector) => {
3722 let owner_configured = match &selector.owner {
3723 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3729 _ => false,
3730 };
3731 let admitted = self
3732 .scopes
3733 .read()
3734 .unwrap_or_else(|poisoned| poisoned.into_inner())
3735 .admit(&principal, &target_module_id, &selector, owner_configured)
3736 .and_then(|admission| {
3737 crate::scopes::check_target_flow_support(
3738 &admission.stamp,
3739 &target_module_id,
3740 registration.manifest.capabilities.as_ref(),
3741 )?;
3742 crate::scopes::check_target_agent_run_support(
3743 &admission.stamp,
3744 &target_module_id,
3745 registration.manifest.capabilities.as_ref(),
3746 )?;
3747 Ok(admission)
3748 });
3749 match admitted {
3750 Ok(admission) => (
3751 Some(BoundScope {
3752 owner: admission.owner,
3753 scope_ref: admission.stamp.scope_ref.clone(),
3754 tag: admission.tag,
3755 }),
3756 Some(admission.stamp),
3757 ),
3758 Err(refusal) => {
3759 return Ok(vec![self.route_open_refusal_frame(
3760 ctx,
3761 &frame,
3762 &target_module_id,
3763 refusal.code,
3764 refusal.code,
3765 refusal.message,
3766 )?]);
3767 }
3768 }
3769 }
3770 };
3771
3772 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3794 Ok(project_root) => project_root,
3795 Err(err) => {
3796 return Ok(vec![control_error_frame(
3797 &frame,
3798 "invalid_project_root",
3799 err.to_string(),
3800 )?])
3801 }
3802 };
3803 identity.project_root = project_root.as_path().to_path_buf();
3804
3805 let _concurrency_guard = match self
3816 .route_bind_concurrency
3817 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3818 {
3819 Ok(guard) => guard,
3820 Err(in_flight) => {
3821 return Ok(vec![self.route_open_target_capacity_refusal(
3822 ctx,
3823 &frame,
3824 &target_module_id,
3825 in_flight,
3826 )?]);
3827 }
3828 };
3829
3830 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3833 RouteBindAdmission::Admitted { guard, probe } => {
3834 if probe {
3835 info!(
3836 module_id = %target_module_id,
3837 connection_id = ctx.connection_id.get(),
3838 "route.bind breaker half-open: admitting one probe"
3839 );
3840 }
3841 guard
3842 }
3843 RouteBindAdmission::Refused {
3844 consecutive_timeouts,
3845 retry_in,
3846 probe_in_flight,
3847 } => {
3848 return Ok(vec![self.route_open_breaker_refusal_frame(
3849 ctx,
3850 &frame,
3851 &target_module_id,
3852 consecutive_timeouts,
3853 retry_in,
3854 probe_in_flight,
3855 )?]);
3856 }
3857 };
3858
3859 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3864 let relay_deadline = Instant::now() + route_bind_relay_timeout;
3865 let pending = match self
3866 .forwarding
3867 .begin_route_bind_relay_for(
3868 ctx.connection_id,
3869 ctx.egress.clone(),
3870 response_version(&frame),
3871 frame.header.corr,
3872 &target_module_id,
3873 principal.clone(),
3874 bound_scope,
3875 Some(project_root),
3876 relay_deadline,
3877 )
3878 .await
3879 {
3880 Ok(pending) => pending,
3881 Err(err) => {
3882 return Ok(vec![self.route_open_refusal_frame(
3883 ctx,
3884 &frame,
3885 &target_module_id,
3886 "relay_reservation_failed",
3887 forwarding_error_code(&err),
3888 err.to_string(),
3889 )?])
3890 }
3891 };
3892 let crate::forwarding::PendingRouteBindRelay {
3893 endpoint,
3894 module_sink,
3895 negotiated_ver,
3896 client_channel,
3897 client_epoch,
3898 module_channel,
3899 module_epoch,
3900 corr: relay_corr,
3901 receiver,
3902 } = pending;
3903 let mut reservation =
3904 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3905
3906 if let Some(stamp) = scope_stamp
3911 .as_ref()
3912 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
3913 {
3914 let relay_registration = self
3915 .registry
3916 .get_module_by_connection(endpoint.connection_id)
3917 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3918 if let Err(refusal) = crate::scopes::check_target_flow_support(
3919 stamp,
3920 &target_module_id,
3921 relay_registration
3922 .as_ref()
3923 .and_then(|registration| registration.manifest.capabilities.as_ref()),
3924 )
3925 .and_then(|()| {
3926 crate::scopes::check_target_agent_run_support(
3927 stamp,
3928 &target_module_id,
3929 relay_registration
3930 .as_ref()
3931 .and_then(|registration| registration.manifest.capabilities.as_ref()),
3932 )
3933 }) {
3934 reservation.release_and_disarm();
3935 return Ok(vec![self.route_open_refusal_frame(
3936 ctx,
3937 &frame,
3938 &target_module_id,
3939 refusal.code,
3940 refusal.code,
3941 refusal.message,
3942 )?]);
3943 }
3944 }
3945
3946 debug!(
3947 connection_id = ctx.connection_id.get(),
3948 client_channel,
3949 client_epoch,
3950 module_channel,
3951 module_epoch,
3952 "reserved route handle pair"
3953 );
3954 let principal_label = principal_label(&principal);
3957 let relay = ModuleControlRequest::RouteBind {
3958 route_channel: module_channel,
3959 epoch: module_epoch,
3960 target,
3961 identity,
3962 principal: Some(principal),
3963 consumer_capabilities,
3964 role_versions,
3965 admission_facts,
3966 scope: scope_stamp,
3967 };
3968 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3969 RouterError::backend(
3970 0,
3971 frame.header.corr,
3972 format!("failed to encode route.bind request: {err}"),
3973 )
3974 })?;
3975 let relay_frame = Frame::build_with_version(
3976 negotiated_ver,
3977 FrameType::Request,
3978 control_flags(),
3979 0,
3980 0,
3981 relay_corr,
3982 relay_body,
3983 )
3984 .map_err(RouterError::FrameBuild)?;
3985
3986 if let Err(err) = module_sink.send(relay_frame).await {
3987 reservation.release_and_disarm();
3988 return Ok(vec![self.route_open_refusal_frame(
3989 ctx,
3990 &frame,
3991 &target_module_id,
3992 "relay_send_failed",
3993 "target_unavailable",
3994 err.to_string(),
3995 )?]);
3996 }
3997
3998 #[cfg(test)]
3999 if let Some(hold) = &self.route_bind_relay_sent_hold {
4000 hold.wait().await;
4004 hold.wait().await;
4005 }
4006
4007 if self
4012 .forwarding
4013 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4014 .map_err(RouterError::Forwarding)?
4015 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4016 {
4017 self.send_abandoned_route_bind_goodbye(
4018 &module_sink,
4019 negotiated_ver,
4020 module_channel,
4021 module_epoch,
4022 );
4023 }
4024
4025 match timeout_at(relay_deadline, receiver).await {
4026 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4027 reservation.disarm();
4028 if breaker.record_accepted() {
4029 info!(
4030 module_id = %target_module_id,
4031 "route.bind breaker closed: the probe was accepted"
4032 );
4033 }
4034 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4035 Ok(Vec::new())
4036 }
4037 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4038 reservation.release_and_disarm();
4039 breaker.record_inconclusive();
4043 let scope_code = match body.code.as_str() {
4047 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4048 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4049 _ => None,
4050 };
4051 if let Some(code) = scope_code {
4052 self.observe_route_open_refusal(
4053 ctx,
4054 &target_module_id,
4055 "scope_changed_before_commit",
4056 code,
4057 );
4058 return Ok(vec![control_error_body_frame(&frame, body)?]);
4059 }
4060 self.counters
4061 .increment_route_open_refused("module_rejected");
4062 info!(
4063 target: "control",
4064 code = "module_rejected",
4065 module_code = ?body.code,
4066 module_id = ?target_module_id,
4067 connection_id = ctx.connection_id.get(),
4068 "route.open refused"
4069 );
4070 Ok(vec![control_error_body_frame(&frame, body)?])
4071 }
4072 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4073 reservation.release_and_disarm();
4074 breaker.record_inconclusive();
4075 tracing::warn!(
4082 module_id = %target_module_id,
4083 "route.bind relay abandoned: {message}"
4084 );
4085 Ok(vec![self.route_open_refusal_frame(
4086 ctx,
4087 &frame,
4088 &target_module_id,
4089 "relay_abandoned",
4090 "target_unavailable",
4091 message,
4092 )?])
4093 }
4094 Ok(Err(_)) => {
4095 reservation.release_and_disarm();
4096 breaker.record_inconclusive();
4097 Ok(vec![self.route_open_refusal_frame(
4098 ctx,
4099 &frame,
4100 &target_module_id,
4101 "relay_waiter_canceled",
4102 "target_unavailable",
4103 "route.bind relay waiter was canceled before the module responded",
4104 )?])
4105 }
4106 Err(_) => {
4107 reservation.release_and_disarm();
4108 if let Some(opened) = breaker.record_timeout(
4112 self.route_bind_breaker_threshold,
4113 self.route_bind_breaker_cooldown,
4114 ) {
4115 warn!(
4116 module_id = %target_module_id,
4117 consecutive_timeouts = opened.consecutive_timeouts,
4118 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4119 reopened_after_probe = opened.reopened_after_probe,
4120 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4121 );
4122 }
4123 tracing::warn!(
4131 module_id = %target_module_id,
4132 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4133 "route.bind relay timed out: module did not ack within budget"
4134 );
4135 Ok(vec![self.route_open_refusal_frame(
4136 ctx,
4137 &frame,
4138 &target_module_id,
4139 "relay_timed_out",
4140 "module_timeout",
4141 format!(
4142 "module_id '{target_module_id}' did not answer route.bind within {:?}",
4143 route_bind_relay_timeout
4144 ),
4145 )?])
4146 }
4147 }
4148 }
4149
4150 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4151 let response = ClientControlResponse::SupervisorSpawnSnapshot {
4152 snapshot: self.supervisor.spawn_snapshot(),
4153 };
4154 Ok(vec![control_response_body_frame(
4155 &frame,
4156 &response,
4157 "ClientControlResponse::SupervisorSpawnSnapshot",
4158 )?])
4159 }
4160
4161 fn handle_supervisor_spawn_subscribe(
4162 &self,
4163 ctx: &RouteCtx,
4164 frame: Frame,
4165 since: Option<SpawnCursor>,
4166 ) -> Result<Vec<Frame>, RouterError> {
4167 match self.supervisor.subscribe_spawns(
4168 ctx.connection_id,
4169 frame.header.corr,
4170 response_version(&frame),
4171 since,
4172 ctx.egress.clone(),
4173 ) {
4174 Ok(()) => Ok(Vec::new()),
4175 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4176 Ok(vec![control_error_body_frame(
4177 &frame,
4178 ErrorBody {
4179 code: "spawn_cursor_incarnation_mismatch".to_string(),
4180 message: "spawn cursor belongs to a different daemon incarnation"
4181 .to_string(),
4182 detail: Some(serde_json::json!({
4183 "current_daemon_incarnation": current
4184 })),
4185 },
4186 )?])
4187 }
4188 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4189 &frame,
4190 ErrorBody {
4191 code: "spawn_cursor_too_old".to_string(),
4192 message: "spawn cursor predates the retained event ring".to_string(),
4193 detail: Some(serde_json::json!({
4194 "oldest_retained_cursor": oldest
4195 })),
4196 },
4197 )?]),
4198 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4199 0,
4200 frame.header.corr,
4201 format!("failed to open supervisor spawn subscription: {error}"),
4202 )),
4203 }
4204 }
4205
4206 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4207 let generation = self
4208 .registry
4209 .generation()
4210 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4211 let mut modules = Vec::new();
4212 for module in self.supervisor.list() {
4213 let status = module.status_for_control("list").map_err(|err| {
4214 RouterError::backend(
4215 0,
4216 frame.header.corr,
4217 format!("failed to read supervisor status: {err}"),
4218 )
4219 })?;
4220 let (configured, _) = module.configuration().map_err(|err| {
4221 RouterError::backend(
4222 0,
4223 frame.header.corr,
4224 format!("failed to read module configuration: {err}"),
4225 )
4226 })?;
4227 let image = module.running_image_agreement().await;
4229 let resources = Some(module.child_resource_usage());
4232 let pending_reload = Some(reload_verdict(
4233 &configured.program,
4234 status.spawned_from.as_deref(),
4235 image,
4236 ));
4237 modules.push(
4240 SupervisorEntry::new(
4241 status.module_id,
4242 status.state.to_string(),
4243 status.enabled,
4244 status.live,
4245 status.health.status,
4246 )
4247 .with_launch_nonce_env(Some(!cfg!(unix)))
4248 .with_protocol(status.protocol)
4249 .with_pending_reload(pending_reload)
4250 .with_last_probe_ms(status.health.last_probe_ms)
4251 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
4252 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
4253 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
4254 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
4255 .with_restart_count(Some(status.restart_count))
4256 .with_max_restarts(Some(status.max_restarts))
4257 .with_lifetime_restarts(Some(status.lifetime_restarts))
4258 .with_spawn_generation(Some(status.spawn_generation))
4259 .with_restart_window_secs(Some(status.restart_window.as_secs()))
4260 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
4261 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
4262 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
4263 .with_resources(resources),
4264 );
4265 }
4266 let response = ClientControlResponse::SupervisorList {
4267 generation,
4268 modules,
4269 };
4270 Ok(vec![control_response_body_frame(
4271 &frame,
4272 &response,
4273 "ClientControlResponse::SupervisorList",
4274 )?])
4275 }
4276
4277 fn handle_supervisor_stderr_tail(
4278 &self,
4279 frame: Frame,
4280 module_id: String,
4281 max_lines: Option<u32>,
4282 max_bytes: Option<u32>,
4283 ) -> Result<Vec<Frame>, RouterError> {
4284 let Some(module) = self.supervisor.get(&module_id) else {
4285 return Ok(vec![control_error_frame(
4286 &frame,
4287 "unknown_module",
4288 format!("module_id '{module_id}' is not supervised"),
4289 )?]);
4290 };
4291
4292 let snapshot = module.stderr_tail(
4293 max_lines.map(|value| value as usize),
4294 max_bytes.map(|value| value as usize),
4295 );
4296
4297 let response = ClientControlResponse::SupervisorStderrTail {
4298 module_id,
4299 tail: StderrTail {
4300 capture: match snapshot.capture {
4301 CaptureState::Captured => StderrCaptureState::Captured,
4302 CaptureState::Incomplete { reason } => {
4303 StderrCaptureState::Incomplete { reason }
4304 }
4305 CaptureState::NotCaptured { reason } => {
4306 StderrCaptureState::NotCaptured { reason }
4307 }
4308 },
4309 entries: snapshot
4310 .entries
4311 .into_iter()
4312 .map(|entry| match entry {
4313 TailEntry::Line {
4314 text,
4315 truncated,
4316 at_ms,
4317 } => StderrTailEntry::Line {
4318 text,
4319 truncated,
4320 at_ms,
4321 },
4322 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
4323 })
4324 .collect(),
4325 dropped_lines: snapshot.dropped_lines,
4326 },
4327 };
4328 Ok(vec![control_response_body_frame(
4329 &frame,
4330 &response,
4331 "ClientControlResponse::SupervisorStderrTail",
4332 )?])
4333 }
4334
4335 async fn handle_supervisor_terminals(
4336 &self,
4337 frame: Frame,
4338 module_id: String,
4339 ) -> Result<Vec<Frame>, RouterError> {
4340 let Some(module) = self.supervisor.get(&module_id) else {
4341 return Ok(vec![control_error_frame(
4342 &frame,
4343 "unknown_module",
4344 format!("module_id '{module_id}' is not supervised"),
4345 )?]);
4346 };
4347
4348 let terminals = module
4351 .read_durable_terminal_history()
4352 .await
4353 .map_err(|error| {
4354 RouterError::backend(
4355 0,
4356 frame.header.corr,
4357 format!("failed to read terminal history: {error}"),
4358 )
4359 })?;
4360 let response = ClientControlResponse::SupervisorTerminals {
4361 module_id,
4362 terminals,
4363 };
4364 Ok(vec![control_response_body_frame(
4365 &frame,
4366 &response,
4367 "ClientControlResponse::SupervisorTerminals",
4368 )?])
4369 }
4370
4371 fn handle_supervisor_routes(
4372 &self,
4373 frame: Frame,
4374 module_id: Option<String>,
4375 ) -> Result<Vec<Frame>, RouterError> {
4376 let modules = self
4377 .forwarding
4378 .route_census(module_id.as_deref())
4379 .map_err(RouterError::Forwarding)?
4380 .into_iter()
4381 .map(|(module_id, routes)| SupervisorRouteModule {
4382 module_id,
4383 routes: routes
4384 .into_iter()
4385 .map(|route| SupervisorRoute {
4386 consumer: match route.principal {
4387 Principal::Reserved { module_id } => {
4388 SupervisorRouteConsumer::Reserved { module_id }
4389 }
4390 Principal::Direct | Principal::Unverified => {
4391 SupervisorRouteConsumer::Direct {
4392 connection_id: route.goodbye_target.connection_id.get(),
4393 }
4394 }
4395 },
4396 age_ms: Instant::now()
4397 .saturating_duration_since(route.bound_at)
4398 .as_millis()
4399 .try_into()
4400 .unwrap_or(u64::MAX),
4401 draining: route.draining,
4402 drain_reason: route.drain_reason,
4403 })
4404 .collect(),
4405 })
4406 .collect();
4407 let response = ClientControlResponse::SupervisorRoutes { modules };
4408 Ok(vec![control_response_body_frame(
4409 &frame,
4410 &response,
4411 "ClientControlResponse::SupervisorRoutes",
4412 )?])
4413 }
4414
4415 async fn handle_supervisor_provenance(
4416 &self,
4417 frame: Frame,
4418 module_id: Option<String>,
4419 ) -> Result<Vec<Frame>, RouterError> {
4420 let mut selected = if let Some(module_id) = module_id {
4421 let Some(module) = self.supervisor.get(&module_id) else {
4422 return Ok(vec![control_error_frame(
4423 &frame,
4424 "unknown_module",
4425 format!("module_id '{module_id}' is not supervised"),
4426 )?]);
4427 };
4428 vec![module]
4429 } else {
4430 self.supervisor.list()
4431 };
4432
4433 let mut modules = Vec::with_capacity(selected.len());
4434 for module in selected.drain(..) {
4435 let (status, observed_image) = module
4436 .status_and_running_image_agreement()
4437 .await
4438 .map_err(|err| {
4439 RouterError::backend(
4440 0,
4441 frame.header.corr,
4442 format!("failed to read supervisor status: {err}"),
4443 )
4444 })?;
4445 let module_declared = self
4446 .registry
4447 .get_module(&status.module_id)
4448 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4449 .and_then(|registration| registration.manifest.provenance)
4450 .map(|build| ModuleDeclaredProvenance::Reported { build })
4451 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4452 #[cfg(test)]
4453 let running_image = match &self.provenance_probe_override {
4454 Some(result) => result.clone(),
4455 None => observed_image,
4456 };
4457 #[cfg(not(test))]
4458 let running_image = observed_image;
4459 modules.push(SupervisorModuleProvenance {
4460 module_id: status.module_id,
4461 module_declared,
4462 daemon_observed: SupervisorObservedProcess {
4463 pid: status.pid,
4464 spawned_at_ms: status.spawned_at_ms,
4465 spawned_from: status.spawned_from,
4466 running_image,
4467 },
4468 });
4469 }
4470 let daemon = SupervisorDaemonProvenance {
4471 daemon_build: self.daemon_provenance.build.clone(),
4472 daemon_observed: DaemonObservedProcess {
4473 pid: self.daemon_provenance.pid,
4474 started_at_ms: self
4475 .daemon_provenance
4476 .start_clock
4477 .map(|clock| clock.started_at_ms())
4478 .or(self.daemon_provenance.started_at_ms),
4479 running_image: self
4480 .daemon_provenance
4481 .probe
4482 .observe(
4483 self.daemon_provenance.pid,
4484 self.daemon_provenance.executable_path.as_deref(),
4485 self.daemon_provenance.executable_identity,
4486 self.daemon_provenance.process_start_time,
4487 )
4488 .await,
4489 },
4490 };
4491 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
4492 Ok(vec![control_response_body_frame(
4493 &frame,
4494 &response,
4495 "ClientControlResponse::SupervisorProvenance",
4496 )?])
4497 }
4498
4499 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4500 self.refresh_capability_requirements();
4501 let generation = self
4502 .registry
4503 .generation()
4504 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4505 let modules = self
4506 .supervisor
4507 .list()
4508 .into_iter()
4509 .map(|module| {
4510 let status = module.status_for_control("health").map_err(|err| {
4511 RouterError::backend(
4512 0,
4513 frame.header.corr,
4514 format!("failed to read supervisor health: {err}"),
4515 )
4516 })?;
4517 let module_id = status.module_id;
4518 let capability_detail = self
4519 .capability_evaluator
4520 .required_problem_detail(&module_id);
4521 Ok(SupervisorHealthEntry {
4522 module_id,
4523 status: status.health.status,
4524 detail: append_capability_problem_detail(
4525 status.health.detail,
4526 capability_detail,
4527 ),
4528 metrics: status.health.metrics,
4529 consecutive_failures: status.health.consecutive_failures,
4530 late_answer_count: status.health.late_answer_count,
4531 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4532 last_action: status.health.last_action,
4533 last_action_ms: status.health.last_action_ms,
4534 last_probe_ms: status.health.last_probe_ms,
4535 })
4536 })
4537 .collect::<Result<Vec<_>, RouterError>>()?;
4538 let response = ClientControlResponse::SupervisorHealth {
4539 generation,
4540 modules,
4541 };
4542 Ok(vec![control_response_body_frame(
4543 &frame,
4544 &response,
4545 "ClientControlResponse::SupervisorHealth",
4546 )?])
4547 }
4548
4549 async fn handle_supervisor_restart(
4550 &self,
4551 frame: Frame,
4552 module_id: String,
4553 drain_timeout_ms: Option<u64>,
4554 ) -> Result<Vec<Frame>, RouterError> {
4555 let operation_lock = self.supervisor.operation_lock();
4556 let _operation_guard = operation_lock.lock().await;
4557 let Some(module) = self.supervisor.get(&module_id) else {
4558 return Ok(vec![control_error_frame(
4559 &frame,
4560 "unknown_module",
4561 format!("module_id '{module_id}' is not supervised"),
4562 )?]);
4563 };
4564
4565 self.route_outages.mark_operator_action(&module_id);
4566 if let Err(err) = module.restart(drain_timeout_ms).await {
4567 self.route_outages
4568 .operator_action_ended_unrefused(&module_id);
4569 let (code, message) = match err {
4570 crate::supervise::SuperviseError::Disabled { .. } => {
4571 ("module_disabled", err.to_string())
4572 }
4573 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4574 ("swap_in_progress", err.to_string())
4575 }
4576 _ => (
4577 "target_unavailable",
4578 format!("failed to restart module_id '{module_id}': {err}"),
4579 ),
4580 };
4581 return Ok(vec![control_error_frame(&frame, code, message)?]);
4582 }
4583
4584 let response = ClientControlResponse::SupervisorAck {
4585 module_id,
4586 applied: true,
4587 };
4588 Ok(vec![control_response_body_frame(
4589 &frame,
4590 &response,
4591 "ClientControlResponse::SupervisorAck",
4592 )?])
4593 }
4594
4595 async fn handle_supervisor_swap(
4599 &self,
4600 frame: Frame,
4601 module_id: String,
4602 ready_timeout_ms: Option<u64>,
4603 ) -> Result<Vec<Frame>, RouterError> {
4604 let module = {
4611 let operation_lock = self.supervisor.operation_lock();
4612 let _operation_guard = operation_lock.lock().await;
4613 self.supervisor.get(&module_id)
4614 };
4615 let Some(module) = module else {
4616 return Ok(vec![control_error_frame(
4617 &frame,
4618 "unknown_module",
4619 format!("module_id '{module_id}' is not supervised"),
4620 )?]);
4621 };
4622
4623 self.route_outages.mark_operator_action(&module_id);
4624 if let Err(err) = module
4625 .swap(ready_timeout_ms.map(Duration::from_millis))
4626 .await
4627 {
4628 self.route_outages
4629 .operator_action_ended_unrefused(&module_id);
4630 use crate::supervise::SuperviseError;
4631 let message = err.to_string();
4632 let error = match err {
4633 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4634 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4635 code: "swap_refused".to_string(),
4636 message,
4637 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4638 },
4639 SuperviseError::SwapFailed {
4640 arm,
4641 candidate_exit,
4642 ..
4643 } => ErrorBody {
4644 code: "swap_failed".to_string(),
4645 message,
4646 detail: Some(serde_json::json!({
4647 "arm": arm.as_str(),
4648 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4649 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4650 })),
4651 },
4652 _ => ErrorBody::new(
4653 "target_unavailable",
4654 format!("failed to swap module_id '{module_id}': {message}"),
4655 ),
4656 };
4657 return Ok(vec![control_error_body_frame(&frame, error)?]);
4658 }
4659 self.route_outages
4663 .operator_action_ended_unrefused(&module_id);
4664
4665 let response = ClientControlResponse::SupervisorAck {
4666 module_id,
4667 applied: true,
4668 };
4669 Ok(vec![control_response_body_frame(
4670 &frame,
4671 &response,
4672 "ClientControlResponse::SupervisorAck",
4673 )?])
4674 }
4675
4676 async fn handle_supervisor_reload(
4677 &self,
4678 frame: Frame,
4679 module_id: String,
4680 ) -> Result<Vec<Frame>, RouterError> {
4681 let operation_lock = self.supervisor.operation_lock();
4682 let _operation_guard = operation_lock.lock().await;
4683 let Some(module) = self.supervisor.get(&module_id) else {
4684 return Ok(vec![control_error_frame(
4685 &frame,
4686 "unknown_module",
4687 format!("module_id '{module_id}' is not supervised"),
4688 )?]);
4689 };
4690
4691 self.route_outages.mark_operator_action(&module_id);
4692 if let Err(err) = module.reload().await {
4693 self.route_outages
4694 .operator_action_ended_unrefused(&module_id);
4695 let (code, message) = match err {
4696 crate::supervise::SuperviseError::Disabled { .. } => {
4697 ("module_disabled", err.to_string())
4698 }
4699 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4700 ("swap_in_progress", err.to_string())
4701 }
4702 _ => (
4703 "reload_failed",
4704 format!("failed to reload module_id '{module_id}': {err}"),
4705 ),
4706 };
4707 return Ok(vec![control_error_frame(&frame, code, message)?]);
4708 }
4709
4710 let response = ClientControlResponse::SupervisorAck {
4711 module_id,
4712 applied: true,
4713 };
4714 Ok(vec![control_response_body_frame(
4715 &frame,
4716 &response,
4717 "ClientControlResponse::SupervisorAck",
4718 )?])
4719 }
4720
4721 async fn handle_supervisor_rescan(
4722 &self,
4723 frame: Frame,
4724 preview: bool,
4725 ) -> Result<Vec<Frame>, RouterError> {
4726 let Some(context) = self.rescan.clone() else {
4727 return Ok(vec![control_error_frame(
4728 &frame,
4729 "rescan_unavailable",
4730 "the daemon was not started with a reloadable config path".to_string(),
4731 )?]);
4732 };
4733
4734 let operation_lock = self.supervisor.operation_lock();
4735 let _operation_guard = operation_lock.lock().await;
4736 let loaded = match crate::daemon_config::load(&context.config_path) {
4737 Ok(config) => config,
4738 Err(err) => {
4739 return Ok(vec![control_error_frame(
4740 &frame,
4741 "invalid_daemon_config",
4742 format!("supervisor rescan rejected daemon config: {err}"),
4743 )?])
4744 }
4745 };
4746 let Some(config) = loaded else {
4754 return Ok(vec![control_error_frame(
4755 &frame,
4756 "invalid_daemon_config",
4757 format!(
4758 "daemon config not found at {}; refusing to rescan (an absent config would \
4759 retire every supervised module)",
4760 context.config_path.display()
4761 ),
4762 )?]);
4763 };
4764 let (
4765 configured_port,
4766 storage_config,
4767 admission_facts_carrier_module_id,
4768 admission_facts_targets,
4769 scope_authority_owners,
4770 modules,
4771 reserved_capabilities,
4772 ) = (
4773 config.port,
4774 config.storage,
4775 config.admission_facts_carrier_module_id,
4776 config.admission_facts_targets,
4777 config.scope_authority_owners,
4778 config.modules,
4779 config.reserved_capabilities,
4780 );
4781
4782 let mut restart_required = Vec::new();
4791 for section in RestartRequiredSection::ALL {
4792 let changed = match section {
4793 RestartRequiredSection::Port => configured_port != context.configured_port,
4794 RestartRequiredSection::Storage => storage_config != context.storage_config,
4795 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4796 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4797 }
4798 RestartRequiredSection::AdmissionFactsTargets => {
4799 admission_facts_targets != context.admission_facts_targets
4800 }
4801 RestartRequiredSection::ScopeAuthorityOwners => {
4802 scope_authority_owners != context.scope_authority_owners
4803 }
4804 };
4805 if changed {
4806 restart_required.push(section.label().to_string());
4807 }
4808 }
4809 if !restart_required.is_empty() {
4810 warn!(
4811 config_path = %context.config_path.display(),
4812 sections = %restart_required.join(", "),
4813 "daemon config changed outside the modules section; restart the daemon to apply those changes"
4814 );
4815 }
4816
4817 for configured in &modules {
4818 if let Err(err) = validate_spec(&configured.module_spec()) {
4819 return Ok(vec![control_error_frame(
4820 &frame,
4821 "invalid_daemon_config",
4822 format!("supervisor rescan rejected daemon config: {err}"),
4823 )?]);
4824 }
4825 }
4826
4827 let configured_capabilities = modules
4828 .iter()
4829 .map(|module| (module.module_id.clone(), module.enabled))
4830 .collect::<Vec<_>>();
4831 let preview_capability_warnings = if preview {
4832 let (_, registrations) = self.runtime_capability_snapshot()?;
4833 let current_modules = self
4834 .supervisor
4835 .list()
4836 .into_iter()
4837 .map(|module| module.module_id().to_string())
4838 .collect::<BTreeSet<_>>();
4839 let resulting_modules = configured_capabilities.clone();
4840 let removed = current_modules
4841 .into_iter()
4842 .filter(|module_id| {
4843 !resulting_modules
4844 .iter()
4845 .any(|(configured_id, _)| configured_id == module_id)
4846 })
4847 .collect::<Vec<_>>();
4848 self.capability_evaluator.preview_removal_warnings(
4849 resulting_modules,
4850 &removed,
4851 ®istrations,
4852 )
4853 } else {
4854 Vec::new()
4855 };
4856 let result = match self
4857 .reconcile_supervised_modules(&context.supervisor, modules, preview)
4858 .await
4859 {
4860 Ok(result) => result,
4861 Err(message) => {
4862 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4863 }
4864 };
4865 if !preview {
4866 self.capability_evaluator
4867 .configure(configured_capabilities, reserved_capabilities);
4868 self.capability_evaluator.wake_deadline_loop();
4869 self.refresh_capability_requirements();
4870 }
4871 let mut result = result;
4872 result.restart_required = restart_required;
4873 result.capability_warnings = preview_capability_warnings;
4874 let response = ClientControlResponse::SupervisorRescan { result };
4875 Ok(vec![control_response_body_frame(
4876 &frame,
4877 &response,
4878 "ClientControlResponse::SupervisorRescan",
4879 )?])
4880 }
4881
4882 async fn handle_supervisor_release_reserved(
4883 &self,
4884 frame: Frame,
4885 module_id: String,
4886 ) -> Result<Vec<Frame>, RouterError> {
4887 let Some(context) = self.rescan.clone() else {
4888 return Ok(vec![control_error_frame(
4889 &frame,
4890 "release_unavailable",
4891 "reserved-id release requires a daemon started with a reloadable config path",
4892 )?]);
4893 };
4894 let operation_lock = self.supervisor.operation_lock();
4895 let _operation_guard = operation_lock.lock().await;
4896 let loaded = match crate::daemon_config::load(&context.config_path) {
4897 Ok(Some(config)) => config,
4898 Ok(None) => {
4899 return Ok(vec![control_error_frame(
4900 &frame,
4901 "invalid_daemon_config",
4902 format!(
4903 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4904 context.config_path.display()
4905 ),
4906 )?])
4907 }
4908 Err(err) => {
4909 return Ok(vec![control_error_frame(
4910 &frame,
4911 "invalid_daemon_config",
4912 format!("unable to verify reserved-id release against daemon config: {err}"),
4913 )?])
4914 }
4915 };
4916 if loaded
4917 .modules
4918 .iter()
4919 .any(|configured| configured.module_id == module_id)
4920 {
4921 return Ok(vec![control_error_frame(
4922 &frame,
4923 "reserved_module_configured",
4924 format!(
4925 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4926 ),
4927 )?]);
4928 }
4929 if !self.supervisor.release_retained_reserved_gate(&module_id) {
4930 return Ok(vec![control_error_frame(
4931 &frame,
4932 "reserved_gate_not_retained",
4933 format!(
4934 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4935 ),
4936 )?]);
4937 }
4938
4939 let response = ClientControlResponse::SupervisorAck {
4940 module_id,
4941 applied: true,
4942 };
4943 Ok(vec![control_response_body_frame(
4944 &frame,
4945 &response,
4946 "ClientControlResponse::SupervisorAck",
4947 )?])
4948 }
4949
4950 async fn reconcile_supervised_modules(
4959 &self,
4960 supervisor: &Supervisor,
4961 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4962 preview: bool,
4963 ) -> Result<SupervisorRescanResult, String> {
4964 let mut current = BTreeMap::new();
4965 for module in self.supervisor.list() {
4966 let (spec, health) = module.configuration().map_err(|err| {
4967 format!(
4968 "failed to read configuration for module_id '{}': {err}",
4969 module.module_id()
4970 )
4971 })?;
4972 let enabled = module
4973 .status()
4974 .map_err(|err| {
4975 format!(
4976 "failed to read status for module_id '{}': {err}",
4977 module.module_id()
4978 )
4979 })?
4980 .enabled;
4981 current.insert(
4982 module.module_id().to_string(),
4983 (module, spec, health, enabled),
4984 );
4985 }
4986 let configured = configured_modules
4987 .into_iter()
4988 .map(|module| (module.module_id.clone(), module))
4989 .collect::<BTreeMap<_, _>>();
4990
4991 let added = configured
4992 .keys()
4993 .filter(|module_id| !current.contains_key(*module_id))
4994 .cloned()
4995 .collect::<Vec<_>>();
4996 let removed = current
4997 .keys()
4998 .filter(|module_id| !configured.contains_key(*module_id))
4999 .cloned()
5000 .collect::<Vec<_>>();
5001 let mut changed_pending_reload = Vec::new();
5002 let mut configuration_changes = BTreeSet::new();
5003 let mut enabled_changes = BTreeSet::new();
5004 let mut unchanged = 0_u32;
5005
5006 for (module_id, configured_module) in &configured {
5007 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5008 else {
5009 continue;
5010 };
5011 let launch_changed = *current_spec != configured_module.module_spec();
5015 let configuration_changed =
5016 launch_changed || *current_health != configured_module.health;
5017 let enabled_changed = *current_enabled != configured_module.enabled;
5018 if configuration_changed {
5019 configuration_changes.insert(module_id.clone());
5020 }
5021 if launch_changed {
5022 changed_pending_reload.push(module_id.clone());
5023 }
5024 if enabled_changed {
5025 enabled_changes.insert(module_id.clone());
5026 }
5027 if !configuration_changed && !enabled_changed {
5028 unchanged = unchanged.saturating_add(1);
5029 }
5030 }
5031
5032 if preview {
5038 return Ok(SupervisorRescanResult {
5039 added,
5040 removed,
5041 changed_pending_reload,
5042 enabled_changes: enabled_changes.iter().cloned().collect(),
5043 unchanged,
5044 preview: true,
5045 restart_required: Vec::new(),
5049 capability_warnings: Vec::new(),
5050 });
5051 }
5052
5053 for module_id in &removed {
5054 let module = ¤t
5055 .get(module_id)
5056 .expect("removed module came from current supervisor state")
5057 .0;
5058 module.retire().await.map_err(|err| {
5059 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5060 })?;
5061 self.supervisor.record_rescan_removal(module_id);
5095 self.supervisor.retire(module_id);
5096 self.route_outages.forget(module_id);
5097 }
5098
5099 for module_id in configured.keys() {
5100 let Some((module, _, _, _)) = current.get(module_id) else {
5101 continue;
5102 };
5103 let configured_module = configured
5104 .get(module_id)
5105 .expect("configured module id came from configured map");
5106 if configuration_changes.contains(module_id) {
5107 module
5108 .update_configuration(
5109 configured_module.module_spec(),
5110 configured_module.health.clone(),
5111 configured_module.drain_timeout_ms,
5112 )
5113 .await
5114 .map_err(|err| {
5115 format!(
5116 "failed to update module_id '{module_id}' configuration during rescan: {err}"
5117 )
5118 })?;
5119 }
5120 if enabled_changes.contains(module_id) {
5121 self.route_outages.mark_operator_action(module_id);
5124 module
5125 .set_enabled(configured_module.enabled)
5126 .await
5127 .map_err(|err| {
5128 self.route_outages.operator_action_ended_unrefused(module_id);
5129 format!(
5130 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5131 configured_module.enabled
5132 )
5133 })?;
5134 }
5135 }
5136
5137 for module_id in &added {
5138 let configured_module = configured
5139 .get(module_id)
5140 .expect("added module id came from configured map");
5141 supervisor
5142 .supervise_configured_with_health(
5143 configured_module.module_spec(),
5144 configured_module.enabled,
5145 configured_module.health.clone(),
5146 configured_module.drain_timeout_ms,
5147 configured_module.restart,
5148 )
5149 .map_err(|err| {
5150 format!("failed to add module_id '{module_id}' during rescan: {err}")
5151 })?;
5152 }
5153
5154 Ok(SupervisorRescanResult {
5155 added,
5156 removed,
5157 changed_pending_reload,
5158 enabled_changes: enabled_changes.iter().cloned().collect(),
5159 unchanged,
5160 preview: false,
5161 restart_required: Vec::new(),
5164 capability_warnings: Vec::new(),
5165 })
5166 }
5167
5168 async fn handle_supervisor_set_enabled(
5169 &self,
5170 frame: Frame,
5171 module_id: String,
5172 enabled: bool,
5173 ) -> Result<Vec<Frame>, RouterError> {
5174 let operation_lock = self.supervisor.operation_lock();
5175 let _operation_guard = operation_lock.lock().await;
5176 let Some(module) = self.supervisor.get(&module_id) else {
5177 return Ok(vec![control_error_frame(
5178 &frame,
5179 "unknown_module",
5180 format!("module_id '{module_id}' is not supervised"),
5181 )?]);
5182 };
5183
5184 self.route_outages.mark_operator_action(&module_id);
5187 let applied = match module.set_enabled(enabled).await {
5188 Ok(applied) => applied,
5189 Err(err) => {
5190 self.route_outages
5191 .operator_action_ended_unrefused(&module_id);
5192 return Ok(vec![control_error_frame(
5193 &frame,
5194 "target_unavailable",
5195 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
5196 )?]);
5197 }
5198 };
5199 if !applied {
5200 self.route_outages
5203 .operator_action_ended_unrefused(&module_id);
5204 }
5205
5206 self.capability_evaluator.wake_deadline_loop();
5207 self.refresh_capability_requirements();
5208 let response = ClientControlResponse::SupervisorAck { module_id, applied };
5209 Ok(vec![control_response_body_frame(
5210 &frame,
5211 &response,
5212 "ClientControlResponse::SupervisorAck",
5213 )?])
5214 }
5215
5216 async fn handle_supervisor_health_probe(
5217 &self,
5218 frame: Frame,
5219 module_id: String,
5220 ) -> Result<Vec<Frame>, RouterError> {
5221 self.refresh_capability_requirements();
5222 let Some(registration) = self
5223 .registry
5224 .get_module(&module_id)
5225 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5226 else {
5227 return Ok(vec![control_error_frame(
5228 &frame,
5229 "unknown_module",
5230 format!("module_id '{module_id}' is not registered"),
5231 )?]);
5232 };
5233
5234 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
5247 {
5248 return Ok(vec![control_error_frame(
5249 &frame,
5250 "health_not_advertised",
5251 format!("module_id '{module_id}' did not advertise health.check"),
5252 )?]);
5253 }
5254
5255 let deadline = Instant::now() + self.health_probe_timeout;
5256 let pending = match self.forwarding.begin_module_control_rpc_for(
5257 &module_id,
5258 MODULE_CONTROL_OP_HEALTH_CHECK,
5259 deadline,
5260 ) {
5261 Ok(pending) => pending,
5262 Err(err) => {
5263 return Ok(vec![control_error_frame(
5264 &frame,
5265 forwarding_error_code(&err),
5266 err.to_string(),
5267 )?])
5268 }
5269 };
5270
5271 let PendingModuleControlRpc {
5272 endpoint,
5273 module_sink,
5274 negotiated_ver,
5275 corr: probe_corr,
5276 receiver,
5277 } = pending;
5278 let mut guard =
5279 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
5280 let probe_body =
5281 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5282 RouterError::backend(
5283 0,
5284 frame.header.corr,
5285 format!("failed to encode health.check request: {err}"),
5286 )
5287 })?;
5288 let probe_frame = Frame::build_with_version(
5289 negotiated_ver,
5290 FrameType::Request,
5291 control_flags(),
5292 0,
5293 0,
5294 probe_corr,
5295 probe_body,
5296 )
5297 .map_err(RouterError::FrameBuild)?;
5298
5299 if let Err(err) = module_sink.send(probe_frame).await {
5300 return Ok(vec![control_error_frame(
5301 &frame,
5302 "target_unavailable",
5303 err.to_string(),
5304 )?]);
5305 }
5306
5307 match timeout_at(deadline, receiver).await {
5308 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5309 guard.disarm();
5310 let Some(report) = response.health_report() else {
5311 return Ok(vec![control_error_frame(
5312 &frame,
5313 "invalid_control_body",
5314 "health.check RPC returned a non-health response",
5315 )?]);
5316 };
5317 let HealthReport {
5322 status,
5323 detail,
5324 metrics,
5325 } = report;
5326 let capability_detail = self
5327 .capability_evaluator
5328 .required_problem_detail(&module_id);
5329 let response = ClientControlResponse::SupervisorHealthProbe {
5330 module_id,
5331 status,
5332 detail: append_capability_problem_detail(detail, capability_detail),
5333 metrics,
5334 };
5335 Ok(vec![control_response_body_frame(
5336 &frame,
5337 &response,
5338 "ClientControlResponse::SupervisorHealthProbe",
5339 )?])
5340 }
5341 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
5342 guard.disarm();
5343 Ok(vec![control_error_body_frame(&frame, body)?])
5344 }
5345 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5346 guard.disarm();
5347 Ok(vec![control_error_frame(
5348 &frame,
5349 "target_unavailable",
5350 message,
5351 )?])
5352 }
5353 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5354 guard.disarm();
5355 Ok(vec![control_error_frame(
5356 &frame,
5357 "invalid_control_body",
5358 message,
5359 )?])
5360 }
5361 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5362 guard.disarm();
5363 Ok(vec![control_error_frame(
5364 &frame,
5365 "invalid_control_body",
5366 format!("expected module-control op '{expected}', got '{actual}'"),
5367 )?])
5368 }
5369 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
5370 guard.disarm();
5371 Ok(vec![control_error_frame(
5372 &frame,
5373 "module_timeout",
5374 format!(
5375 "module_id '{module_id}' answered health.check after {:?}",
5376 self.health_probe_timeout
5377 ),
5378 )?])
5379 }
5380 Ok(Err(_)) => Ok(vec![control_error_frame(
5381 &frame,
5382 "target_unavailable",
5383 "health.check waiter was canceled before the module responded",
5384 )?]),
5385 Err(_) => Ok(vec![control_error_frame(
5386 &frame,
5387 "module_timeout",
5388 format!(
5389 "module_id '{module_id}' did not answer health.check within {:?}",
5390 self.health_probe_timeout
5391 ),
5392 )?]),
5393 }
5394 }
5395
5396 fn supervisor_status(
5397 &self,
5398 module_id: &str,
5399 corr: u64,
5400 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5401 self.supervisor
5402 .get(module_id)
5403 .map(|module| {
5404 let warming = module.is_warming_for_control("status").map_err(|err| {
5405 RouterError::backend(
5406 0,
5407 corr,
5408 format!(
5409 "failed to read supervisor warming state for module_id '{module_id}': {err}"
5410 ),
5411 )
5412 })?;
5413 module.status_for_control("status").map_err(|err| {
5414 RouterError::backend(
5415 0,
5416 corr,
5417 format!(
5418 "failed to read supervisor status for module_id '{module_id}': {err}"
5419 ),
5420 )
5421 }).map(|status| (status, warming))
5422 })
5423 .transpose()
5424 }
5425
5426 fn guard_module_control_op(
5427 &self,
5428 frame: &Frame,
5429 module_id: &str,
5430 op: &str,
5431 ) -> Result<Option<Frame>, RouterError> {
5432 if self.module_grants_op(module_id, op, frame.header.corr)? {
5433 return Ok(None);
5434 }
5435
5436 Ok(Some(control_error_frame(
5437 frame,
5438 "op_not_allowed",
5439 format!("module_id '{module_id}' did not grant control op '{op}'"),
5440 )?))
5441 }
5442
5443 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
5444 let Some(registration) = self
5445 .registry
5446 .get_module(module_id)
5447 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
5448 else {
5449 return Ok(false);
5450 };
5451 Ok(module_registration_grants_op(®istration.control_ops, op))
5452 }
5453
5454 fn handle_status_update(
5455 &self,
5456 endpoint: ModuleEndpointId,
5457 frame: Frame,
5458 ) -> Result<Vec<Frame>, RouterError> {
5459 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
5460 Ok(update) => update,
5461 Err(err) => {
5462 if is_known_module_push_op(&frame.body) {
5467 return Ok(vec![control_error_frame(
5468 &frame,
5469 "invalid_control_body",
5470 format!("malformed module control push body: {err}"),
5471 )?]);
5472 }
5473 return Ok(Vec::new());
5474 }
5475 };
5476
5477 match update {
5478 ModuleControlPush::RouteStatus {
5479 route_channel,
5480 route_epoch,
5481 status,
5482 } => {
5483 self.forwarding
5484 .cache_status(endpoint, route_channel, route_epoch, status)
5485 .map_err(RouterError::Forwarding)?;
5486 }
5487 }
5488 Ok(Vec::new())
5489 }
5490
5491 fn handle_route_poll(
5492 &self,
5493 ctx: &RouteCtx,
5494 frame: Frame,
5495 route_channel: u16,
5496 route_epoch: u32,
5497 kind: PollKind,
5498 ) -> Result<Vec<Frame>, RouterError> {
5499 let snapshot = self
5500 .forwarding
5501 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
5502 .map_err(RouterError::Forwarding)?;
5503 let response = match (kind, snapshot) {
5504 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
5505 ClientControlResponse::RoutePoll {
5506 route_channel,
5507 route_epoch,
5508 status,
5509 live: None,
5510 }
5511 }
5512 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5513 route_channel,
5514 route_epoch,
5515 status: None,
5516 live: None,
5517 },
5518 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5519 let live = self
5545 .process_liveness
5546 .as_ref()
5547 .and_then(|source| source.process_live(&module_id))
5548 .unwrap_or(true);
5549 ClientControlResponse::RoutePoll {
5550 route_channel,
5551 route_epoch,
5552 status: None,
5553 live: Some(live),
5554 }
5555 }
5556 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5557 route_channel,
5558 route_epoch,
5559 status: None,
5560 live: Some(false),
5561 },
5562 };
5563
5564 Ok(vec![control_response_body_frame(
5565 &frame,
5566 &response,
5567 "ClientControlResponse::RoutePoll",
5568 )?])
5569 }
5570
5571 pub(crate) fn observe_module_control_completion(
5572 &self,
5573 completion: ModuleControlRpcCompletion,
5574 ) -> bool {
5575 match completion {
5576 ModuleControlRpcCompletion::Unknown => false,
5577 ModuleControlRpcCompletion::Settled => true,
5578 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5579 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5580 info!(
5581 module_id = %module_id,
5582 latency_ms,
5583 "late health.check answer proves the module is alive"
5584 );
5585 match self
5586 .supervisor
5587 .record_late_health_answer(&module_id, latency_ms)
5588 {
5589 Ok(true) => {}
5590 Ok(false) => debug!(
5591 module_id = %module_id,
5592 latency_ms,
5593 "late health.check answer has no active supervisor snapshot"
5594 ),
5595 Err(err) => warn!(
5596 module_id = %module_id,
5597 latency_ms,
5598 error = %err,
5599 "failed to record late health.check answer"
5600 ),
5601 }
5602 true
5603 }
5604 }
5605 }
5606
5607 fn refuse_to_end_module_connection_for_a_client(
5628 &self,
5629 module_connection_id: ConnectionId,
5630 corr: u64,
5631 err: ForwardingError,
5632 ) -> Result<(), RouterError> {
5633 if let ForwardingError::ConnectionClosing { connection_id } = err {
5634 if connection_id != module_connection_id {
5635 warn!(
5636 module_connection_id = module_connection_id.get(),
5637 client_connection_id = connection_id.get(),
5638 corr,
5639 "dropping a route.bind response for a closing client; the module connection keeps serving"
5640 );
5641 return Ok(());
5642 }
5643 }
5644 Err(RouterError::Forwarding(err))
5645 }
5646
5647 fn handle_module_relay_response(
5648 &self,
5649 connection_id: ConnectionId,
5650 frame: Frame,
5651 ) -> Result<Vec<Frame>, RouterError> {
5652 let mut secondary_error = None;
5653 let outcome = match frame.header.ty {
5654 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5655 Ok(probe) if probe.op == "route.bind" => {
5656 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5657 Ok(ModuleControlResponse::RouteBindAck {}) => {
5658 RouteBindRelayOutcome::Accepted
5659 }
5660 Ok(other) => {
5661 let message =
5662 format!("route.bind response carried unexpected body: {other:?}");
5663 secondary_error = Some(control_error_frame(
5664 &frame,
5665 "invalid_control_body",
5666 message.clone(),
5667 )?);
5668 RouteBindRelayOutcome::ModuleGone(message)
5669 }
5670 Err(err) => {
5671 let message = format!("malformed route.bind response body: {err}");
5672 secondary_error = Some(control_error_frame(
5673 &frame,
5674 "invalid_control_body",
5675 message.clone(),
5676 )?);
5677 RouteBindRelayOutcome::ModuleGone(message)
5678 }
5679 }
5680 }
5681 Ok(probe) => {
5682 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5683 {
5684 Ok(response) => ModuleControlRpcOutcome::Response(response),
5685 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5686 "malformed {} response body: {err}",
5687 probe.op
5688 )),
5689 };
5690 let completion = self
5691 .forwarding
5692 .complete_module_control_rpc(
5693 connection_id,
5694 frame.header.corr,
5695 Some(&probe.op),
5696 outcome,
5697 )
5698 .map_err(RouterError::Forwarding)?;
5699 if !self.observe_module_control_completion(completion) {
5700 debug!(
5701 connection_id = connection_id.get(),
5702 corr = frame.header.corr,
5703 op = %probe.op,
5704 "dropping late or unknown module-control RPC response"
5705 );
5706 }
5707 return Ok(Vec::new());
5708 }
5709 Err(err) => {
5710 if let Some(expected_op) = self
5711 .forwarding
5712 .pending_module_control_op(connection_id, frame.header.corr)
5713 .map_err(RouterError::Forwarding)?
5714 {
5715 let completion = self
5716 .forwarding
5717 .complete_module_control_rpc(
5718 connection_id,
5719 frame.header.corr,
5720 None,
5721 ModuleControlRpcOutcome::MalformedResponse(format!(
5722 "malformed {expected_op} response body: {err}"
5723 )),
5724 )
5725 .map_err(RouterError::Forwarding)?;
5726 if !self.observe_module_control_completion(completion) {
5727 debug!(
5728 connection_id = connection_id.get(),
5729 corr = frame.header.corr,
5730 "dropping late malformed module-control RPC response"
5731 );
5732 }
5733 return Ok(Vec::new());
5734 }
5735 let message = format!("malformed route.bind response body: {err}");
5736 secondary_error = Some(control_error_frame(
5737 &frame,
5738 "invalid_control_body",
5739 message.clone(),
5740 )?);
5741 RouteBindRelayOutcome::ModuleGone(message)
5742 }
5743 },
5744 FrameType::Error => {
5745 if self
5746 .forwarding
5747 .pending_module_control_op(connection_id, frame.header.corr)
5748 .map_err(RouterError::Forwarding)?
5749 .is_some()
5750 {
5751 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5752 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5753 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5754 "malformed module-control ERROR body: {err}"
5755 )),
5756 };
5757 let completion = self
5758 .forwarding
5759 .complete_module_control_rpc(
5760 connection_id,
5761 frame.header.corr,
5762 None,
5763 outcome,
5764 )
5765 .map_err(RouterError::Forwarding)?;
5766 if !self.observe_module_control_completion(completion) {
5767 debug!(
5768 connection_id = connection_id.get(),
5769 corr = frame.header.corr,
5770 "dropping late or unknown module-control RPC error"
5771 );
5772 }
5773 return Ok(Vec::new());
5774 }
5775 match serde_json::from_slice::<ErrorBody>(&frame.body) {
5776 Ok(body) => RouteBindRelayOutcome::Rejected(body),
5777 Err(err) => {
5778 let message = format!("malformed route.bind ERROR body: {err}");
5779 secondary_error = Some(control_error_frame(
5780 &frame,
5781 "invalid_control_body",
5782 message.clone(),
5783 )?);
5784 RouteBindRelayOutcome::ModuleGone(message)
5785 }
5786 }
5787 }
5788 ty => {
5789 return Ok(vec![control_error_frame(
5790 &frame,
5791 "unsupported_control_frame",
5792 format!("unsupported module channel-0 frame {ty:?}"),
5793 )?])
5794 }
5795 };
5796
5797 let settled =
5798 self.forwarding
5799 .complete_pending_relay(connection_id, frame.header.corr, outcome);
5800 let completion = match settled {
5801 Ok(completion) => completion,
5802 Err(err) => {
5803 self.refuse_to_end_module_connection_for_a_client(
5804 connection_id,
5805 frame.header.corr,
5806 err,
5807 )?;
5808 return Ok(secondary_error.into_iter().collect());
5809 }
5810 };
5811 if let Some(target) = completion.abandoned.as_ref() {
5812 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5813 }
5814 if !completion.settled {
5815 debug!(
5816 connection_id = connection_id.get(),
5817 corr = frame.header.corr,
5818 frame_type = ?frame.header.ty,
5819 "dropping late or unknown route.bind relay response"
5820 );
5821 }
5822 Ok(secondary_error.into_iter().collect())
5823 }
5824
5825 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5826 debug!(connection_id = connection_id.get(), "handling GOODBYE");
5827 self.cleanup_connection_with_end_reason(
5831 connection_id,
5832 RegistrationEndReason::ExplicitGoodbye,
5833 )
5834 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5835 Ok(Vec::new())
5836 }
5837}
5838
5839impl Default for ControlHandler {
5840 fn default() -> Self {
5841 Self::new(Arc::new(Registry::default()))
5842 }
5843}
5844
5845impl crate::supervise::SwapPromotionObserver for ControlHandler {
5846 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5847 self.apply_registration_capabilities(registration);
5848 }
5849}
5850
5851fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5852 CapabilityRequirementStatus {
5853 consumer: status.consumer,
5854 capability: status.capability,
5855 need: match status.need {
5856 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5857 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5858 },
5859 verdict: status.verdict.as_str().to_string(),
5860 episode_seq: status.episode_seq,
5861 config_satisfiable: status.config_satisfiable,
5862 runtime_available: status.runtime_available,
5863 detail: status.detail,
5864 }
5865}
5866
5867fn append_capability_problem_detail(
5868 detail: Option<String>,
5869 capability_detail: Option<String>,
5870) -> Option<String> {
5871 match (detail, capability_detail) {
5872 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5873 (Some(detail), None) => Some(detail),
5874 (None, Some(capability_detail)) => Some(capability_detail),
5875 (None, None) => None,
5876 }
5877}
5878
5879fn subc_ops() -> Vec<String> {
5880 SUBC_CONTROL_OPS
5881 .iter()
5882 .map(|op| (*op).to_string())
5883 .collect()
5884}
5885
5886fn module_subc_ops() -> Vec<String> {
5887 SUBC_CONTROL_OPS
5888 .iter()
5889 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5890 .map(|op| (*op).to_string())
5891 .collect()
5892}
5893
5894#[cfg(test)]
5895fn module_baseline_control_ops() -> Vec<String> {
5896 MODULE_BASELINE_CONTROL_OPS
5897 .iter()
5898 .map(|op| (*op).to_string())
5899 .collect()
5900}
5901
5902fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5903 let mut seen = HashSet::new();
5904 let mut effective = Vec::new();
5905 for op in MODULE_BASELINE_CONTROL_OPS {
5906 if seen.insert((*op).to_string()) {
5907 effective.push((*op).to_string());
5908 }
5909 }
5910 for op in declared.unwrap_or_default() {
5911 if seen.insert(op.clone()) {
5912 effective.push(op);
5913 }
5914 }
5915 effective
5916}
5917
5918fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5919 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5920}
5921
5922fn target_module_id(target: &RouteTarget) -> &str {
5923 match target {
5924 RouteTarget::ToolProvider { module_id }
5925 | RouteTarget::ManagementSurface { module_id }
5926 | RouteTarget::InternalService { module_id, .. } => module_id,
5927 }
5928}
5929
5930fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5931 roles.iter().any(|role| match (target, role) {
5932 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5933 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5934 (
5935 RouteTarget::InternalService { service_id, .. },
5936 ProviderRole::InternalService {
5937 service_id: provided,
5938 ..
5939 },
5940 ) => service_id == provided,
5941 _ => false,
5942 })
5943}
5944
5945fn is_routable_role(role: &ProviderRole) -> bool {
5946 matches!(
5947 role,
5948 ProviderRole::ToolProvider { .. }
5949 | ProviderRole::ManagementSurface { .. }
5950 | ProviderRole::InternalService { .. }
5951 )
5952}
5953
5954#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5955enum ControlRequestBodyError {
5956 UnknownOp,
5957 InvalidBody,
5958}
5959
5960#[derive(Debug, Deserialize)]
5961struct ControlOpProbe {
5962 op: String,
5963}
5964
5965const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5968
5969fn is_known_module_push_op(body: &[u8]) -> bool {
5970 serde_json::from_slice::<ControlOpProbe>(body)
5971 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5972 .unwrap_or(false)
5973}
5974
5975fn is_known_module_request_op(body: &[u8]) -> bool {
5976 serde_json::from_slice::<ControlOpProbe>(body)
5977 .map(|probe| is_module_to_subc_op(&probe.op))
5978 .unwrap_or(false)
5979}
5980
5981fn is_module_to_subc_op(op: &str) -> bool {
5982 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5983}
5984
5985fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5986 debug!(
5987 op = %op,
5988 connection_id = connection_id.get(),
5989 corr,
5990 "control dispatch"
5991 );
5992}
5993
5994fn log_slow_control_dispatch(
5995 dispatch_started_at: Option<StdInstant>,
5996 op: &'static str,
5997 connection_id: ConnectionId,
5998 corr: u64,
5999) {
6000 let Some(dispatch_started_at) = dispatch_started_at else {
6001 return;
6002 };
6003 let elapsed = dispatch_started_at.elapsed();
6004 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6005 warn!(
6006 op = %op,
6007 connection_id = connection_id.get(),
6008 corr,
6009 elapsed_ms = elapsed.as_millis() as u64,
6010 "slow control dispatch"
6011 );
6012 }
6013}
6014
6015fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6016 match request {
6017 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6018 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6019 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6020 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6021 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6022 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6023 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6024 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6025 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6026 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6027 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6028 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6029 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6030 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6031 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6032 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6033 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6034 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6035 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6036 }
6037}
6038
6039fn principal_label(principal: &Principal) -> String {
6040 match principal {
6041 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6042 Principal::Direct => "direct".to_string(),
6043 other => format!("{other:?}"),
6044 }
6045}
6046
6047fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6048 match request {
6049 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6050 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6051 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6052 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6053 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6054 }
6055}
6056
6057fn parse_client_control_request(
6058 body: &[u8],
6059) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6060 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6061 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6062 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6063 ControlRequestBodyError::InvalidBody
6064 }
6065 Ok(_) => ControlRequestBodyError::UnknownOp,
6066 Err(_) => ControlRequestBodyError::InvalidBody,
6067 };
6068 (err, classification)
6069 })
6070}
6071
6072fn parse_module_control_request_from_module(
6073 body: &[u8],
6074) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6075 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6076 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6077 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6078 Ok(_) => ControlRequestBodyError::UnknownOp,
6079 Err(_) => ControlRequestBodyError::InvalidBody,
6080 };
6081 (err, classification)
6082 })
6083}
6084
6085#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6086enum ProviderRoleKind {
6087 ToolProvider,
6088 PipelineStage,
6089 ManagementSurface,
6090 InternalService,
6091}
6092
6093fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6094 match role {
6095 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6096 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6097 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6098 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
6099 }
6100}
6101
6102fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
6103 roles.iter().map(provider_role_kind).collect()
6104}
6105
6106const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
6109
6110#[derive(Debug, Default, PartialEq, Eq)]
6112struct ScopeOutcomeCounts {
6113 created: usize,
6114 replaced: usize,
6115 updated: usize,
6116 unchanged: usize,
6117 refused: usize,
6118}
6119
6120impl ScopeOutcomeCounts {
6121 fn of(results: &[ScopeRecordResult]) -> Self {
6122 let mut counts = Self::default();
6123 for result in results {
6124 let slot = match result.outcome {
6125 ScopeRecordOutcome::Created => &mut counts.created,
6126 ScopeRecordOutcome::Replaced => &mut counts.replaced,
6127 ScopeRecordOutcome::Updated => &mut counts.updated,
6128 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
6129 ScopeRecordOutcome::Refused => &mut counts.refused,
6130 };
6131 *slot += 1;
6132 }
6133 counts
6134 }
6135}
6136
6137#[cfg(test)]
6138mod scope_outcome_count_tests {
6139 use super::*;
6140
6141 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
6142 ScopeRecordResult {
6143 scope_ref: "r".to_string(),
6144 scope_epoch: 1,
6145 outcome,
6146 code: None,
6147 message: None,
6148 version: None,
6149 parent_state: None,
6150 }
6151 }
6152
6153 #[test]
6156 fn every_outcome_is_counted_in_its_own_field() {
6157 let results = [
6158 result(ScopeRecordOutcome::Created),
6159 result(ScopeRecordOutcome::Created),
6160 result(ScopeRecordOutcome::Replaced),
6161 result(ScopeRecordOutcome::Updated),
6162 result(ScopeRecordOutcome::Unchanged),
6163 result(ScopeRecordOutcome::Refused),
6164 result(ScopeRecordOutcome::Refused),
6165 result(ScopeRecordOutcome::Refused),
6166 ];
6167 assert_eq!(
6168 ScopeOutcomeCounts::of(&results),
6169 ScopeOutcomeCounts {
6170 created: 2,
6171 replaced: 1,
6172 updated: 1,
6173 unchanged: 1,
6174 refused: 3,
6175 }
6176 );
6177 }
6178}
6179
6180fn capability_census_trigger(
6184 old: Option<&CapabilityDeclarations>,
6185 new: Option<&CapabilityDeclarations>,
6186) -> bool {
6187 let old_provides = old
6188 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
6189 .unwrap_or_default();
6190 let old_denies = old
6191 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
6192 .unwrap_or_default();
6193 let new = new.cloned().unwrap_or(CapabilityDeclarations {
6194 provides: Vec::new(),
6195 requires: Vec::new(),
6196 must_never_reach: Vec::new(),
6197 });
6198
6199 new.provides
6200 .iter()
6201 .any(|capability| !old_provides.contains(capability))
6202 || new
6203 .must_never_reach
6204 .iter()
6205 .any(|capability| !old_denies.contains(capability))
6206}
6207
6208fn denied_capability<'a>(
6211 opening_manifest: &'a ModuleManifest,
6212 target_manifest: &ModuleManifest,
6213) -> Option<&'a str> {
6214 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
6215 let target_capabilities = target_manifest.capabilities.as_ref()?;
6216 opening_capabilities
6217 .must_never_reach
6218 .iter()
6219 .find(|denied| {
6220 target_capabilities
6221 .provides
6222 .iter()
6223 .any(|provided| provided == *denied)
6224 })
6225 .map(String::as_str)
6226}
6227
6228fn catalog_update_frozen_field_message(
6229 registered: &ModuleManifest,
6230 provides: &[ProviderRole],
6231) -> Option<String> {
6232 let old_has_provides = !registered.provides.is_empty();
6233 let new_has_provides = !provides.is_empty();
6234 if old_has_provides != new_has_provides {
6235 return Some(format!(
6236 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
6237 registered.module_id
6238 ));
6239 }
6240
6241 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
6242 return Some(format!(
6243 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
6244 registered.module_id
6245 ));
6246 }
6247
6248 let registered_concurrency = manifest_concurrency(registered);
6249 let mut candidate = registered.clone();
6250 candidate.provides = provides.to_vec();
6251 let candidate_concurrency = manifest_concurrency(&candidate);
6252 if candidate_concurrency != registered_concurrency {
6253 return Some(format!(
6254 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
6255 registered.module_id, registered_concurrency, candidate_concurrency
6256 ));
6257 }
6258
6259 None
6262}
6263
6264fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
6265 manifest.provides.iter().any(is_routable_role)
6266}
6267
6268fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
6274 manifest
6275 .provides
6276 .iter()
6277 .find_map(|provider| match provider {
6278 ProviderRole::ToolProvider { concurrency, .. }
6279 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
6280 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
6281 })
6282 .unwrap_or(Concurrency::ModuleManaged)
6283}
6284
6285fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
6292 let has_management_surface = manifest
6293 .provides
6294 .iter()
6295 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
6296 if !has_management_surface {
6297 return false;
6298 }
6299 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
6300 return false;
6301 };
6302 let Some(provides) = raw
6303 .get("manifest")
6304 .and_then(|manifest| manifest.get("provides"))
6305 .and_then(serde_json::Value::as_array)
6306 else {
6307 return false;
6308 };
6309 provides.iter().any(|role| {
6314 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
6315 && role.get("concurrency").is_none()
6316 })
6317}
6318
6319fn negotiate_version(peer_version: u8) -> Result<u8, String> {
6320 if peer_version != PROTOCOL_VERSION {
6321 return Err(format!(
6322 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
6323 ));
6324 }
6325 Ok(PROTOCOL_VERSION)
6326}
6327
6328fn pong(frame: &Frame) -> Result<Frame, RouterError> {
6329 Frame::build_with_version(
6330 response_version(frame),
6331 FrameType::Pong,
6332 frame.header.flags,
6333 0,
6334 0,
6335 frame.header.corr,
6336 Vec::new(),
6337 )
6338 .map_err(RouterError::FrameBuild)
6339}
6340
6341fn control_error_frame(
6342 frame: &Frame,
6343 code: &'static str,
6344 message: impl Into<String>,
6345) -> Result<Frame, RouterError> {
6346 control_error_body_frame(
6347 frame,
6348 ErrorBody {
6349 code: code.to_string(),
6350 message: message.into(),
6351 detail: None,
6352 },
6353 )
6354}
6355
6356fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
6357 let body = serde_json::to_vec(&error).map_err(|err| {
6358 RouterError::backend(
6359 0,
6360 frame.header.corr,
6361 format!("failed to encode control ERROR: {err}"),
6362 )
6363 })?;
6364
6365 Frame::build_with_version(
6366 response_version(frame),
6367 FrameType::Error,
6368 control_flags(),
6369 0,
6370 0,
6371 frame.header.corr,
6372 body,
6373 )
6374 .map_err(RouterError::FrameBuild)
6375}
6376
6377fn control_response_body_frame<T: Serialize>(
6378 frame: &Frame,
6379 reply: &T,
6380 label: &'static str,
6381) -> Result<Frame, RouterError> {
6382 let body = serde_json::to_vec(reply).map_err(|err| {
6383 RouterError::backend(
6384 0,
6385 frame.header.corr,
6386 format!("failed to encode {label}: {err}"),
6387 )
6388 })?;
6389
6390 Frame::build_with_version(
6391 response_version(frame),
6392 FrameType::Response,
6393 control_flags(),
6394 0,
6395 0,
6396 frame.header.corr,
6397 body,
6398 )
6399 .map_err(RouterError::FrameBuild)
6400}
6401
6402fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6432 match err {
6433 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6434 ForwardingError::NoModuleConnection => "target_unavailable",
6435 ForwardingError::ModuleReloading { .. } => "module_reloading",
6436 ForwardingError::ClientRouteChannelExhausted { .. }
6437 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6438 ForwardingError::StaleModuleEndpoint
6439 | ForwardingError::UnknownReservation { .. }
6440 | ForwardingError::ConnectionClosing { .. }
6441 | ForwardingError::ClientEgressClosed { .. }
6442 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
6443 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
6446 ForwardingError::RelayCorrelationExhausted
6447 | ForwardingError::RouteOpenBuild(_)
6448 | ForwardingError::Poisoned => "forwarding_error",
6449 }
6450}
6451
6452fn response_version(frame: &Frame) -> u8 {
6453 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
6454 frame.header.ver
6455 } else {
6456 PROTOCOL_VERSION
6457 }
6458}
6459
6460fn control_flags() -> Flags {
6461 Flags::new(false, Priority::Passive, false)
6462}
6463
6464fn send_goodbye_target_best_effort(
6469 counters: &DaemonCounters,
6470 target: &GoodbyeTarget,
6471 context: &'static str,
6472) {
6473 let Ok(frame) = Frame::build_with_version(
6474 target.negotiated_ver,
6475 FrameType::Goodbye,
6476 control_flags(),
6477 target.channel,
6478 target.epoch,
6479 0,
6480 Vec::new(),
6481 ) else {
6482 return;
6483 };
6484 crate::forwarding::send_module_route_goodbye(
6485 counters,
6486 &target.sink,
6487 frame,
6488 target.module_id.as_deref(),
6489 context,
6490 );
6491}
6492
6493pub(crate) fn send_route_control_pushes(
6494 forwarding: &ForwardingTable,
6495 routes: Vec<EndpointRoute>,
6496 push: ClientControlPush,
6497) {
6498 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
6499 for route in routes {
6500 let target = route.goodbye_target;
6501 if let Some((existing, channels)) = targets
6502 .iter_mut()
6503 .find(|(existing, _)| existing.connection_id == target.connection_id)
6504 {
6505 debug_assert_eq!(
6506 existing.negotiated_ver, target.negotiated_ver,
6507 "one connection cannot negotiate multiple frame versions"
6508 );
6509 if !channels.contains(&target.channel) {
6510 channels.push(target.channel);
6511 }
6512 continue;
6513 }
6514 let channel = target.channel;
6515 targets.push((target, vec![channel]));
6516 }
6517 for (target, mut channels) in targets {
6518 channels.sort_unstable();
6519 let mut push = push.clone();
6520 match &mut push {
6521 ClientControlPush::RouteClosing {
6522 channels: covered, ..
6523 }
6524 | ClientControlPush::RouteClosed {
6525 channels: covered, ..
6526 } => *covered = channels,
6527 }
6528 let body = match serde_json::to_vec(&push) {
6529 Ok(body) => body,
6530 Err(err) => {
6531 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
6532 continue;
6533 }
6534 };
6535 let frame = match Frame::build_with_version(
6536 target.negotiated_ver,
6537 FrameType::Push,
6538 control_flags(),
6539 0,
6540 0,
6541 0,
6542 body.clone(),
6543 ) {
6544 Ok(frame) => frame,
6545 Err(err) => {
6546 warn!(
6547 route_channel = target.channel,
6548 error = %err,
6549 "failed to build route lifecycle control PUSH frame"
6550 );
6551 continue;
6552 }
6553 };
6554 if let Err(err) = target.sink.try_send(frame) {
6555 if target.close_on_delivery_failure() {
6556 warn!(
6557 target_connection_id = target.connection_id.get(),
6558 route_channel = target.channel,
6559 error = %err,
6560 "route lifecycle control PUSH was not delivered to client; closing target connection"
6561 );
6562 let _ = forwarding.escalate_client_delivery_failure(
6563 target.connection_id,
6564 target.channel,
6565 target.epoch,
6566 CloseReason::new(
6567 "route_lifecycle_push_delivery_failed",
6568 format!(
6569 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6570 target.channel
6571 ),
6572 ),
6573 crate::forwarding::UndeliveredFrame {
6574 module_id: target.module_id.as_deref(),
6575 sink: &target.sink,
6576 },
6577 );
6578 }
6579 }
6580 }
6581}
6582
6583#[cfg(test)]
6584mod tests {
6585 #[cfg(unix)]
6586 #[tokio::test]
6587 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
6588 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
6589 let path = dir.join("subc.jsonc");
6590 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
6591 "program":"/bin/sleep","args":["60"],"protocol":"none",
6592 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
6593 }}}).to_string()).unwrap();
6594 let mut configured = crate::daemon_config::load(&path)
6595 .unwrap()
6596 .unwrap()
6597 .modules
6598 .pop()
6599 .unwrap();
6600 let registry = std::sync::Arc::new(crate::Registry::default());
6601 let handle = crate::SupervisorHandle::new();
6602 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
6603 .with_handle(handle.clone());
6604 let module = supervisor
6605 .supervise_configured_with_health(
6606 configured.module_spec(),
6607 true,
6608 configured.health.clone(),
6609 None,
6610 configured.restart,
6611 )
6612 .unwrap();
6613 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
6614 let before = module.status().unwrap().pid;
6615 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
6616 configured.health.cadence = std::time::Duration::from_secs(3600);
6617 let health_only = handler
6618 .reconcile_supervised_modules(&supervisor, vec![configured.clone()], false)
6619 .await
6620 .unwrap();
6621 assert!(
6622 health_only.changed_pending_reload.is_empty(),
6623 "health policy is already applied live"
6624 );
6625 assert_eq!(module.status().unwrap().pid, before);
6626 assert_eq!(
6627 module.configuration().unwrap().1.http,
6628 configured.health.http
6629 );
6630 configured.args = vec!["61".into()];
6631 let launch = handler
6632 .reconcile_supervised_modules(&supervisor, vec![configured], false)
6633 .await
6634 .unwrap();
6635 assert_eq!(launch.changed_pending_reload, ["stock"]);
6636 assert_eq!(
6637 module.status().unwrap().pid,
6638 before,
6639 "a launch edit is stored until reload"
6640 );
6641 module.drain().await.unwrap();
6642 }
6643 use cortexkit_test_support::ScratchDir;
6644 use std::{
6645 collections::BTreeMap,
6646 fmt,
6647 path::PathBuf,
6648 sync::{Arc, Mutex},
6649 time::Duration,
6650 };
6651
6652 use serde_json::{json, Value};
6653 use subc_protocol::{
6654 manifest::{
6655 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6656 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6657 },
6658 session::HealthStatus,
6659 FrameType,
6660 };
6661
6662 use super::*;
6663 use crate::{
6664 forwarding::{DataRoute, DataRouteState},
6665 registry::ChannelState,
6666 router::FrameSink,
6667 stderr_tail::DEFAULT_MAX_LINE_BYTES,
6668 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6669 RouteCtx, Router,
6670 };
6671 use tokio::{
6672 sync::mpsc,
6673 time::{sleep, Instant},
6674 };
6675 use tracing::{
6676 field::{Field, Visit},
6677 Event, Subscriber,
6678 };
6679 use tracing_subscriber::{layer::Context, prelude::*, Layer};
6680
6681 fn fake_aft_stub_path() -> PathBuf {
6700 let mut path = std::env::current_exe().expect("current_exe available in tests");
6701 path.pop(); path.pop(); path.push(if cfg!(windows) {
6704 "fake-aft-stub.exe"
6705 } else {
6706 "fake-aft-stub"
6707 });
6708 assert!(
6709 path.exists(),
6710 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6711 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6712 path.display()
6713 );
6714 path
6715 }
6716
6717 fn client_retries(code: &str) -> bool {
6722 subc_protocol::error_codes::is_retryable_route_open(code)
6723 }
6724
6725 #[test]
6736 fn retryability_of_forwarding_codes_matches_the_failure() {
6737 let transient = [
6740 ForwardingError::NoModuleConnection,
6741 ForwardingError::ModuleReloading {
6742 module_id: "m".into(),
6743 },
6744 ForwardingError::StaleModuleEndpoint,
6745 ForwardingError::UnknownReservation {
6746 client_channel: 1,
6747 module_channel: 1,
6748 },
6749 ForwardingError::ConnectionClosing {
6750 connection_id: ConnectionId::new(1),
6751 },
6752 ForwardingError::ClientEgressClosed {
6753 connection_id: ConnectionId::new(1),
6754 },
6755 ForwardingError::ModuleEgressUnavailable {
6756 connection_id: ConnectionId::new(1),
6757 },
6758 ];
6759 for err in transient {
6760 let code = forwarding_error_code(&err);
6761 assert!(
6762 client_retries(code),
6763 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6764 );
6765 }
6766
6767 let permanent = [
6772 ForwardingError::ConnectionRoleConflict {
6773 connection_id: ConnectionId::new(1),
6774 },
6775 ForwardingError::ClientRouteChannelExhausted {
6776 connection_id: ConnectionId::new(1),
6777 },
6778 ForwardingError::ModuleRouteChannelExhausted {
6779 endpoint: ModuleEndpointId {
6780 connection_id: ConnectionId::new(1),
6781 generation: 1,
6782 },
6783 },
6784 ForwardingError::RelayCorrelationExhausted,
6785 ForwardingError::RouteOpenBuild("x".into()),
6786 ForwardingError::Poisoned,
6787 ];
6788 for err in permanent {
6789 let code = forwarding_error_code(&err);
6790 assert!(
6791 !client_retries(code),
6792 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6793 );
6794 }
6795 }
6796
6797 #[tokio::test]
6825 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6826 let handler = ControlHandler::default();
6827 let frame =
6828 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6829
6830 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6834 assert_eq!(
6835 stamped,
6836 Principal::Direct,
6837 "a caller that proved nothing must not be stamped as a supervised module"
6838 );
6839
6840 let forged = handler
6845 .route_open_principal(
6846 &frame,
6847 Some(ConsumerIdentity {
6848 module_id: "aft".to_string(),
6849 launch_nonce: "not-a-real-nonce".to_string(),
6850 }),
6851 )
6852 .unwrap();
6853 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6854 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6855 }
6856
6857 #[test]
6874 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6875 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6876 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6877 let ClientControlRequest::RouteOpen {
6878 consumer_identity, ..
6879 } = parsed
6880 else {
6881 panic!("route.open body must parse as RouteOpen");
6882 };
6883 assert_eq!(
6884 consumer_identity,
6885 Some(ConsumerIdentity {
6886 module_id: "aft".to_string(),
6887 launch_nonce: "n".to_string(),
6888 }),
6889 "the wire field name must reach the value route_open_principal reads"
6890 );
6891 }
6892
6893 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6894 ModuleManifest::builder(module_id, "0.1.0")
6895 .protocol_ver(protocol_ver)
6896 .provides(vec![ProviderRole::ToolProvider {
6897 tools: vec![Tool {
6898 name: "read".to_string(),
6899 description: None,
6900 execution_mode: ExecutionMode::Pure,
6901 schema: json!({"type": "object"}),
6902 }],
6903 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6904 concurrency: Concurrency::ModuleManaged,
6905 emits_push: true,
6906 sub_supervises: true,
6907 }])
6908 .build()
6909 }
6910
6911 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6912 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6913 }
6914
6915 fn hello_frame_with_control_ops(
6916 module_id: &str,
6917 protocol_ver: u8,
6918 corr: u64,
6919 control_ops: Option<Vec<String>>,
6920 ) -> Frame {
6921 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6922 }
6923
6924 fn hello_frame_with_nonce(
6925 module_id: &str,
6926 protocol_ver: u8,
6927 corr: u64,
6928 launch_nonce: Option<&str>,
6929 ) -> Frame {
6930 hello_frame_full(
6931 module_id,
6932 protocol_ver,
6933 corr,
6934 None,
6935 launch_nonce.map(ToOwned::to_owned),
6936 )
6937 }
6938
6939 fn hello_frame_full(
6940 module_id: &str,
6941 protocol_ver: u8,
6942 corr: u64,
6943 control_ops: Option<Vec<String>>,
6944 launch_nonce: Option<String>,
6945 ) -> Frame {
6946 let body = serde_json::to_vec(&ModuleHelloBody {
6947 manifest: manifest(module_id, protocol_ver),
6948 protocol_ver,
6949 control_ops,
6950 launch_nonce,
6951 })
6952 .unwrap();
6953 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6954 }
6955
6956 fn non_routable_hello_frame_with_control_ops(
6957 module_id: &str,
6958 corr: u64,
6959 control_ops: Option<Vec<String>>,
6960 ) -> Frame {
6961 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6962 manifest.provides.clear();
6963 let body = serde_json::to_vec(&ModuleHelloBody {
6964 manifest,
6965 protocol_ver: PROTOCOL_VERSION,
6966 control_ops,
6967 launch_nonce: None,
6968 })
6969 .unwrap();
6970 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6971 }
6972
6973 fn capability_grammar_hello_frame(
6974 capabilities: Value,
6975 runtime_computed: Option<Value>,
6976 corr: u64,
6977 ) -> Frame {
6978 let mut body = serde_json::to_value(ModuleHelloBody {
6979 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6980 protocol_ver: PROTOCOL_VERSION,
6981 control_ops: None,
6982 launch_nonce: None,
6983 })
6984 .expect("HELLO body serializes");
6985 body["manifest"]["capabilities"] = capabilities;
6986 if let Some(runtime_computed) = runtime_computed {
6987 body["runtime_computed"] = runtime_computed;
6988 }
6989 Frame::build(
6990 FrameType::Hello,
6991 control_flags(),
6992 0,
6993 0,
6994 corr,
6995 serde_json::to_vec(&body).expect("HELLO body reserializes"),
6996 )
6997 .expect("HELLO frame builds")
6998 }
6999
7000 fn channel_request(channel: u16, corr: u64) -> Frame {
7001 Frame::build(
7002 FrameType::Request,
7003 Flags::new(true, Priority::Interactive, false),
7004 channel,
7005 0,
7006 corr,
7007 b"opaque".to_vec(),
7008 )
7009 .unwrap()
7010 }
7011
7012 fn route_ctx(
7013 connection_id: ConnectionId,
7014 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7015 let (tx, rx) = mpsc::channel(8);
7016 (
7017 RouteCtx {
7018 connection_id,
7019 egress: FrameSink::new(tx),
7020 },
7021 rx,
7022 )
7023 }
7024
7025 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7026 serde_json::from_slice(&frame.body).unwrap()
7027 }
7028
7029 async fn hello_via_sink(
7034 handler: &ControlHandler,
7035 ctx: &RouteCtx,
7036 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7037 hello: Frame,
7038 ) -> Frame {
7039 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7040 assert!(
7041 replies.is_empty(),
7042 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7043 );
7044 let ack = rx
7045 .try_recv()
7046 .expect("HELLO_ACK is queued on the module sink")
7047 .frame;
7048 assert_eq!(ack.header.ty, FrameType::HelloAck);
7049 ack
7050 }
7051
7052 fn parse_error(frame: &Frame) -> Value {
7053 serde_json::from_slice(&frame.body).unwrap()
7054 }
7055
7056 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7057 serde_json::from_slice(&frame.body).unwrap()
7058 }
7059
7060 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7061 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7062 route_channel,
7063 route_epoch: 0,
7064 kind,
7065 })
7066 .unwrap();
7067 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7068 }
7069
7070 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7071 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7072 module_id: module_id.to_string(),
7073 })
7074 .unwrap();
7075 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7076 }
7077
7078 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7079 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7080 }
7081
7082 fn route_open_frame_with_consumer_capabilities(
7083 corr: u64,
7084 module_id: &str,
7085 project_root: ScratchDir,
7086 consumer_capabilities: Option<Vec<String>>,
7087 ) -> Frame {
7088 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7089 target: RouteTarget::ToolProvider {
7090 module_id: module_id.to_string(),
7091 },
7092 identity: BindIdentity::new(
7093 project_root.path().to_path_buf(),
7094 "unit".to_string(),
7095 "session".to_string(),
7096 ),
7097 consumer_identity: None,
7098 consumer_capabilities,
7099 role_versions: None,
7100 admission_facts: None,
7101 scope: None,
7102 })
7103 .unwrap();
7104 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7105 }
7106
7107 fn route_open_frame_with_role_versions(
7108 corr: u64,
7109 module_id: &str,
7110 project_root: ScratchDir,
7111 role_versions: Option<BTreeMap<String, String>>,
7112 ) -> Frame {
7113 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7114 target: RouteTarget::ToolProvider {
7115 module_id: module_id.to_string(),
7116 },
7117 identity: BindIdentity::new(
7118 project_root.path().to_path_buf(),
7119 "unit".to_string(),
7120 format!("session-{corr}"),
7121 ),
7122 consumer_identity: None,
7123 consumer_capabilities: None,
7124 role_versions,
7125 admission_facts: None,
7126 scope: None,
7127 })
7128 .unwrap();
7129 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7130 }
7131
7132 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
7133 entries
7134 .iter()
7135 .map(|(role, version)| (role.to_string(), version.to_string()))
7136 .collect()
7137 }
7138
7139 fn route_open_frame_with_admission_facts(
7140 corr: u64,
7141 module_id: &str,
7142 project_root: ScratchDir,
7143 consumer_identity: Option<subc_control::ConsumerIdentity>,
7144 facts: Option<Value>,
7145 ) -> Frame {
7146 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7147 target: RouteTarget::ToolProvider {
7148 module_id: module_id.to_string(),
7149 },
7150 identity: BindIdentity::new(
7151 project_root.path().to_path_buf(),
7152 "unit".to_string(),
7153 format!("session-{corr}"),
7154 ),
7155 consumer_identity,
7156 consumer_capabilities: None,
7157 role_versions: None,
7158 admission_facts: facts,
7159 scope: None,
7160 })
7161 .unwrap();
7162 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7163 }
7164
7165 #[derive(Clone, Default)]
7166 struct EventCapture {
7167 events: Arc<Mutex<Vec<CapturedEvent>>>,
7168 }
7169
7170 #[derive(Clone, Debug)]
7171 struct CapturedEvent {
7172 target: String,
7173 level: tracing::Level,
7174 fields: BTreeMap<String, String>,
7175 }
7176
7177 impl EventCapture {
7178 fn events(&self) -> Vec<CapturedEvent> {
7179 self.events.lock().unwrap().clone()
7180 }
7181 }
7182
7183 impl<S> Layer<S> for EventCapture
7184 where
7185 S: Subscriber,
7186 {
7187 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
7188 let mut visitor = EventFieldVisitor::default();
7189 event.record(&mut visitor);
7190 self.events.lock().unwrap().push(CapturedEvent {
7191 target: event.metadata().target().to_string(),
7192 level: *event.metadata().level(),
7193 fields: visitor.fields,
7194 });
7195 }
7196 }
7197
7198 #[derive(Default)]
7199 struct EventFieldVisitor {
7200 fields: BTreeMap<String, String>,
7201 }
7202
7203 impl Visit for EventFieldVisitor {
7204 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
7205 self.fields
7206 .insert(field.name().to_string(), format!("{value:?}"));
7207 }
7208 }
7209
7210 fn health_response(corr: u64, status: HealthStatus) -> Frame {
7211 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
7212 status,
7213 detail: Some("warming".to_string()),
7214 metrics: Some(json!({"queue_depth": 3})),
7215 })
7216 .unwrap();
7217 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7218 }
7219
7220 fn route_bind_ack(corr: u64) -> Frame {
7221 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
7222 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7223 }
7224
7225 fn unique_project_root(label: &str) -> ScratchDir {
7226 ScratchDir::new(label)
7227 }
7228
7229 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
7230 match parse_route_poll(frame) {
7231 ClientControlResponse::RoutePoll {
7232 status: None,
7233 live: Some(live),
7234 ..
7235 } => assert_eq!(live, expected_live),
7236 other => panic!("unexpected route.poll response: {other:?}"),
7237 }
7238 }
7239
7240 fn bind_liveness_route(
7241 registry: &Registry,
7242 forwarding: &ForwardingTable,
7243 module_id: &str,
7244 ) -> (RouteCtx, u16, u32) {
7245 let module_connection = ConnectionId::new(101);
7246 let client_connection = ConnectionId::new(202);
7247 let registration = registry
7248 .register_with_control_ops(
7249 manifest(module_id, PROTOCOL_VERSION),
7250 PROTOCOL_VERSION,
7251 module_connection,
7252 module_baseline_control_ops(),
7253 )
7254 .unwrap();
7255 let (module_tx, _module_rx) = mpsc::channel(8);
7256 let endpoint = forwarding
7257 .register_module_connection(
7258 module_connection,
7259 module_id.to_string(),
7260 PROTOCOL_VERSION,
7261 manifest_concurrency(®istration.manifest),
7262 FrameSink::new(module_tx),
7263 )
7264 .unwrap();
7265 let (client_ctx, _client_rx) = route_ctx(client_connection);
7266 let pending = forwarding
7267 .begin_route_bind_relay_for_test(
7268 client_connection,
7269 client_ctx.egress.clone(),
7270 1,
7271 module_id,
7272 )
7273 .unwrap();
7274 assert_eq!(pending.endpoint, endpoint);
7275 let route_channel = pending.client_channel;
7276 let route_epoch = pending.client_epoch;
7277 forwarding
7278 .complete_pending_relay(
7279 module_connection,
7280 pending.corr,
7281 RouteBindRelayOutcome::Accepted,
7282 )
7283 .unwrap();
7284 (client_ctx, route_channel, route_epoch)
7285 }
7286
7287 struct FakeProcessLiveness {
7288 live: Option<bool>,
7289 }
7290
7291 impl ModuleProcessLiveness for FakeProcessLiveness {
7292 fn process_live(&self, _module_id: &str) -> Option<bool> {
7293 self.live
7294 }
7295 }
7296
7297 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7298 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
7299 {
7300 let registry = Arc::new(Registry::default());
7301 let supervisor_handle = SupervisorHandle::new();
7302 let supervisor = Supervisor::new_for_test(
7303 Arc::clone(®istry),
7304 RestartPolicy::new(1, Duration::from_millis(10)),
7305 )
7306 .with_handle(supervisor_handle.clone());
7307 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
7308 let module = supervisor
7309 .spawn(ModuleSpec {
7310 module_id: "stderr-tail-wire".to_string(),
7311 program: fake_aft_stub_path(),
7312 args: Vec::new(),
7313 env: vec![
7314 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
7315 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
7316 ],
7317 reserved: false,
7318 reserved_prefixes: Vec::new(),
7319 protocol: ModuleProtocol::Subc,
7320 overlap: Default::default(),
7321 })
7322 .unwrap();
7323
7324 let deadline = Instant::now() + Duration::from_secs(5);
7325 loop {
7326 let tail = module.stderr_tail(None, None);
7327 if tail
7328 .entries
7329 .iter()
7330 .any(|entry| matches!(entry, TailEntry::ProcessStart))
7331 && tail.entries.iter().any(|entry| {
7332 matches!(
7333 entry,
7334 TailEntry::Line {
7335 truncated: true,
7336 ..
7337 }
7338 )
7339 })
7340 {
7341 break;
7342 }
7343 assert!(
7344 Instant::now() < deadline,
7345 "module did not produce a truncated line and restart boundary: {tail:?}"
7346 );
7347 sleep(Duration::from_millis(10)).await;
7348 }
7349
7350 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7351 let request = ClientControlRequest::SupervisorStderrTail {
7352 module_id: "stderr-tail-wire".to_string(),
7353 max_lines: None,
7354 max_bytes: None,
7355 };
7356 let frame = Frame::build(
7357 FrameType::Request,
7358 control_flags(),
7359 0,
7360 0,
7361 1,
7362 serde_json::to_vec(&request).unwrap(),
7363 )
7364 .unwrap();
7365 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7366 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7367 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
7368 serde_json::from_slice(&responses[0].body).unwrap()
7369 else {
7370 panic!("expected supervisor.stderr_tail response");
7371 };
7372
7373 assert!(
7374 tail.entries
7375 .iter()
7376 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
7377 "the control response lost the restart boundary"
7378 );
7379 let Some(StderrTailEntry::Line {
7380 text,
7381 truncated,
7382 at_ms,
7383 }) = tail.entries.iter().find(|entry| {
7384 matches!(
7385 entry,
7386 StderrTailEntry::Line {
7387 truncated: true,
7388 ..
7389 }
7390 )
7391 })
7392 else {
7393 panic!("the control response lost the truncated line");
7394 };
7395 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
7396 assert!(*truncated);
7397 assert!(
7398 at_ms.is_some(),
7399 "the control response lost the line's capture time"
7400 );
7401 }
7402
7403 #[tokio::test(flavor = "current_thread")]
7408 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
7409 let dir = ScratchDir::new("terminals-off-worker");
7410 let journal_path = dir.join("terminals.jsonl");
7411 let registry = Arc::new(Registry::default());
7412 let supervisor_handle = SupervisorHandle::new();
7413 let supervisor =
7414 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7415 .with_handle(supervisor_handle.clone())
7416 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7417 let module = supervisor
7418 .spawn(ModuleSpec {
7419 module_id: "terminal-off-worker".to_string(),
7420 program: fake_aft_stub_path(),
7421 args: Vec::new(),
7422 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7423 reserved: false,
7424 reserved_prefixes: Vec::new(),
7425 protocol: ModuleProtocol::Subc,
7426 overlap: Default::default(),
7427 })
7428 .unwrap();
7429 let deadline = Instant::now() + Duration::from_secs(5);
7430 while module.terminal_history().entries.len() != 2 {
7431 assert!(Instant::now() < deadline, "module did not record two exits");
7432 sleep(Duration::from_millis(10)).await;
7433 }
7434
7435 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
7436 let handler =
7437 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
7438 let frame = Frame::build(
7439 FrameType::Request,
7440 control_flags(),
7441 0,
7442 0,
7443 1,
7444 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
7445 module_id: "terminal-off-worker".to_string(),
7446 })
7447 .unwrap(),
7448 )
7449 .unwrap();
7450 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7451 let spawned_at = std::time::Instant::now();
7452 let read = tokio::spawn({
7453 let handler = Arc::clone(&handler);
7454 async move { handler.handle_control_frame(&ctx, frame).await }
7455 });
7456 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
7459 .await
7460 .unwrap()
7461 .expect("the history read reached its pause");
7462 let elapsed = spawned_at.elapsed();
7463 assert!(
7464 elapsed < Duration::from_secs(2) && !read.is_finished(),
7465 "this task could not run while the history read was paused \
7466 (resumed after {elapsed:?}, read finished: {})",
7467 read.is_finished()
7468 );
7469
7470 drop(release);
7471 let responses = read.await.unwrap().unwrap();
7472 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7473 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
7474 panic!("expected supervisor.terminals response");
7475 };
7476 assert_eq!(terminals.entries.len(), 2);
7477 assert_eq!(terminals.journal_skipped_lines, 0);
7478 assert_eq!(terminals.journal_read_errors, 0);
7479 }
7480
7481 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7482 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
7483 let registry = Arc::new(Registry::default());
7484 let supervisor_handle = SupervisorHandle::new();
7485 let supervisor =
7486 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7487 .with_handle(supervisor_handle.clone());
7488 let module = supervisor
7489 .spawn(ModuleSpec {
7490 module_id: "terminal-golden".to_string(),
7491 program: fake_aft_stub_path(),
7492 args: Vec::new(),
7493 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7494 reserved: false,
7495 reserved_prefixes: Vec::new(),
7496 protocol: ModuleProtocol::Subc,
7497 overlap: Default::default(),
7498 })
7499 .unwrap();
7500
7501 let deadline = Instant::now() + Duration::from_secs(5);
7502 while module.terminal_history().entries.len() != 2 {
7503 assert!(
7504 Instant::now() < deadline,
7505 "module did not retain two terminal exits: {:?}",
7506 module.terminal_history()
7507 );
7508 sleep(Duration::from_millis(10)).await;
7509 }
7510
7511 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7512 let request = ClientControlRequest::SupervisorTerminals {
7513 module_id: "terminal-golden".to_string(),
7514 };
7515 let frame = Frame::build(
7516 FrameType::Request,
7517 control_flags(),
7518 0,
7519 0,
7520 1,
7521 serde_json::to_vec(&request).unwrap(),
7522 )
7523 .unwrap();
7524 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7525 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7526 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7527 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
7528 panic!("expected supervisor.terminals response");
7529 };
7530 assert_eq!(terminals.entries.len(), 2);
7531 assert_eq!(terminals.dropped, 0);
7532
7533 let mut rendered = serde_json::to_value(response).unwrap();
7534 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
7537 for (index, entry) in rendered["entries"]
7538 .as_array_mut()
7539 .expect("terminal response entries array")
7540 .iter_mut()
7541 .enumerate()
7542 {
7543 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
7544 }
7545
7546 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7547 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
7548 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
7549 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7550 std::fs::write(&golden_path, &serialized).unwrap();
7551 }
7552 let expected: Value =
7553 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
7554 assert_eq!(rendered, expected);
7555 }
7556
7557 #[test]
7558 fn hello_registers_manifest_and_returns_ack() {
7559 let registry = Arc::new(Registry::default());
7560 let handler = ControlHandler::new(Arc::clone(®istry));
7561 let conn = ConnectionId::new(1);
7562
7563 let responses = handler
7564 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
7565 .unwrap();
7566
7567 assert_eq!(responses.len(), 1);
7568 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7569 assert_eq!(responses[0].header.channel, 0);
7570 assert_eq!(responses[0].header.corr, 7);
7571 let ack = parse_ack(&responses[0]);
7572 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
7573 assert!(ack
7574 .subc_capabilities
7575 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
7576 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
7577 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
7578 assert!(ack
7579 .subc_ops
7580 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
7581 assert!(ack
7582 .subc_ops
7583 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
7584
7585 let registration = registry.get_module("aft").unwrap().unwrap();
7586 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
7587 assert_eq!(registration.state, ChannelState::Active);
7588 assert_eq!(registration.connection_id, conn);
7589 assert_eq!(registration.control_ops, module_baseline_control_ops());
7590 }
7591
7592 #[test]
7593 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7594 let invalid_identifiers = [
7595 ("case_change", "credentials-Provider/v1"),
7596 ("leading_zero", "credentials-provider/v01"),
7597 ("trailing_hyphen", "credentials-provider-/v1"),
7598 ("consecutive_hyphens", "credentials--provider/v1"),
7599 ("uppercase", "Credentials-provider/v1"),
7600 ("missing_v", "credentials-provider/1"),
7601 ("whitespace", "credentials provider/v1"),
7602 ("zero_version", "credentials-provider/v0"),
7603 ("out_of_range_version", "credentials-provider/v4294967296"),
7604 (
7605 "overlength_name",
7606 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7607 ),
7608 ];
7609 let mut cases = invalid_identifiers
7610 .into_iter()
7611 .map(|(name, identifier)| {
7612 (
7613 format!("identifier_{name}"),
7614 "capabilities.provides[0]".to_string(),
7615 identifier.to_string(),
7616 json!({ "provides": [identifier] }),
7617 None,
7618 )
7619 })
7620 .collect::<Vec<_>>();
7621 cases.extend([
7622 (
7623 "unknown_need".to_string(),
7624 "capabilities.requires[0].need".to_string(),
7625 "deferred".to_string(),
7626 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7627 None,
7628 ),
7629 (
7630 "duplicate_provides".to_string(),
7631 "capabilities.provides[1]".to_string(),
7632 "credentials-provider/v1".to_string(),
7633 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7634 None,
7635 ),
7636 (
7637 "duplicate_must_never_reach".to_string(),
7638 "capabilities.must_never_reach[1]".to_string(),
7639 "credentials-provider/v1".to_string(),
7640 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7641 None,
7642 ),
7643 (
7644 "duplicate_requires_same_need".to_string(),
7645 "capabilities.requires[1]".to_string(),
7646 "credentials-provider/v1".to_string(),
7647 json!({ "requires": [
7648 { "capability": "credentials-provider/v1", "need": "required" },
7649 { "capability": "credentials-provider/v1", "need": "required" }
7650 ] }),
7651 None,
7652 ),
7653 (
7654 "duplicate_requires_conflicting_need".to_string(),
7655 "capabilities.requires[1]".to_string(),
7656 "credentials-provider/v1".to_string(),
7657 json!({ "requires": [
7658 { "capability": "credentials-provider/v1", "need": "required" },
7659 { "capability": "credentials-provider/v1", "need": "optional" }
7660 ] }),
7661 None,
7662 ),
7663 (
7664 "capabilities_root_pointer".to_string(),
7665 "runtime_computed[0]".to_string(),
7666 "/capabilities".to_string(),
7667 json!({}),
7668 Some(json!(["/capabilities"])),
7669 ),
7670 (
7671 "capabilities_descendant_pointer".to_string(),
7672 "runtime_computed[0]".to_string(),
7673 "/capabilities/provides".to_string(),
7674 json!({}),
7675 Some(json!(["/capabilities/provides"])),
7676 ),
7677 (
7678 "malformed_pointer_without_leading_slash".to_string(),
7679 "runtime_computed[0]".to_string(),
7680 "capabilities".to_string(),
7681 json!({}),
7682 Some(json!(["capabilities"])),
7683 ),
7684 (
7685 "malformed_pointer_escape".to_string(),
7686 "runtime_computed[0]".to_string(),
7687 "/roles/~2/tools".to_string(),
7688 json!({}),
7689 Some(json!(["/roles/~2/tools"])),
7690 ),
7691 (
7692 "unknown_capabilities_field".to_string(),
7693 "capabilities.future".to_string(),
7694 "<array>".to_string(),
7695 json!({ "future": [] }),
7696 None,
7697 ),
7698 ]);
7699
7700 for (index, (name, field, value, capabilities, runtime_computed)) in
7701 cases.into_iter().enumerate()
7702 {
7703 let registry = Arc::new(Registry::default());
7704 let handler = ControlHandler::new(Arc::clone(®istry));
7705 let response = handler
7706 .handle_control(
7707 ConnectionId::new((index + 1) as u64),
7708 capability_grammar_hello_frame(
7709 capabilities,
7710 runtime_computed,
7711 index as u64 + 1,
7712 ),
7713 )
7714 .expect("invalid HELLO returns a refusal");
7715
7716 assert_eq!(response.len(), 1, "{name} must emit one refusal");
7717 let error = parse_error(&response[0]);
7718 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7719 let message = error["message"]
7720 .as_str()
7721 .expect("error message is a string");
7722 assert!(
7723 message.contains(&field),
7724 "{name}: field missing from {message}"
7725 );
7726 assert!(
7727 message.contains(&value),
7728 "{name}: value missing from {message}"
7729 );
7730 assert_eq!(
7731 registry
7732 .active_registration_count()
7733 .expect("registry reads"),
7734 0,
7735 "{name}: refused HELLO must not create a catalog entry"
7736 );
7737 }
7738 }
7739
7740 #[test]
7741 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7742 let registry = Arc::new(Registry::default());
7743 let handler = ControlHandler::new(Arc::clone(®istry));
7744 let capabilities = json!({
7745 "provides": ["credentials-provider/v1"],
7746 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7747 "must_never_reach": ["federation-transport/v1"]
7748 });
7749 let response = handler
7750 .handle_control(
7751 ConnectionId::new(99),
7752 capability_grammar_hello_frame(
7753 capabilities.clone(),
7754 Some(json!(["/roles/0/tools"])),
7755 99,
7756 ),
7757 )
7758 .expect("valid HELLO registers");
7759 assert_eq!(response[0].header.ty, FrameType::HelloAck);
7760
7761 let request = Frame::build(
7762 FrameType::Request,
7763 control_flags(),
7764 0,
7765 0,
7766 100,
7767 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7768 .expect("catalog request serializes"),
7769 )
7770 .expect("catalog request frame builds");
7771 let response = handler
7772 .handle_catalog_list(request, None)
7773 .expect("catalog list succeeds");
7774 let ClientControlResponse::CatalogList { modules, .. } =
7775 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7776 else {
7777 panic!("catalog request must return catalog.list");
7778 };
7779 assert_eq!(modules.len(), 1);
7780 assert_eq!(
7781 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7782 capabilities
7783 );
7784 }
7785
7786 #[test]
7787 fn catalog_list_mirrors_management_operation_description() {
7788 let registry = Arc::new(Registry::default());
7789 let handler = ControlHandler::new(Arc::clone(®istry));
7790 let description = "List managed records and return their identifiers and metadata.";
7791 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7792 manifest.provides = vec![ProviderRole::ManagementSurface {
7793 operations: vec![ManagementOperation {
7794 name: "records.list".to_string(),
7795 kind: ManagementOperationKind::Query,
7796 description: Some(description.to_string()),
7797 }],
7798 config_schema: json!({"type": "object"}),
7799 observability: vec![ObservabilitySurface {
7800 name: "records.stats".to_string(),
7801 kind: ObservabilityKind::Snapshot,
7802 }],
7803 identity_scope: vec![IdentityScope::Project],
7804 concurrency: Concurrency::ModuleManaged,
7805 }];
7806 registry
7807 .register_with_control_ops(
7808 manifest,
7809 PROTOCOL_VERSION,
7810 ConnectionId::new(99),
7811 Vec::new(),
7812 )
7813 .expect("described management manifest registers");
7814
7815 let request = Frame::build(
7816 FrameType::Request,
7817 control_flags(),
7818 0,
7819 0,
7820 100,
7821 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7822 .expect("catalog request serializes"),
7823 )
7824 .expect("catalog request frame builds");
7825 let response = handler
7826 .handle_catalog_list(request, None)
7827 .expect("catalog list succeeds");
7828 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7829 assert_eq!(
7830 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7831 "catalog.list must preserve the declared operation description verbatim"
7832 );
7833 }
7834
7835 #[test]
7836 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7837 let registry = Arc::new(Registry::default());
7838 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7839 [("vault".to_string(), true), ("squatter".to_string(), true)],
7840 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7841 );
7842 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7843 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7844 provides: vec!["credentials-provider/v1".to_string()],
7845 requires: Vec::new(),
7846 must_never_reach: Vec::new(),
7847 });
7848 let frame = Frame::build(
7849 FrameType::Hello,
7850 control_flags(),
7851 0,
7852 0,
7853 77,
7854 serde_json::to_vec(&ModuleHelloBody {
7855 manifest: squatter,
7856 protocol_ver: PROTOCOL_VERSION,
7857 control_ops: None,
7858 launch_nonce: None,
7859 })
7860 .expect("HELLO serializes"),
7861 )
7862 .expect("HELLO frame builds");
7863 let response = handler
7864 .handle_control(ConnectionId::new(77), frame)
7865 .expect("reserved claim receives a typed refusal");
7866 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7867 assert_eq!(
7868 registry
7869 .active_registration_count()
7870 .expect("registry reads"),
7871 0,
7872 "a reserved capability refusal must not leave a catalog entry"
7873 );
7874 }
7875
7876 #[test]
7877 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
7878 for settlement in ["timeout", "inconclusive", "drop"] {
7879 let breakers = RouteBindBreakers::default();
7880 let RouteBindAdmission::Admitted {
7881 guard: mut old,
7882 probe: false,
7883 } = breakers.admit("prov")
7884 else {
7885 panic!("ordinary relay admitted")
7886 };
7887 let RouteBindAdmission::Admitted {
7888 guard: mut opener, ..
7889 } = breakers.admit("prov")
7890 else {
7891 panic!("second relay admitted")
7892 };
7893 assert!(
7894 !opener
7895 .record_timeout(1, Duration::ZERO)
7896 .unwrap()
7897 .reopened_after_probe
7898 );
7899 let RouteBindAdmission::Admitted {
7900 guard: mut probe,
7901 probe: true,
7902 } = breakers.admit("prov")
7903 else {
7904 panic!("one cooldown probe admitted")
7905 };
7906 match settlement {
7907 "timeout" => assert!(
7908 !old.record_timeout(1, Duration::ZERO)
7909 .unwrap()
7910 .reopened_after_probe
7911 ),
7912 "inconclusive" => old.record_inconclusive(),
7913 "drop" => drop(old),
7914 _ => unreachable!(),
7915 }
7916 assert!(
7917 matches!(
7918 breakers.admit("prov"),
7919 RouteBindAdmission::Refused {
7920 probe_in_flight: true,
7921 ..
7922 }
7923 ),
7924 "{settlement} of a pre-open relay cannot release the real probe"
7925 );
7926 assert!(
7927 probe
7928 .record_timeout(1, Duration::ZERO)
7929 .unwrap()
7930 .reopened_after_probe
7931 );
7932 assert!(matches!(
7933 breakers.admit("prov"),
7934 RouteBindAdmission::Admitted { probe: true, .. }
7935 ));
7936 }
7937 let breakers = RouteBindBreakers::default();
7938 let admit = || match breakers.admit("prov") {
7939 RouteBindAdmission::Admitted { guard, .. } => guard,
7940 _ => panic!("relay admitted"),
7941 };
7942 admit().record_timeout(1, Duration::ZERO);
7943 let mut old_probe = admit();
7944 breakers.reset_for_new_module_connection("prov");
7945 admit().record_timeout(1, Duration::ZERO);
7946 let _new_probe = admit();
7947 old_probe.record_inconclusive();
7948 assert!(matches!(
7949 breakers.admit("prov"),
7950 RouteBindAdmission::Refused {
7951 probe_in_flight: true,
7952 ..
7953 }
7954 ));
7955 }
7956
7957 #[tokio::test]
7958 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
7959 for candidate in [false, true] {
7960 let registry = Arc::new(Registry::default());
7961 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7962 [("vault".to_string(), true), ("squatter".to_string(), true)],
7963 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7964 );
7965 let conn = ConnectionId::new(77);
7966 let (ctx, mut rx) = route_ctx(conn);
7967 let initial = capability_manifest("squatter", &[], &[]);
7968 if candidate {
7969 registry
7970 .register_candidate_with_control_ops(
7971 initial.clone(),
7972 PROTOCOL_VERSION,
7973 conn,
7974 module_baseline_control_ops(),
7975 )
7976 .unwrap();
7977 handler
7978 .forwarding
7979 .register_candidate_module_connection(
7980 conn,
7981 "squatter".to_string(),
7982 PROTOCOL_VERSION,
7983 manifest_concurrency(&initial),
7984 ctx.egress.clone(),
7985 )
7986 .unwrap();
7987 } else {
7988 hello_via_sink(
7989 &handler,
7990 &ctx,
7991 &mut rx,
7992 hello_frame_with_manifest(initial.clone(), 1),
7993 )
7994 .await;
7995 }
7996 let response = handler
7997 .handle_control_frame(
7998 &ctx,
7999 catalog_update_with_capabilities_frame(
8000 2,
8001 capability_manifest("squatter", &["credentials-provider/v1"], &[])
8002 .capabilities
8003 .unwrap(),
8004 ),
8005 )
8006 .await
8007 .unwrap();
8008 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8009 assert_eq!(
8010 registry
8011 .get_module_by_connection(conn)
8012 .unwrap()
8013 .unwrap()
8014 .manifest,
8015 initial
8016 );
8017 }
8018 }
8019
8020 #[test]
8021 fn server_describe_surfaces_required_capability_verdict_fields() {
8022 let registry = Arc::new(Registry::default());
8023 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8024 [
8025 ("consumer".to_string(), true),
8026 ("provider".to_string(), false),
8027 ],
8028 BTreeMap::new(),
8029 );
8030 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
8031 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8032 provides: Vec::new(),
8033 requires: vec![subc_protocol::manifest::CapabilityRequirement {
8034 capability: "credentials-provider/v1".to_string(),
8035 need: subc_protocol::manifest::CapabilityNeed::Required,
8036 }],
8037 must_never_reach: Vec::new(),
8038 });
8039 let hello = Frame::build(
8040 FrameType::Hello,
8041 control_flags(),
8042 0,
8043 0,
8044 78,
8045 serde_json::to_vec(&ModuleHelloBody {
8046 manifest: consumer,
8047 protocol_ver: PROTOCOL_VERSION,
8048 control_ops: None,
8049 launch_nonce: None,
8050 })
8051 .expect("HELLO serializes"),
8052 )
8053 .expect("HELLO frame builds");
8054 handler
8055 .handle_control(ConnectionId::new(78), hello)
8056 .expect("consumer registers");
8057 let describe = Frame::build(
8058 FrameType::Request,
8059 control_flags(),
8060 0,
8061 0,
8062 79,
8063 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8064 .expect("request serializes"),
8065 )
8066 .expect("describe frame builds");
8067 let response = handler
8068 .handle_server_describe(describe)
8069 .expect("server.describe succeeds");
8070 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8071 let requirement = &rendered["capability_requirements"][0];
8072 assert_eq!(requirement["consumer"], "consumer");
8073 assert_eq!(requirement["verdict"], "never_provided");
8074 assert_eq!(requirement["episode_seq"], 1);
8075 assert_eq!(requirement["config_satisfiable"], false);
8076 assert_eq!(requirement["runtime_available"], false);
8077 assert!(requirement["detail"]
8078 .as_str()
8079 .expect("detail string")
8080 .contains("credentials-provider/v1"));
8081 }
8082
8083 #[test]
8084 fn catalog_list_omits_capabilities_for_legacy_manifest() {
8085 let registry = Arc::new(Registry::default());
8086 let handler = ControlHandler::new(Arc::clone(®istry));
8087 let hello = handler
8088 .handle_control(
8089 ConnectionId::new(101),
8090 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
8091 )
8092 .expect("legacy HELLO registers");
8093 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
8094
8095 let request = Frame::build(
8096 FrameType::Request,
8097 control_flags(),
8098 0,
8099 0,
8100 102,
8101 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8102 .expect("catalog request serializes"),
8103 )
8104 .expect("catalog request frame builds");
8105 let response = handler
8106 .handle_catalog_list(request, None)
8107 .expect("catalog list succeeds");
8108 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8109 assert!(
8110 body["modules"][0].get("capabilities").is_none(),
8111 "legacy manifest must retain an absent capabilities field on catalog.list"
8112 );
8113 }
8114
8115 #[test]
8116 fn hello_ack_omits_storage_when_no_storage_config() {
8117 let registry = Arc::new(Registry::default());
8118 let handler = ControlHandler::new(Arc::clone(®istry));
8119 let responses = handler
8120 .handle_control(
8121 ConnectionId::new(1),
8122 hello_frame("aft", PROTOCOL_VERSION, 7),
8123 )
8124 .unwrap();
8125 let ack = parse_ack(&responses[0]);
8126 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
8127 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
8128 }
8129
8130 #[tokio::test]
8131 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
8132 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
8133 let registry = Arc::new(Registry::default());
8134 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
8135 let responses = handler
8136 .handle_control(
8137 ConnectionId::new(1),
8138 hello_frame("aft", PROTOCOL_VERSION, 7),
8139 )
8140 .unwrap();
8141 let ack = parse_ack(&responses[0]);
8142 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
8143
8144 let described = handler
8145 .handle_control_frame(
8146 &route_ctx(ConnectionId::new(2)).0,
8147 Frame::build(
8148 FrameType::Request,
8149 control_flags(),
8150 0,
8151 0,
8152 9,
8153 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8154 )
8155 .unwrap(),
8156 )
8157 .await
8158 .unwrap();
8159 let ClientControlResponse::ServerDescribe { machine_id, .. } =
8160 serde_json::from_slice(&described[0].body).unwrap()
8161 else {
8162 panic!("server.describe answered with another shape");
8163 };
8164 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
8165 }
8166
8167 #[test]
8168 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
8169 let registry = Arc::new(Registry::default());
8172 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
8173 crate::daemon_config::StorageConfig::Sqlite {
8174 data_home: std::path::PathBuf::from("/data"),
8175 },
8176 ));
8177
8178 let responses = handler
8179 .handle_control(
8180 ConnectionId::new(1),
8181 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
8182 )
8183 .unwrap();
8184 let ack = parse_ack(&responses[0]);
8185 assert_eq!(
8186 ack.storage,
8187 Some(serde_json::json!({
8188 "module_id": "alfonso-routing",
8189 "storage_namespace": "default",
8190 "isolation": { "kind": "module" },
8191 "backend": {
8192 "backend": "sqlite",
8193 "path": "/data/cortexkit/alfonso-routing/store.db"
8194 }
8195 })),
8196 "the delivered descriptor is the module's own sqlite store path"
8197 );
8198 }
8199
8200 #[test]
8201 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
8202 let registry = Arc::new(Registry::default());
8203 let handler = ControlHandler::new(Arc::clone(®istry));
8204 let conn = ConnectionId::new(1);
8205 let responses = handler
8206 .handle_control(
8207 conn,
8208 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8209 )
8210 .unwrap();
8211 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8212 let registration = registry.get_module("aft").unwrap().unwrap();
8213 assert_eq!(registration.control_ops, module_baseline_control_ops());
8214
8215 let frame =
8216 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
8217 assert!(handler
8218 .guard_module_control_op(&frame, "aft", "route.bind")
8219 .unwrap()
8220 .is_none());
8221 let error = handler
8222 .guard_module_control_op(&frame, "aft", "test.synthetic")
8223 .unwrap()
8224 .expect("synthetic ungranted op should be rejected");
8225 assert_eq!(error.header.ty, FrameType::Error);
8226 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
8227 }
8228
8229 #[test]
8230 fn hello_control_ops_some_adds_optional_grants() {
8231 let registry = Arc::new(Registry::default());
8232 let handler = ControlHandler::new(Arc::clone(®istry));
8233 handler
8234 .handle_control(
8235 ConnectionId::new(1),
8236 hello_frame_with_control_ops(
8237 "aft",
8238 PROTOCOL_VERSION,
8239 7,
8240 Some(vec![
8241 "future.synthetic".to_string(),
8242 "route.bind".to_string(),
8243 ]),
8244 ),
8245 )
8246 .unwrap();
8247 let registration = registry.get_module("aft").unwrap().unwrap();
8248 assert_eq!(
8249 registration.control_ops,
8250 vec![
8251 "route.bind".to_string(),
8252 "route.status".to_string(),
8253 "future.synthetic".to_string(),
8254 ]
8255 );
8256 let frame =
8257 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
8258 assert!(handler
8259 .guard_module_control_op(&frame, "aft", "future.synthetic")
8260 .unwrap()
8261 .is_none());
8262 }
8263
8264 #[tokio::test]
8265 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
8266 let registry = Arc::new(Registry::default());
8267 let forwarding = Arc::new(ForwardingTable::default());
8268 let handler =
8269 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8270 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
8271 hello_via_sink(
8272 &handler,
8273 &module_ctx,
8274 &mut module_rx,
8275 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8276 )
8277 .await;
8278
8279 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
8280 let responses = handler
8281 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
8282 .await
8283 .unwrap();
8284 assert_eq!(responses.len(), 1);
8285 assert_eq!(responses[0].header.ty, FrameType::Error);
8286 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
8287 assert!(module_rx.try_recv().is_err());
8288 }
8289
8290 #[tokio::test]
8291 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
8292 let registry = Arc::new(Registry::default());
8293 let forwarding = Arc::new(ForwardingTable::default());
8294 let handler =
8295 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8296 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
8297 hello_via_sink(
8298 &handler,
8299 &module_ctx,
8300 &mut module_rx,
8301 hello_frame_with_control_ops(
8302 "aft",
8303 PROTOCOL_VERSION,
8304 7,
8305 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8306 ),
8307 )
8308 .await;
8309
8310 let project_root = unique_project_root("demux");
8311 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
8312 let route_handler = handler.clone();
8313 let route_task = tokio::spawn(async move {
8314 route_handler
8315 .handle_control_frame(
8316 &route_client_ctx,
8317 route_open_frame(100, "aft", project_root),
8318 )
8319 .await
8320 .unwrap()
8321 });
8322 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8323 .await
8324 .unwrap()
8325 .unwrap();
8326 assert!(matches!(
8327 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
8328 ModuleControlRequest::RouteBind { .. }
8329 ));
8330
8331 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
8332 let health_handler = handler.clone();
8333 let health_task = tokio::spawn(async move {
8334 health_handler
8335 .handle_control_frame(
8336 &health_client_ctx,
8337 supervisor_health_probe_frame(101, "aft"),
8338 )
8339 .await
8340 .unwrap()
8341 });
8342 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8343 .await
8344 .unwrap()
8345 .unwrap();
8346 assert_eq!(
8347 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8348 ModuleControlRequest::HealthCheck {}
8349 );
8350
8351 handler
8352 .handle_control_frame(
8353 &module_ctx,
8354 health_response(health_frame.header.corr, HealthStatus::Degraded),
8355 )
8356 .await
8357 .unwrap();
8358 let health_response = health_task.await.unwrap();
8359 assert_eq!(health_response.len(), 1);
8360 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8361 ClientControlResponse::SupervisorHealthProbe {
8362 module_id,
8363 status,
8364 detail,
8365 metrics,
8366 } => {
8367 assert_eq!(module_id, "aft");
8368 assert_eq!(status, HealthStatus::Degraded);
8369 assert_eq!(detail.as_deref(), Some("warming"));
8370 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
8371 }
8372 other => panic!("unexpected health response: {other:?}"),
8373 }
8374
8375 handler
8376 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8377 .await
8378 .unwrap();
8379 let route_response = route_task.await.unwrap();
8380 assert!(route_response.is_empty());
8381 let published = route_client_rx.recv().await.unwrap();
8382 assert!(matches!(
8383 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8384 ClientControlResponse::RouteOpen { .. }
8385 ));
8386 }
8387
8388 async fn relay_route_open(
8393 handler: &ControlHandler,
8394 client_connection: ConnectionId,
8395 client_egress: &FrameSink,
8396 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8397 corr: u64,
8398 module_id: &str,
8399 project_root_label: &str,
8400 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
8401 let ctx = RouteCtx {
8402 connection_id: client_connection,
8403 egress: client_egress.clone(),
8404 };
8405 let handler = handler.clone();
8406 let project_root = unique_project_root(project_root_label);
8407 let module_id = module_id.to_string();
8408 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
8409 let task = tokio::spawn(async move {
8410 let _guard = tracing::dispatcher::set_default(&dispatch);
8411 handler
8412 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8413 .await
8414 .unwrap()
8415 });
8416 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8417 .await
8418 .expect("module receives the relayed route.bind")
8419 .expect("module egress is open");
8420 (task, bind.frame)
8421 }
8422
8423 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8424 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8425 ModuleControlRequest::RouteBind {
8426 route_channel,
8427 epoch,
8428 ..
8429 } => (route_channel, epoch),
8430 other => panic!("expected a route.bind request, got {other:?}"),
8431 }
8432 }
8433
8434 fn published_route(frame: &Frame) -> (u16, u32) {
8435 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
8436 ClientControlResponse::RouteOpen {
8437 route_channel,
8438 route_epoch,
8439 } => (route_channel, route_epoch),
8440 other => panic!("expected a route.open response, got {other:?}"),
8441 }
8442 }
8443
8444 #[tokio::test]
8464 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
8465 let registry = Arc::new(Registry::default());
8466 let forwarding = Arc::new(ForwardingTable::default());
8467 let handler =
8468 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8469
8470 let module_connection = ConnectionId::new(30);
8471 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8472 hello_via_sink(
8473 &handler,
8474 &module_ctx,
8475 &mut module_rx,
8476 hello_frame("aft", PROTOCOL_VERSION, 7),
8477 )
8478 .await;
8479
8480 let dying_client = ConnectionId::new(31);
8481 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
8482
8483 let (first_task, first_bind) = relay_route_open(
8486 &handler,
8487 dying_client,
8488 &dying_ctx.egress,
8489 &mut module_rx,
8490 100,
8491 "aft",
8492 "closing-first",
8493 )
8494 .await;
8495 handler
8496 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
8497 .await
8498 .unwrap();
8499 assert!(first_task.await.unwrap().is_empty());
8500 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
8501
8502 let (second_task, second_bind) = relay_route_open(
8504 &handler,
8505 dying_client,
8506 &dying_ctx.egress,
8507 &mut module_rx,
8508 101,
8509 "aft",
8510 "closing-second",
8511 )
8512 .await;
8513 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
8514
8515 assert!(forwarding
8518 .escalate_client_delivery_failure(
8519 dying_client,
8520 first_channel,
8521 first_epoch,
8522 CloseReason::new(
8523 "module_to_client_delivery_failed",
8524 "client egress refused a module frame",
8525 ),
8526 crate::forwarding::UndeliveredFrame {
8527 module_id: None,
8528 sink: &dying_ctx.egress,
8529 },
8530 )
8531 .unwrap());
8532 assert!(!dying_ctx.egress.is_closed());
8533
8534 let ack = handler
8536 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
8537 .await;
8538 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
8539 if module_loop_error.is_some() {
8540 handler.cleanup_connection(module_connection).unwrap();
8544 }
8545 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8549 .await
8550 .ok()
8551 .flatten();
8552
8553 assert!(
8555 registry
8556 .get_module_by_connection(module_connection)
8557 .unwrap()
8558 .is_some(),
8559 "one client's closing connection ended the shared module connection: \
8560 {module_loop_error:?}"
8561 );
8562 let cotenant = ConnectionId::new(32);
8564 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
8565 let (cotenant_task, cotenant_bind) = relay_route_open(
8566 &handler,
8567 cotenant,
8568 &cotenant_ctx.egress,
8569 &mut module_rx,
8570 102,
8571 "aft",
8572 "closing-cotenant",
8573 )
8574 .await;
8575 handler
8576 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
8577 .await
8578 .unwrap();
8579 assert!(cotenant_task.await.unwrap().is_empty());
8580 let (cotenant_channel, cotenant_epoch) =
8581 published_route(&cotenant_rx.recv().await.unwrap());
8582 assert!(matches!(
8583 forwarding
8584 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
8585 .unwrap(),
8586 DataRoute::Client(DataRouteState::Bound(_))
8587 ));
8588
8589 let goodbye = post_ack_module_frame
8592 .expect("module receives a GOODBYE for the abandoned route channel");
8593 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8594 assert_eq!(goodbye.header.channel, abandoned_channel);
8595 assert_eq!(goodbye.header.epoch, abandoned_epoch);
8596
8597 assert!(dying_rx.try_recv().is_err());
8601 let second_response = second_task.await.unwrap();
8602 assert_eq!(second_response.len(), 1);
8603 assert_eq!(
8604 parse_error(&second_response[0])["code"],
8605 "target_unavailable"
8606 );
8607 }
8608
8609 async fn route_open_held_after_bind_send() -> (
8613 ControlHandler,
8614 Arc<tokio::sync::Barrier>,
8615 RouteCtx,
8616 mpsc::Receiver<crate::router::OutboundFrame>,
8617 ) {
8618 let hold = Arc::new(tokio::sync::Barrier::new(2));
8619 let handler = ControlHandler::with_forwarding(
8620 Arc::new(Registry::default()),
8621 Arc::new(ForwardingTable::default()),
8622 )
8623 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
8624 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
8625 hello_via_sink(
8626 &handler,
8627 &module_ctx,
8628 &mut module_rx,
8629 hello_frame("aft", PROTOCOL_VERSION, 7),
8630 )
8631 .await;
8632 (handler, hold, module_ctx, module_rx)
8633 }
8634
8635 #[tokio::test]
8642 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
8643 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8644 let client = ConnectionId::new(41);
8645 let (client_ctx, mut client_rx) = route_ctx(client);
8646
8647 let (open_task, bind) = relay_route_open(
8648 &handler,
8649 client,
8650 &client_ctx.egress,
8651 &mut module_rx,
8652 200,
8653 "aft",
8654 "acked-before-mark",
8655 )
8656 .await;
8657 let (module_channel, module_epoch) = route_bind_channel(&bind);
8658 hold.wait().await;
8660
8661 assert!(handler
8663 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
8664 .await
8665 .unwrap()
8666 .is_empty());
8667 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
8668
8669 hold.wait().await;
8670 assert!(
8671 open_task.await.unwrap().is_empty(),
8672 "route.open answers accepted"
8673 );
8674
8675 match module_rx.try_recv() {
8678 Err(mpsc::error::TryRecvError::Empty) => {}
8679 Ok(outbound) => panic!(
8680 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
8681 outbound.frame.header.ty,
8682 outbound.frame.header.channel,
8683 outbound.frame.header.epoch
8684 ),
8685 Err(err) => panic!("module egress closed: {err}"),
8686 }
8687
8688 let router = Router::with_control_handler(Arc::new(handler.clone()));
8691 let request = Frame::build(
8692 FrameType::Request,
8693 Flags::new(true, Priority::Interactive, false),
8694 client_channel,
8695 client_epoch,
8696 201,
8697 b"first request".to_vec(),
8698 )
8699 .unwrap();
8700 router
8701 .route_for_connection(&client_ctx, request)
8702 .await
8703 .unwrap();
8704 let delivered = module_rx.try_recv().unwrap().frame;
8705 assert_eq!(delivered.header.ty, FrameType::Request);
8706 assert_eq!(
8707 (delivered.header.channel, delivered.header.epoch),
8708 (module_channel, module_epoch)
8709 );
8710 assert_eq!(delivered.body, b"first request");
8711 }
8712
8713 #[tokio::test]
8719 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
8720 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8721 let client = ConnectionId::new(42);
8722 let (client_ctx, _client_rx) = route_ctx(client);
8723
8724 let (open_task, bind) = relay_route_open(
8725 &handler,
8726 client,
8727 &client_ctx.egress,
8728 &mut module_rx,
8729 300,
8730 "aft",
8731 "released-before-mark",
8732 )
8733 .await;
8734 let (module_channel, module_epoch) = route_bind_channel(&bind);
8735 hold.wait().await;
8736
8737 handler.cleanup_connection(client).unwrap();
8738 assert!(
8739 module_rx.try_recv().is_err(),
8740 "client teardown must not GOODBYE a bind route.open is still queueing"
8741 );
8742
8743 hold.wait().await;
8744 let response = open_task.await.unwrap();
8745 assert_eq!(response.len(), 1);
8746 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8747
8748 let goodbye = module_rx
8749 .try_recv()
8750 .expect("module is told to drop the binding the daemon abandoned")
8751 .frame;
8752 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8753 assert_eq!(
8754 (goodbye.header.channel, goodbye.header.epoch),
8755 (module_channel, module_epoch)
8756 );
8757 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
8758 }
8759
8760 #[tokio::test]
8765 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
8766 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8767 let client = ConnectionId::new(43);
8768 let (client_ctx, client_rx) = route_ctx(client);
8769
8770 let (open_task, bind) = relay_route_open(
8771 &handler,
8772 client,
8773 &client_ctx.egress,
8774 &mut module_rx,
8775 400,
8776 "aft",
8777 "gone-client-before-mark",
8778 )
8779 .await;
8780 let (module_channel, module_epoch) = route_bind_channel(&bind);
8781 hold.wait().await;
8782
8783 drop(client_rx);
8784 assert!(client_ctx.egress.is_closed());
8785 handler
8786 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
8787 .await
8788 .unwrap();
8789
8790 hold.wait().await;
8791 let response = open_task.await.unwrap();
8792 assert_eq!(response.len(), 1);
8793 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8794
8795 let goodbye = module_rx
8796 .try_recv()
8797 .expect("module is told to drop the binding it acked")
8798 .frame;
8799 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8800 assert_eq!(
8801 (goodbye.header.channel, goodbye.header.epoch),
8802 (module_channel, module_epoch)
8803 );
8804 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
8805 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
8806 }
8807
8808 #[test]
8815 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
8816 let handler = ControlHandler::default();
8817 let module_connection = ConnectionId::new(30);
8818 let client_connection = ConnectionId::new(31);
8819
8820 handler
8821 .refuse_to_end_module_connection_for_a_client(
8822 module_connection,
8823 77,
8824 ForwardingError::ConnectionClosing {
8825 connection_id: client_connection,
8826 },
8827 )
8828 .expect("a closing client must never end the module connection");
8829
8830 assert!(matches!(
8831 handler.refuse_to_end_module_connection_for_a_client(
8832 module_connection,
8833 78,
8834 ForwardingError::ConnectionClosing {
8835 connection_id: module_connection,
8836 },
8837 ),
8838 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
8839 connection_id
8840 })) if connection_id == module_connection
8841 ));
8842 assert!(matches!(
8843 handler.refuse_to_end_module_connection_for_a_client(
8844 module_connection,
8845 79,
8846 ForwardingError::Poisoned,
8847 ),
8848 Err(RouterError::Forwarding(ForwardingError::Poisoned))
8849 ));
8850 assert!(matches!(
8851 handler.refuse_to_end_module_connection_for_a_client(
8852 module_connection,
8853 80,
8854 ForwardingError::StaleModuleEndpoint,
8855 ),
8856 Err(RouterError::Forwarding(
8857 ForwardingError::StaleModuleEndpoint
8858 ))
8859 ));
8860 }
8861
8862 #[tokio::test]
8875 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
8876 let registry = Arc::new(Registry::default());
8877 let forwarding = Arc::new(ForwardingTable::default());
8878 let supervisor = SupervisorHandle::new();
8879 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8880 let handler =
8881 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8882 .with_supervisor(supervisor);
8883
8884 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8885 hello_via_sink(
8886 &handler,
8887 &target_ctx,
8888 &mut target_rx,
8889 hello_frame("target", PROTOCOL_VERSION, 1),
8890 )
8891 .await;
8892
8893 let wrong_nonce = handler
8897 .handle_control_frame(
8898 &route_ctx(ConnectionId::new(91)).0,
8899 route_open_frame_with_admission_facts(
8900 20,
8901 "target",
8902 unique_project_root("admission-facts"),
8903 Some(subc_control::ConsumerIdentity {
8904 module_id: "fed".to_string(),
8905 launch_nonce: "not-the-real-nonce".to_string(),
8906 }),
8907 None,
8908 ),
8909 )
8910 .await
8911 .unwrap();
8912 assert_eq!(
8913 parse_error(&wrong_nonce[0])["code"],
8914 "bad_consumer_identity",
8915 "a mismatched launch nonce must be refused, not stamped Reserved"
8916 );
8917
8918 let never_spawned = handler
8922 .handle_control_frame(
8923 &route_ctx(ConnectionId::new(92)).0,
8924 route_open_frame_with_admission_facts(
8925 21,
8926 "target",
8927 unique_project_root("admission-facts"),
8928 Some(subc_control::ConsumerIdentity {
8929 module_id: "never-spawned".to_string(),
8930 launch_nonce: "any-nonce".to_string(),
8931 }),
8932 None,
8933 ),
8934 )
8935 .await
8936 .unwrap();
8937 assert_eq!(
8938 parse_error(&never_spawned[0])["code"],
8939 "bad_consumer_identity",
8940 "an unspawned module_id must be refused rather than accepted for lack of a record"
8941 );
8942 }
8943
8944 #[tokio::test]
8967 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
8968 let registry = Arc::new(Registry::default());
8969 let forwarding = Arc::new(ForwardingTable::default());
8970 let supervisor = SupervisorHandle::new();
8971 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8972 let handler =
8973 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8974 .with_supervisor(supervisor);
8975
8976 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
8977 hello_via_sink(
8978 &handler,
8979 &target_ctx,
8980 &mut target_rx,
8981 hello_frame("target", PROTOCOL_VERSION, 1),
8982 )
8983 .await;
8984
8985 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
8986 let route_handler = handler.clone();
8987 let route_task = tokio::spawn(async move {
8988 route_handler
8989 .handle_control_frame(
8990 &client_ctx,
8991 route_open_frame_with_admission_facts(
8992 30,
8993 "target",
8994 unique_project_root("admission-facts"),
8995 Some(subc_control::ConsumerIdentity {
8996 module_id: "fed".to_string(),
8997 launch_nonce: "fed-nonce".to_string(),
8998 }),
8999 None,
9000 ),
9001 )
9002 .await
9003 .unwrap()
9004 });
9005
9006 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9013 .await
9014 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
9015 .expect("module control channel closed before route.bind");
9016 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9017 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9018 panic!("expected route.bind")
9019 };
9020 assert_eq!(
9021 principal,
9022 Some(Principal::Reserved {
9023 module_id: "fed".to_string()
9024 }),
9025 "a correctly attested consumer must be stamped Reserved for its own id"
9026 );
9027
9028 handler
9029 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9030 .await
9031 .unwrap();
9032 assert!(route_task.await.unwrap().is_empty());
9033 assert!(
9034 matches!(
9035 serde_json::from_slice::<ClientControlResponse>(
9036 &client_rx.recv().await.unwrap().body
9037 )
9038 .unwrap(),
9039 ClientControlResponse::RouteOpen { .. }
9040 ),
9041 "the route must actually open, not merely avoid an error"
9042 );
9043 }
9044
9045 #[tokio::test(start_paused = true)]
9046 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
9047 let registry = Arc::new(Registry::default());
9048 let forwarding = Arc::new(ForwardingTable::default());
9049 let supervisor = SupervisorHandle::new();
9050 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9051 let handler =
9052 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9053 .with_supervisor(supervisor);
9054
9055 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
9056 hello_via_sink(
9057 &handler,
9058 &target_ctx,
9059 &mut target_rx,
9060 hello_frame("target", PROTOCOL_VERSION, 1),
9061 )
9062 .await;
9063
9064 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
9065 let direct_handler = handler.clone();
9066 let direct_open = tokio::spawn(async move {
9067 direct_handler
9068 .handle_control_frame(
9069 &direct_ctx,
9070 route_open_frame(2, "target", unique_project_root("route-census-direct")),
9071 )
9072 .await
9073 .unwrap()
9074 });
9075 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9076 .await
9077 .expect("no direct route.bind within 5s")
9078 .expect("target control channel closed before direct route.bind");
9079 handler
9080 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
9081 .await
9082 .unwrap();
9083 assert!(direct_open.await.unwrap().is_empty());
9084 let _ = direct_rx.recv().await.unwrap();
9085
9086 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
9087 let reserved_handler = handler.clone();
9088 let reserved_open = tokio::spawn(async move {
9089 reserved_handler
9090 .handle_control_frame(
9091 &reserved_ctx,
9092 route_open_frame_with_admission_facts(
9093 3,
9094 "target",
9095 unique_project_root("admission-facts"),
9096 Some(ConsumerIdentity {
9097 module_id: "fed".to_string(),
9098 launch_nonce: "fed-nonce".to_string(),
9099 }),
9100 None,
9101 ),
9102 )
9103 .await
9104 .unwrap()
9105 });
9106 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9107 .await
9108 .expect("no reserved route.bind within 5s")
9109 .expect("target control channel closed before reserved route.bind");
9110 handler
9111 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
9112 .await
9113 .unwrap();
9114 assert!(reserved_open.await.unwrap().is_empty());
9115 let _ = reserved_rx.recv().await.unwrap();
9116
9117 forwarding
9118 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
9119 .unwrap();
9120 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
9121 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
9122 module_id: Some("target".to_string()),
9123 })
9124 .unwrap();
9125 let census_frame =
9126 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
9127 let response = handler
9128 .handle_control_frame(&census_ctx, census_frame)
9129 .await
9130 .unwrap()
9131 .pop()
9132 .unwrap();
9133 let actual: Value = serde_json::from_slice(&response.body).unwrap();
9134 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
9135 assert!(matches!(
9136 decoded,
9137 ClientControlResponse::SupervisorRoutes { .. }
9138 ));
9139 let routes = actual["modules"][0]["routes"].as_array().unwrap();
9140 assert_eq!(routes.len(), 2);
9141 assert!(routes.iter().all(|route| route["draining"] == true));
9142 assert!(
9145 routes.iter().all(|route| route["drain_reason"] == "reload"),
9146 "draining routes must name the drain's reason: {routes:?}"
9147 );
9148 assert!(routes.iter().any(|route| {
9149 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
9150 }));
9151 assert!(routes.iter().any(|route| {
9152 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
9153 }));
9154
9155 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9156 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
9157 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9158 std::fs::write(
9159 &golden_path,
9160 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
9161 )
9162 .unwrap();
9163 }
9164 let expected: Value =
9165 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
9166 assert_eq!(actual, expected);
9167 }
9168
9169 async fn query_live_roots(
9170 handler: &ControlHandler,
9171 module_ctx: &RouteCtx,
9172 ) -> ModuleControlResponseToModule {
9173 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
9174 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
9175 let response = handler
9176 .handle_control_frame(module_ctx, frame)
9177 .await
9178 .unwrap()
9179 .pop()
9180 .unwrap();
9181 serde_json::from_slice(&response.body).unwrap()
9182 }
9183
9184 #[tokio::test(start_paused = true)]
9185 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
9186 let registry = Arc::new(Registry::default());
9187 let forwarding = Arc::new(ForwardingTable::default());
9188 let handler = ControlHandler::with_forwarding(registry, forwarding);
9189 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
9190 hello_via_sink(
9191 &handler,
9192 &target_ctx,
9193 &mut target_rx,
9194 hello_frame("target", PROTOCOL_VERSION, 1),
9195 )
9196 .await;
9197 let root = unique_project_root("live-roots-known");
9198 let path = ProjectRootId::from_path_allowing_missing(root.path())
9199 .unwrap()
9200 .as_path()
9201 .to_path_buf();
9202 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
9203 let open_handler = handler.clone();
9204 let opened = tokio::spawn(async move {
9205 open_handler
9206 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
9207 .await
9208 .unwrap()
9209 });
9210 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9211 .await
9212 .unwrap()
9213 .unwrap();
9214 handler
9215 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
9216 .await
9217 .unwrap();
9218 assert!(opened.await.unwrap().is_empty());
9219 let _ = client_rx.recv().await.unwrap();
9220
9221 let root = unique_project_root("live-roots-pending");
9222 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
9223 .unwrap()
9224 .as_path()
9225 .to_path_buf();
9226 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
9227 let open_handler = handler.clone();
9228 let pending = tokio::spawn(async move {
9229 open_handler
9230 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
9231 .await
9232 .unwrap()
9233 });
9234 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9235 .await
9236 .unwrap()
9237 .unwrap();
9238 let actual = query_live_roots(&handler, &target_ctx).await;
9239 let ModuleControlResponseToModule::LiveRoots {
9240 roots,
9241 unknown_root_bindings,
9242 total_bindings,
9243 } = actual
9244 else {
9245 panic!("expected live roots")
9246 };
9247 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
9248 assert_eq!(unknown_root_bindings, 0);
9249 assert_eq!(
9250 roots.len(),
9251 2,
9252 "root-known arm must retain each canonical root"
9253 );
9254 assert_eq!(
9255 total_bindings,
9256 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9257 );
9258 let counts = roots
9259 .iter()
9260 .map(|root| (root.project_root.clone(), root.bound, root.pending))
9261 .collect::<Vec<_>>();
9262 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
9263 expected.sort_by(|a, b| a.0.cmp(&b.0));
9264 assert_eq!(
9265 counts, expected,
9266 "roots must sort by path and count pending separately"
9267 );
9268 handler
9269 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
9270 .await
9271 .unwrap();
9272 assert!(pending.await.unwrap().is_empty());
9273 }
9274
9275 #[tokio::test(start_paused = true)]
9276 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
9277 let forwarding = Arc::new(ForwardingTable::default());
9278 let handler =
9279 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9280 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
9281 hello_via_sink(
9282 &handler,
9283 &target_ctx,
9284 &mut target_rx,
9285 hello_frame("target", PROTOCOL_VERSION, 1),
9286 )
9287 .await;
9288 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
9289 let pending = forwarding
9290 .begin_route_bind_relay_for_test(
9291 client_ctx.connection_id,
9292 client_ctx.egress.clone(),
9293 2,
9294 "target",
9295 )
9296 .unwrap();
9297 forwarding
9298 .complete_pending_relay(
9299 target_ctx.connection_id,
9300 pending.corr,
9301 RouteBindRelayOutcome::Accepted,
9302 )
9303 .unwrap();
9304 let actual = query_live_roots(&handler, &target_ctx).await;
9305 let ModuleControlResponseToModule::LiveRoots {
9306 roots,
9307 unknown_root_bindings,
9308 total_bindings,
9309 } = actual
9310 else {
9311 panic!("expected live roots")
9312 };
9313 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
9314 assert_eq!(
9315 unknown_root_bindings, 1,
9316 "unknown-root arm must not read as no bindings"
9317 );
9318 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
9319 assert_eq!(
9320 total_bindings,
9321 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9322 );
9323 }
9324
9325 #[tokio::test(start_paused = true)]
9333 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
9334 let forwarding = Arc::new(ForwardingTable::default());
9335 let handler =
9336 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9337 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
9338 let replies = handler
9339 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
9340 .await
9341 .unwrap();
9342 let queued_by_hello = module_rx.len();
9343
9344 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
9345 let open_handler = handler.clone();
9346 let open = tokio::spawn(async move {
9347 open_handler
9348 .handle_control_frame(
9349 &client_ctx,
9350 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
9351 )
9352 .await
9353 .unwrap()
9354 });
9355 let mut spins = 0;
9357 while module_rx.len() == queued_by_hello {
9358 spins += 1;
9359 assert!(spins < 10_000, "route.open never queued a route.bind");
9360 tokio::task::yield_now().await;
9361 }
9362
9363 for reply in replies {
9365 module_ctx.egress.send(reply).await.unwrap();
9366 }
9367
9368 let first = module_rx.recv().await.unwrap().frame;
9369 assert_eq!(
9370 first.header.ty,
9371 FrameType::HelloAck,
9372 "the first frame a registering module reads must be its HELLO_ACK"
9373 );
9374 assert_eq!(first.header.corr, 7);
9375 let second = module_rx.recv().await.unwrap().frame;
9376 assert_eq!(second.header.ty, FrameType::Request);
9377 assert!(
9378 matches!(
9379 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
9380 ModuleControlRequest::RouteBind { .. }
9381 ),
9382 "the route.bind follows the ack"
9383 );
9384 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
9385
9386 handler
9387 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
9388 .await
9389 .unwrap();
9390 assert!(open.await.unwrap().is_empty());
9391 let _ = client_rx.recv().await.unwrap();
9392 }
9393
9394 #[tokio::test(start_paused = true)]
9395 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
9396 let handler = ControlHandler::with_forwarding(
9397 Arc::new(Registry::default()),
9398 Arc::new(ForwardingTable::default()),
9399 );
9400 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
9401 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
9402 hello_via_sink(
9403 &handler,
9404 &first_ctx,
9405 &mut first_rx,
9406 hello_frame("first", PROTOCOL_VERSION, 1),
9407 )
9408 .await;
9409 hello_via_sink(
9410 &handler,
9411 &second_ctx,
9412 &mut second_rx,
9413 hello_frame("second", PROTOCOL_VERSION, 2),
9414 )
9415 .await;
9416 let root = unique_project_root("second-only");
9417 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
9418 let cloned = handler.clone();
9419 let open = tokio::spawn(async move {
9420 cloned
9421 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
9422 .await
9423 .unwrap()
9424 });
9425 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
9426 .await
9427 .unwrap()
9428 .unwrap();
9429 let first = query_live_roots(&handler, &first_ctx).await;
9430 let second = query_live_roots(&handler, &second_ctx).await;
9431 assert!(
9432 matches!(
9433 first,
9434 ModuleControlResponseToModule::LiveRoots {
9435 total_bindings: 0,
9436 ..
9437 }
9438 ),
9439 "cross-module scope must not expose another module's roots"
9440 );
9441 assert!(
9442 matches!(
9443 second,
9444 ModuleControlResponseToModule::LiveRoots {
9445 total_bindings: 1,
9446 ..
9447 }
9448 ),
9449 "second module must see its pending route"
9450 );
9451 handler
9452 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
9453 .await
9454 .unwrap();
9455 assert!(open.await.unwrap().is_empty());
9456 }
9457
9458 #[tokio::test(start_paused = true)]
9459 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
9460 let handler = ControlHandler::with_forwarding(
9461 Arc::new(Registry::default()),
9462 Arc::new(ForwardingTable::default()),
9463 );
9464 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
9465 hello_via_sink(
9466 &handler,
9467 &target_ctx,
9468 &mut target_rx,
9469 hello_frame("target", PROTOCOL_VERSION, 1),
9470 )
9471 .await;
9472 let actual = query_live_roots(&handler, &target_ctx).await;
9473 let ModuleControlResponseToModule::LiveRoots {
9474 roots,
9475 unknown_root_bindings,
9476 total_bindings,
9477 } = actual
9478 else {
9479 panic!("expected live roots")
9480 };
9481 assert!(roots.is_empty());
9482 assert_eq!(unknown_root_bindings, 0);
9483 assert_eq!(total_bindings, 0);
9484 assert_eq!(
9485 total_bindings,
9486 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9487 );
9488 }
9489
9490 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
9497 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9498 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
9499 let text = std::fs::read_to_string(&path)
9500 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
9501 let vectors: Vec<(String, Value)> = text
9502 .lines()
9503 .filter(|line| !line.trim().is_empty())
9504 .map(|line| {
9505 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
9506 let id = entry["corpus_id"]
9507 .as_str()
9508 .expect("every vector carries a corpus_id")
9509 .to_string();
9510 (id, entry["package"].clone())
9511 })
9512 .collect();
9513 assert_eq!(
9517 vectors.len(),
9518 3,
9519 "vendored fed corpus changed size; re-sync from subc-federation"
9520 );
9521
9522 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
9537 let richest = vectors
9538 .iter()
9539 .filter_map(|(_, package)| package.as_object())
9540 .map(|object| {
9541 object
9542 .keys()
9543 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
9544 .count()
9545 })
9546 .max()
9547 .unwrap_or(0);
9548 assert!(
9549 richest >= 2,
9550 "vendored corpus no longer carries a package with unmodelled fields, \
9551 so the relay test can no longer distinguish a verbatim relay from a lossy one"
9552 );
9553
9554 vectors
9555 }
9556
9557 #[tokio::test]
9565 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
9566 for (corpus_id, package) in fed_admission_facts_vectors() {
9567 let registry = Arc::new(Registry::default());
9568 let forwarding = Arc::new(ForwardingTable::default());
9569 let supervisor = SupervisorHandle::new();
9570 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9571 let handler =
9572 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9573 .with_supervisor(supervisor)
9574 .with_admission_facts_config(
9575 Some("fed".to_string()),
9576 Some(vec!["target".to_string()]),
9577 );
9578
9579 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9580 hello_via_sink(
9581 &handler,
9582 &target_ctx,
9583 &mut target_rx,
9584 hello_frame("target", PROTOCOL_VERSION, 1),
9585 )
9586 .await;
9587
9588 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
9589 let route_handler = handler.clone();
9590 let expected = package.clone();
9591 let route_task = tokio::spawn(async move {
9592 route_handler
9593 .handle_control_frame(
9594 &client_ctx,
9595 route_open_frame_with_admission_facts(
9596 20,
9597 "target",
9598 unique_project_root("admission-facts"),
9599 Some(subc_control::ConsumerIdentity {
9600 module_id: "fed".to_string(),
9601 launch_nonce: "fed-nonce".to_string(),
9602 }),
9603 Some(package),
9604 ),
9605 )
9606 .await
9607 .unwrap()
9608 });
9609
9610 let bind_frame = target_rx.recv().await.unwrap();
9611 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9612 let ModuleControlRequest::RouteBind {
9613 admission_facts, ..
9614 } = bind
9615 else {
9616 panic!("{corpus_id}: expected route.bind")
9617 };
9618 assert_eq!(
9619 admission_facts,
9620 Some(expected),
9621 "{corpus_id}: relay must not add, drop or reshape any field"
9622 );
9623
9624 handler
9625 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9626 .await
9627 .unwrap();
9628 route_task.await.unwrap();
9629 }
9630 }
9631
9632 #[tokio::test]
9633 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
9634 let registry = Arc::new(Registry::default());
9635 let forwarding = Arc::new(ForwardingTable::default());
9636 let supervisor = SupervisorHandle::new();
9637 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9638 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
9639 let handler =
9640 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9641 .with_supervisor(supervisor)
9642 .with_admission_facts_config(
9643 Some("fed".to_string()),
9644 Some(vec!["target".to_string()]),
9645 );
9646
9647 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
9648 hello_via_sink(
9649 &handler,
9650 &target_ctx,
9651 &mut target_rx,
9652 hello_frame("target", PROTOCOL_VERSION, 1),
9653 )
9654 .await;
9655 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
9656 hello_via_sink(
9657 &handler,
9658 &other_ctx,
9659 &mut other_rx,
9660 hello_frame("other", PROTOCOL_VERSION, 2),
9661 )
9662 .await;
9663
9664 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
9665 let expected_facts = facts.clone();
9666 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
9667 let route_handler = handler.clone();
9668 let route_task = tokio::spawn(async move {
9669 route_handler
9670 .handle_control_frame(
9671 &client_ctx,
9672 route_open_frame_with_admission_facts(
9673 10,
9674 "target",
9675 unique_project_root("admission-facts"),
9676 Some(subc_control::ConsumerIdentity {
9677 module_id: "fed".to_string(),
9678 launch_nonce: "fed-nonce".to_string(),
9679 }),
9680 Some(facts.clone()),
9681 ),
9682 )
9683 .await
9684 .unwrap()
9685 });
9686 let bind_frame = target_rx.recv().await.unwrap();
9687 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9688 let ModuleControlRequest::RouteBind {
9689 admission_facts, ..
9690 } = bind
9691 else {
9692 panic!("expected route.bind")
9693 };
9694 assert_eq!(admission_facts, Some(expected_facts));
9695 handler
9696 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9697 .await
9698 .unwrap();
9699 assert!(route_task.await.unwrap().is_empty());
9700 assert!(matches!(
9701 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
9702 .unwrap(),
9703 ClientControlResponse::RouteOpen { .. }
9704 ));
9705
9706 let direct = handler
9707 .handle_control_frame(
9708 &route_ctx(ConnectionId::new(73)).0,
9709 route_open_frame_with_admission_facts(
9710 11,
9711 "target",
9712 unique_project_root("admission-facts"),
9713 None,
9714 Some(json!({"x": 1})),
9715 ),
9716 )
9717 .await
9718 .unwrap();
9719 assert_eq!(
9720 parse_error(&direct[0])["code"],
9721 "admission_facts_not_permitted"
9722 );
9723
9724 let different_reserved = handler
9725 .handle_control_frame(
9726 &route_ctx(ConnectionId::new(77)).0,
9727 route_open_frame_with_admission_facts(
9728 15,
9729 "target",
9730 unique_project_root("admission-facts"),
9731 Some(subc_control::ConsumerIdentity {
9732 module_id: "other".to_string(),
9733 launch_nonce: "other-nonce".to_string(),
9734 }),
9735 Some(json!({"x": 1})),
9736 ),
9737 )
9738 .await
9739 .unwrap();
9740 assert_eq!(
9741 parse_error(&different_reserved[0])["code"],
9742 "admission_facts_not_permitted"
9743 );
9744
9745 let other_target = handler
9746 .handle_control_frame(
9747 &route_ctx(ConnectionId::new(74)).0,
9748 route_open_frame_with_admission_facts(
9749 12,
9750 "other",
9751 unique_project_root("admission-facts"),
9752 Some(subc_control::ConsumerIdentity {
9753 module_id: "fed".to_string(),
9754 launch_nonce: "fed-nonce".to_string(),
9755 }),
9756 Some(json!({"x": 1})),
9757 ),
9758 )
9759 .await
9760 .unwrap();
9761 assert_eq!(
9762 parse_error(&other_target[0])["code"],
9763 "admission_facts_target_not_allowed"
9764 );
9765
9766 let nonexistent = handler
9767 .handle_control_frame(
9768 &route_ctx(ConnectionId::new(75)).0,
9769 route_open_frame_with_admission_facts(
9770 13,
9771 "missing",
9772 unique_project_root("admission-facts"),
9773 None,
9774 Some(json!({"x": 1})),
9775 ),
9776 )
9777 .await
9778 .unwrap();
9779 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
9780
9781 let described = handler
9782 .handle_control_frame(
9783 &route_ctx(ConnectionId::new(76)).0,
9784 Frame::build(
9785 FrameType::Request,
9786 control_flags(),
9787 0,
9788 0,
9789 14,
9790 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9791 )
9792 .unwrap(),
9793 )
9794 .await
9795 .unwrap();
9796 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9797 serde_json::from_slice(&described[0].body).unwrap()
9798 else {
9799 panic!("expected server.describe response")
9800 };
9801 assert!(capabilities
9802 .iter()
9803 .any(|cap| cap == "admission_facts_relay_v1"));
9804 }
9805
9806 #[tokio::test]
9807 async fn admission_facts_without_configured_carrier_are_rejected() {
9808 let registry = Arc::new(Registry::default());
9809 let forwarding = Arc::new(ForwardingTable::default());
9810 let handler = ControlHandler::with_forwarding(registry, forwarding);
9811 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
9812 hello_via_sink(
9813 &handler,
9814 &target_ctx,
9815 &mut target_rx,
9816 hello_frame("target", PROTOCOL_VERSION, 1),
9817 )
9818 .await;
9819
9820 let responses = handler
9821 .handle_control_frame(
9822 &route_ctx(ConnectionId::new(79)).0,
9823 route_open_frame_with_admission_facts(
9824 16,
9825 "target",
9826 unique_project_root("admission-facts"),
9827 None,
9828 Some(json!({"x": 1})),
9829 ),
9830 )
9831 .await
9832 .unwrap();
9833 assert_eq!(
9834 parse_error(&responses[0])["code"],
9835 "admission_facts_not_permitted"
9836 );
9837 }
9838
9839 #[tokio::test]
9840 async fn route_open_relays_consumer_capabilities_verbatim() {
9841 let registry = Arc::new(Registry::default());
9842 let forwarding = Arc::new(ForwardingTable::default());
9843 let handler =
9844 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9845 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
9846 hello_via_sink(
9847 &handler,
9848 &module_ctx,
9849 &mut module_rx,
9850 hello_frame("aft", PROTOCOL_VERSION, 7),
9851 )
9852 .await;
9853
9854 let expected = vec!["elicitation".to_string(), "roots".to_string()];
9855 let expected_for_request = expected.clone();
9856 let project_root = unique_project_root("consumer-capabilities-present");
9857 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
9858 let route_handler = handler.clone();
9859 let route_task = tokio::spawn(async move {
9860 route_handler
9861 .handle_control_frame(
9862 &client_ctx,
9863 route_open_frame_with_consumer_capabilities(
9864 401,
9865 "aft",
9866 project_root,
9867 Some(expected_for_request),
9868 ),
9869 )
9870 .await
9871 .unwrap()
9872 });
9873 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9874 .await
9875 .unwrap()
9876 .unwrap();
9877 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9878 let ModuleControlRequest::RouteBind {
9879 consumer_capabilities,
9880 ..
9881 } = bind
9882 else {
9883 panic!("expected route.bind request, got {bind:?}");
9884 };
9885 assert_eq!(consumer_capabilities, Some(expected.clone()));
9886
9887 handler
9888 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9889 .await
9890 .unwrap();
9891 let route_response = route_task.await.unwrap();
9892 assert!(route_response.is_empty());
9893 let published = client_rx.recv().await.unwrap();
9894 assert!(matches!(
9895 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9896 ClientControlResponse::RouteOpen { .. }
9897 ));
9898 }
9899
9900 #[tokio::test]
9901 async fn route_open_without_consumer_capabilities_relays_none() {
9902 let registry = Arc::new(Registry::default());
9903 let forwarding = Arc::new(ForwardingTable::default());
9904 let handler =
9905 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9906 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
9907 hello_via_sink(
9908 &handler,
9909 &module_ctx,
9910 &mut module_rx,
9911 hello_frame("aft", PROTOCOL_VERSION, 7),
9912 )
9913 .await;
9914
9915 let project_root = unique_project_root("consumer-capabilities-absent");
9916 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
9917 let route_handler = handler.clone();
9918 let route_task = tokio::spawn(async move {
9919 route_handler
9920 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
9921 .await
9922 .unwrap()
9923 });
9924 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9925 .await
9926 .unwrap()
9927 .unwrap();
9928 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9929 let ModuleControlRequest::RouteBind {
9930 consumer_capabilities,
9931 ..
9932 } = bind
9933 else {
9934 panic!("expected route.bind request, got {bind:?}");
9935 };
9936 assert_eq!(consumer_capabilities, None);
9937
9938 handler
9939 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9940 .await
9941 .unwrap();
9942 let route_response = route_task.await.unwrap();
9943 assert!(route_response.is_empty());
9944 let published = client_rx.recv().await.unwrap();
9945 assert!(matches!(
9946 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9947 ClientControlResponse::RouteOpen { .. }
9948 ));
9949 }
9950
9951 async fn bind_role_versions_for(
9955 sent: Option<BTreeMap<String, String>>,
9956 connection: u64,
9957 ) -> Option<BTreeMap<String, String>> {
9958 let registry = Arc::new(Registry::default());
9959 let forwarding = Arc::new(ForwardingTable::default());
9960 let handler =
9961 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9962 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
9963 hello_via_sink(
9964 &handler,
9965 &module_ctx,
9966 &mut module_rx,
9967 hello_frame("aft", PROTOCOL_VERSION, 7),
9968 )
9969 .await;
9970 let project_root = unique_project_root("role-versions");
9971 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
9972 let route_handler = handler.clone();
9973 let route_task = tokio::spawn(async move {
9974 route_handler
9975 .handle_control_frame(
9976 &client_ctx,
9977 route_open_frame_with_role_versions(403, "aft", project_root, sent),
9978 )
9979 .await
9980 .unwrap()
9981 });
9982 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9983 .await
9984 .expect("a well-formed route.open reaches the module as a bind")
9985 .unwrap();
9986 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9987 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
9988 panic!("expected route.bind request, got {bind:?}");
9989 };
9990 handler
9991 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9992 .await
9993 .unwrap();
9994 assert!(route_task.await.unwrap().is_empty());
9995 let published = client_rx.recv().await.unwrap();
9996 assert!(matches!(
9997 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9998 ClientControlResponse::RouteOpen { .. }
9999 ));
10000 role_versions
10001 }
10002
10003 #[tokio::test]
10004 async fn route_open_relays_role_versions_verbatim() {
10005 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
10006 assert_eq!(
10007 bind_role_versions_for(Some(sent.clone()), 141).await,
10008 Some(sent)
10009 );
10010 }
10011
10012 #[tokio::test]
10015 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
10016 assert_eq!(bind_role_versions_for(None, 143).await, None);
10017 assert_eq!(
10018 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
10019 None
10020 );
10021 }
10022
10023 #[tokio::test]
10024 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
10025 let registry = Arc::new(Registry::default());
10026 let forwarding = Arc::new(ForwardingTable::default());
10027 let handler =
10028 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10029 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
10030 hello_via_sink(
10031 &handler,
10032 &module_ctx,
10033 &mut module_rx,
10034 hello_frame("aft", PROTOCOL_VERSION, 7),
10035 )
10036 .await;
10037
10038 let nine: BTreeMap<String, String> = (0..9)
10039 .map(|index| (format!("role-{index}"), "v1".to_string()))
10040 .collect();
10041 for (label, malformed) in [
10042 (
10043 "invalid role name",
10044 role_versions(&[("Tool_Provider", "v1")]),
10045 ),
10046 ("invalid version", role_versions(&[("tool-provider", "v0")])),
10047 ("nine entries", nine),
10048 ] {
10049 let responses = tokio::time::timeout(
10052 Duration::from_secs(1),
10053 handler.handle_control_frame(
10054 &route_ctx(ConnectionId::new(148)).0,
10055 route_open_frame_with_role_versions(
10056 404,
10057 "aft",
10058 unique_project_root("role-versions-malformed"),
10059 Some(malformed),
10060 ),
10061 ),
10062 )
10063 .await
10064 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
10065 .unwrap();
10066 assert_eq!(responses.len(), 1, "{label}");
10067 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
10068 let error = parse_error(&responses[0]);
10069 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
10070 assert_eq!(
10071 error["detail"]["field"], "role_versions",
10072 "{label}: {error}"
10073 );
10074 assert!(
10075 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
10076 "{label}: a malformed declaration is terminal"
10077 );
10078 assert!(
10079 module_rx.try_recv().is_err(),
10080 "{label}: the module must never see a bind"
10081 );
10082 }
10083 }
10084
10085 #[tokio::test]
10089 async fn route_role_versions_capability_is_advertised() {
10090 let handler = ControlHandler::new(Arc::new(Registry::default()));
10091 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
10092 let ack = hello_via_sink(
10093 &handler,
10094 &ctx,
10095 &mut rx,
10096 hello_frame("m", PROTOCOL_VERSION, 1),
10097 )
10098 .await;
10099 let ack = parse_ack(&ack);
10100 assert!(
10101 ack.subc_capabilities
10102 .iter()
10103 .any(|c| c == "route-role-versions/v1"),
10104 "{:?}",
10105 ack.subc_capabilities
10106 );
10107
10108 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
10109 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
10110 let reply = handler
10111 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
10112 .await
10113 .unwrap()
10114 .pop()
10115 .unwrap();
10116 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10117 serde_json::from_slice(&reply.body).unwrap()
10118 else {
10119 panic!("not a server.describe reply");
10120 };
10121 assert!(
10122 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
10123 "{capabilities:?}"
10124 );
10125 }
10126
10127 #[tokio::test]
10128 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
10129 let registry = Arc::new(Registry::default());
10130 let forwarding = Arc::new(ForwardingTable::default());
10131 let handler =
10132 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10133 .with_health_probe_timeout(Duration::from_secs(5));
10134 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
10135 hello_via_sink(
10136 &handler,
10137 &module_ctx,
10138 &mut module_rx,
10139 non_routable_hello_frame_with_control_ops(
10140 "mcp",
10141 300,
10142 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10143 ),
10144 )
10145 .await;
10146 assert!(registry
10147 .get_module("mcp")
10148 .unwrap()
10149 .unwrap()
10150 .manifest
10151 .provides
10152 .is_empty());
10153
10154 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
10155 let route_response = handler
10156 .handle_control_frame(
10157 &route_client_ctx,
10158 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
10159 )
10160 .await
10161 .unwrap();
10162 assert_eq!(route_response[0].header.ty, FrameType::Error);
10163 assert_eq!(
10164 parse_error(&route_response[0])["code"],
10165 "target_unavailable"
10166 );
10167 assert!(parse_error(&route_response[0])["message"]
10168 .as_str()
10169 .unwrap()
10170 .contains("does not provide the requested target"));
10171 assert!(module_rx.try_recv().is_err());
10172
10173 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
10174 let health_handler = handler.clone();
10175 let health_task = tokio::spawn(async move {
10176 health_handler
10177 .handle_control_frame(
10178 &health_client_ctx,
10179 supervisor_health_probe_frame(302, "mcp"),
10180 )
10181 .await
10182 .unwrap()
10183 });
10184 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10185 .await
10186 .unwrap()
10187 .unwrap();
10188 assert_eq!(
10189 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10190 ModuleControlRequest::HealthCheck {}
10191 );
10192 handler
10193 .handle_control_frame(
10194 &module_ctx,
10195 health_response(health_frame.header.corr, HealthStatus::Ok),
10196 )
10197 .await
10198 .unwrap();
10199 let health_response = health_task.await.unwrap();
10200 assert_eq!(health_response[0].header.ty, FrameType::Response);
10201 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10202 ClientControlResponse::SupervisorHealthProbe {
10203 module_id, status, ..
10204 } => {
10205 assert_eq!(module_id, "mcp");
10206 assert_eq!(status, HealthStatus::Ok);
10207 }
10208 other => panic!("unexpected health response: {other:?}"),
10209 }
10210
10211 forwarding
10216 .cleanup_connection(module_ctx.connection_id)
10217 .unwrap();
10218 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
10219 let cleanup_response = tokio::time::timeout(
10220 Duration::from_millis(200),
10221 handler.handle_control_frame(
10222 &cleanup_probe_ctx,
10223 supervisor_health_probe_frame(303, "mcp"),
10224 ),
10225 )
10226 .await
10227 .expect("probe should fail immediately when the control lane is gone")
10228 .unwrap();
10229 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
10230 assert_eq!(
10231 parse_error(&cleanup_response[0])["code"],
10232 "target_unavailable"
10233 );
10234 assert!(parse_error(&cleanup_response[0])["message"]
10235 .as_str()
10236 .unwrap()
10237 .contains("no module connection"));
10238
10239 handler
10240 .cleanup_connection(module_ctx.connection_id)
10241 .unwrap();
10242 }
10243
10244 #[tokio::test]
10245 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
10246 let registry = Arc::new(Registry::default());
10247 let supervisor_handle = SupervisorHandle::new();
10248 let supervisor =
10249 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10250 .with_handle(supervisor_handle.clone())
10251 .with_connection_file_path(
10252 std::env::temp_dir()
10253 .join(format!("subc-route-open-warming-{}", std::process::id())),
10254 );
10255 let module = supervisor
10256 .supervise_configured(
10257 ModuleSpec {
10258 module_id: "warming".to_string(),
10259 program: fake_aft_stub_path(),
10260 args: Vec::new(),
10261 env: Vec::new(),
10262 reserved: false,
10263 reserved_prefixes: Vec::new(),
10264 protocol: ModuleProtocol::Subc,
10265 overlap: Default::default(),
10266 },
10267 true,
10268 )
10269 .unwrap();
10270 assert_eq!(module.state().unwrap(), ModuleState::Running);
10271
10272 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10273 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
10274 let response = handler
10275 .handle_control_frame(
10276 &ctx,
10277 route_open_frame(304, "warming", unique_project_root("warming")),
10278 )
10279 .await
10280 .unwrap();
10281 module.stop().await.unwrap();
10282
10283 assert_eq!(response[0].header.ty, FrameType::Error);
10284 let error = parse_error(&response[0]);
10285 assert_eq!(error["code"], "module_warming");
10286 assert!(error["message"]
10287 .as_str()
10288 .unwrap()
10289 .contains("state=running, enabled=true, live=false"));
10290 }
10291
10292 #[test]
10293 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
10294 let handler = ControlHandler::new(Arc::new(Registry::default()));
10295 let capture = EventCapture::default();
10296 let _subscriber =
10297 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10298 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
10299 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
10300 let pending = (0..limit).collect::<Vec<_>>();
10301 let response = handler
10302 .route_open_capacity_refusal(
10303 &ctx,
10304 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
10305 "busy",
10306 pending.len(),
10307 limit,
10308 )
10309 .unwrap();
10310 assert_eq!(parse_error(&response)["code"], "target_unavailable");
10311 let event = capture
10312 .events()
10313 .into_iter()
10314 .find(|event| {
10315 event.target == "control"
10316 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
10317 })
10318 .expect("connection admission refusal event");
10319 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10320 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10321 }
10322
10323 #[test]
10324 fn route_open_target_cap_logs_admission_reason_and_capacity() {
10325 let handler = ControlHandler::new(Arc::new(Registry::default()));
10326 let capture = EventCapture::default();
10327 let _subscriber =
10328 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10329 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
10330 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
10331 let guards = (0..limit)
10332 .map(|_| {
10333 handler
10334 .route_bind_concurrency
10335 .try_admit("busy", limit)
10336 .unwrap()
10337 })
10338 .collect::<Vec<_>>();
10339 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
10340 Err(in_flight) => in_flight,
10341 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
10342 };
10343 let response = handler
10344 .route_open_target_capacity_refusal(
10345 &ctx,
10346 &route_open_frame(397, "busy", unique_project_root("target-cap")),
10347 "busy",
10348 in_flight,
10349 )
10350 .unwrap();
10351 assert_eq!(parse_error(&response)["code"], "target_unavailable");
10352 let event = capture
10353 .events()
10354 .into_iter()
10355 .find(|event| {
10356 event.target == "control"
10357 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
10358 })
10359 .expect("target admission refusal event");
10360 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10361 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10362 drop(guards);
10363 }
10364
10365 #[tokio::test(flavor = "current_thread")]
10366 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
10367 let handler = ControlHandler::new(Arc::new(Registry::default()));
10368 let capture = EventCapture::default();
10369 let _subscriber =
10370 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10371 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
10372 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10373
10374 let _ = handler
10375 .handle_client_control_request(
10376 &ctx,
10377 frame,
10378 ClientControlRequest::SupervisorSetEnabled {
10379 module_id: "broca".to_string(),
10380 enabled: false,
10381 },
10382 )
10383 .await
10384 .unwrap();
10385
10386 let events = capture
10387 .events()
10388 .into_iter()
10389 .filter(|event| {
10390 event.target == "control"
10391 && event.fields.get("message").map(String::as_str)
10392 == Some("supervisor request received")
10393 })
10394 .collect::<Vec<_>>();
10395 assert_eq!(events.len(), 1, "one supervisor request log line");
10396 let event = &events[0];
10397 assert_eq!(event.level, tracing::Level::INFO);
10398 assert_eq!(
10399 event.fields.get("op"),
10400 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
10401 );
10402 assert_eq!(
10403 event.fields.get("module_id"),
10404 Some(&"\"broca\"".to_string())
10405 );
10406 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
10407 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
10408 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
10409 }
10410
10411 #[tokio::test(flavor = "current_thread")]
10412 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
10413 let registry = Arc::new(Registry::default());
10414 let module_connection = ConnectionId::new(303);
10415 registry
10416 .register_with_control_ops(
10417 manifest("aft", PROTOCOL_VERSION),
10418 PROTOCOL_VERSION,
10419 module_connection,
10420 module_baseline_control_ops(),
10421 )
10422 .unwrap();
10423 let handler = ControlHandler::new(registry);
10424 let capture = EventCapture::default();
10425 let _subscriber =
10426 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10427 let (ctx, _rx) = route_ctx(module_connection);
10428 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10429
10430 let _ = handler
10431 .handle_client_control_request(
10432 &ctx,
10433 frame,
10434 ClientControlRequest::SupervisorSetEnabled {
10435 module_id: "broca".to_string(),
10436 enabled: false,
10437 },
10438 )
10439 .await
10440 .unwrap();
10441
10442 let callers = capture
10443 .events()
10444 .into_iter()
10445 .filter(|event| {
10446 event.fields.get("message").map(String::as_str)
10447 == Some("supervisor request received")
10448 })
10449 .map(|event| event.fields.get("caller").cloned())
10450 .collect::<Vec<_>>();
10451 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
10452 }
10453
10454 #[tokio::test(flavor = "current_thread")]
10455 async fn supervisor_list_does_not_log_request_received() {
10456 let handler = ControlHandler::new(Arc::new(Registry::default()));
10457 let capture = EventCapture::default();
10458 let _subscriber =
10459 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10460 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
10461 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
10462
10463 handler
10464 .handle_client_control_request(&ctx, frame, ClientControlRequest::SupervisorList {})
10465 .await
10466 .unwrap();
10467
10468 assert!(capture.events().into_iter().all(|event| {
10469 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10470 }));
10471 }
10472
10473 #[tokio::test(flavor = "current_thread")]
10474 async fn supervisor_rescan_preview_does_not_log_request_received() {
10475 let handler = ControlHandler::new(Arc::new(Registry::default()));
10476 let capture = EventCapture::default();
10477 let _subscriber =
10478 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10479 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
10480 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
10481
10482 handler
10483 .handle_client_control_request(
10484 &ctx,
10485 frame,
10486 ClientControlRequest::SupervisorRescan { preview: true },
10487 )
10488 .await
10489 .unwrap();
10490
10491 assert!(capture.events().into_iter().all(|event| {
10492 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10493 }));
10494 }
10495
10496 #[tokio::test]
10501 async fn route_open_refusal_names_the_check_that_refused() {
10502 let handler = ControlHandler::new(Arc::new(Registry::default()));
10503 let capture = EventCapture::default();
10504 let _subscriber =
10505 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10506 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10507 let response = handler
10508 .handle_control_frame(
10509 &ctx,
10510 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
10511 )
10512 .await
10513 .unwrap();
10514
10515 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10516 let event = capture
10517 .events()
10518 .into_iter()
10519 .find(|event| {
10520 event.target == "control"
10521 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10522 })
10523 .expect("route.open refusal event");
10524 assert_eq!(
10525 event.fields.get("reason"),
10526 Some(&"\"not_registered\"".to_string())
10527 );
10528 }
10529
10530 #[tokio::test]
10531 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
10532 let registry = Arc::new(Registry::default());
10533 let supervisor_handle = SupervisorHandle::new();
10534 let supervisor =
10535 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10536 .with_handle(supervisor_handle.clone())
10537 .with_connection_file_path(std::env::temp_dir().join(format!(
10538 "subc-route-open-refusal-info-{}",
10539 std::process::id()
10540 )));
10541 let module = supervisor
10542 .supervise_configured(
10543 ModuleSpec {
10544 module_id: "warming".to_string(),
10545 program: fake_aft_stub_path(),
10546 args: Vec::new(),
10547 env: Vec::new(),
10548 reserved: false,
10549 reserved_prefixes: Vec::new(),
10550 protocol: ModuleProtocol::Subc,
10551 overlap: Default::default(),
10552 },
10553 true,
10554 )
10555 .unwrap();
10556 assert_eq!(module.state().unwrap(), ModuleState::Running);
10557
10558 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10559 assert!(handler
10560 .counters()
10561 .snapshot()
10562 .get("route_open_refused_by_code")
10563 .is_none());
10564 let capture = EventCapture::default();
10565 let _subscriber =
10566 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10567 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
10568 let response = handler
10569 .handle_control_frame(
10570 &ctx,
10571 route_open_frame(394, "warming", unique_project_root("refusal-info")),
10572 )
10573 .await
10574 .unwrap();
10575 module.stop().await.unwrap();
10576
10577 assert_eq!(parse_error(&response[0])["code"], "module_warming");
10578 let event = capture
10579 .events()
10580 .into_iter()
10581 .find(|event| {
10582 event.target == "control"
10583 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
10584 })
10585 .expect("route.open refusal event");
10586 assert_eq!(
10587 event.fields.get("module_id"),
10588 Some(&"\"warming\"".to_string())
10589 );
10590 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
10591 assert_eq!(
10592 event.fields.get("reason"),
10593 Some(&"\"supervised_not_registered\"".to_string())
10594 );
10595 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
10596 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
10597 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
10598 assert_eq!(
10599 handler.counters().snapshot()["route_open_refused_by_code"],
10600 json!({ "module_warming": 1 })
10601 );
10602 }
10603
10604 const OUTAGE_START: &str = "route.open refusing module: not serving";
10605 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
10606
10607 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
10608 capture
10609 .events()
10610 .into_iter()
10611 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
10612 .collect()
10613 }
10614
10615 fn supervise_stub(
10616 registry: &Arc<Registry>,
10617 module_id: &str,
10618 enabled: bool,
10619 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
10620 let supervisor_handle = SupervisorHandle::new();
10621 let supervisor =
10622 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
10623 .with_handle(supervisor_handle.clone())
10624 .with_connection_file_path(std::env::temp_dir().join(format!(
10625 "subc-route-outage-{module_id}-{}",
10626 std::process::id()
10627 )));
10628 let module = supervisor
10629 .supervise_configured(
10630 ModuleSpec {
10631 module_id: module_id.to_string(),
10632 program: fake_aft_stub_path(),
10633 args: Vec::new(),
10634 env: Vec::new(),
10635 reserved: false,
10636 reserved_prefixes: Vec::new(),
10637 protocol: ModuleProtocol::Subc,
10638 overlap: Default::default(),
10639 },
10640 enabled,
10641 )
10642 .unwrap();
10643 (supervisor_handle, module)
10644 }
10645
10646 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
10647 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
10648 module_id: module_id.to_string(),
10649 drain_timeout_ms: Some(50),
10650 })
10651 .unwrap();
10652 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
10653 }
10654
10655 #[test]
10659 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
10660 let registry = Arc::new(Registry::default());
10661 let forwarding = Arc::new(ForwardingTable::default());
10662 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10663 let second = ControlHandler::with_forwarding(registry, forwarding);
10664 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
10665 }
10666
10667 #[tokio::test(flavor = "current_thread")]
10671 async fn route_open_unknown_module_refusals_add_no_outage_state() {
10672 let handler = ControlHandler::new(Arc::new(Registry::default()));
10673 let capture = EventCapture::default();
10674 let _subscriber =
10675 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10676 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
10677 for corr in 0..8 {
10678 let response = handler
10679 .handle_control_frame(
10680 &ctx,
10681 route_open_frame(
10682 380 + corr,
10683 &format!("nobody-{corr}"),
10684 unique_project_root("outage-unknown"),
10685 ),
10686 )
10687 .await
10688 .unwrap();
10689 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10690 }
10691
10692 assert_eq!(handler.route_outages.tracked_module_count(), 0);
10693 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
10694 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
10695 }
10696
10697 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10702 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
10703 let registry = Arc::new(Registry::default());
10704 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
10705 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10706 let capture = EventCapture::default();
10707 let _subscriber =
10708 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10709 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
10710 handler.route_outages.record_accepted("outage-restart");
10713
10714 let response = handler
10715 .handle_control_frame(
10716 &ctx,
10717 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
10718 )
10719 .await
10720 .unwrap();
10721 assert_eq!(response[0].header.ty, FrameType::Error);
10722 let starts = outage_lines(&capture, OUTAGE_START);
10723 assert_eq!(starts.len(), 1, "{starts:?}");
10724 assert_eq!(starts[0].level, tracing::Level::WARN);
10725 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
10726 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
10727 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
10728 handler.route_outages.record_accepted("outage-restart");
10729 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
10730
10731 let restart = handler
10732 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
10733 .await
10734 .unwrap();
10735 assert_eq!(
10736 restart[0].header.ty,
10737 FrameType::Response,
10738 "{:?}",
10739 parse_error(&restart[0])
10740 );
10741 handler
10742 .handle_control_frame(
10743 &ctx,
10744 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
10745 )
10746 .await
10747 .unwrap();
10748 module.stop().await.unwrap();
10749
10750 let starts = outage_lines(&capture, OUTAGE_START);
10751 assert_eq!(starts.len(), 2, "{starts:?}");
10752 assert_eq!(starts[1].level, tracing::Level::INFO);
10753 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
10754 }
10755
10756 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10760 async fn failed_operator_restart_leaves_no_operator_mark() {
10761 let registry = Arc::new(Registry::default());
10762 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
10763 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10764 let capture = EventCapture::default();
10765 let _subscriber =
10766 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10767 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
10768 handler.route_outages.record_accepted("outage-disabled");
10769
10770 let restart = handler
10771 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
10772 .await
10773 .unwrap();
10774 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
10775 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
10776
10777 handler
10778 .handle_control_frame(
10779 &ctx,
10780 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
10781 )
10782 .await
10783 .unwrap();
10784 let starts = outage_lines(&capture, OUTAGE_START);
10785 assert_eq!(starts.len(), 1, "{starts:?}");
10786 assert_eq!(starts[0].level, tracing::Level::WARN);
10787 }
10788
10789 #[tokio::test(flavor = "current_thread")]
10790 async fn route_open_unknown_module_escapes_target_module_id() {
10791 let handler = ControlHandler::new(Arc::new(Registry::default()));
10792 let capture = EventCapture::default();
10793 let _subscriber =
10794 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10795 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
10796 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10797 let response = handler
10798 .handle_control_frame(
10799 &ctx,
10800 route_open_frame(
10801 395,
10802 hostile_module_id,
10803 unique_project_root("hostile-target-module-id"),
10804 ),
10805 )
10806 .await
10807 .unwrap();
10808
10809 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10810 let event = capture
10811 .events()
10812 .into_iter()
10813 .find(|event| {
10814 event.target == "control"
10815 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10816 })
10817 .expect("route.open unknown-module refusal event");
10818 let logged = event.fields.get("module_id").expect("module_id field");
10819 assert!(!logged.bytes().any(|byte| byte < 0x20));
10820 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10821 }
10822
10823 #[tokio::test(flavor = "current_thread")]
10824 async fn route_open_module_rejection_uses_daemon_counter_key() {
10825 let registry = Arc::new(Registry::default());
10826 let forwarding = Arc::new(ForwardingTable::default());
10827 let handler =
10828 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10829 let module_connection = ConnectionId::new(95);
10830 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10831 hello_via_sink(
10832 &handler,
10833 &module_ctx,
10834 &mut module_rx,
10835 hello_frame("aft", PROTOCOL_VERSION, 395),
10836 )
10837 .await;
10838
10839 let client_connection = ConnectionId::new(96);
10840 let (client_ctx, _client_rx) = route_ctx(client_connection);
10841 let capture = EventCapture::default();
10842 let _subscriber =
10843 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10844 let (route_task, bind) = relay_route_open(
10845 &handler,
10846 client_connection,
10847 &client_ctx.egress,
10848 &mut module_rx,
10849 396,
10850 "aft",
10851 "hostile-module-code",
10852 )
10853 .await;
10854 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
10855 let rejection = Frame::build(
10856 FrameType::Error,
10857 control_flags(),
10858 0,
10859 0,
10860 bind.header.corr,
10861 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
10862 )
10863 .unwrap();
10864 handler
10865 .handle_control_frame(&module_ctx, rejection)
10866 .await
10867 .unwrap();
10868
10869 let response = route_task.await.unwrap();
10870 assert_eq!(parse_error(&response[0])["code"], hostile_code);
10871 let counters = handler.counters().snapshot();
10872 assert_eq!(
10873 counters["route_open_refused_by_code"],
10874 json!({ "module_rejected": 1 })
10875 );
10876 assert!(counters["route_open_refused_by_code"]
10877 .get(hostile_code)
10878 .is_none());
10879
10880 let event = capture
10881 .events()
10882 .into_iter()
10883 .find(|event| {
10884 event.target == "control"
10885 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
10886 })
10887 .expect("route.open module-rejection refusal event");
10888 let logged = event.fields.get("module_code").expect("module_code field");
10889 assert!(!logged.bytes().any(|byte| byte < 0x20));
10890 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10891 }
10892
10893 #[tokio::test]
10894 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
10895 let registry = Arc::new(Registry::default());
10896 let supervisor_handle = SupervisorHandle::new();
10897 let missing_program = std::env::temp_dir().join(format!(
10898 "subc-route-open-missing-program-{}",
10899 std::process::id()
10900 ));
10901 let supervisor =
10902 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10903 .with_handle(supervisor_handle.clone());
10904 let module = supervisor
10905 .supervise_configured(
10906 ModuleSpec {
10907 module_id: "failed".to_string(),
10908 program: missing_program,
10909 args: Vec::new(),
10910 env: Vec::new(),
10911 reserved: false,
10912 reserved_prefixes: Vec::new(),
10913 protocol: ModuleProtocol::Subc,
10914 overlap: Default::default(),
10915 },
10916 true,
10917 )
10918 .unwrap();
10919 assert_eq!(module.state().unwrap(), ModuleState::Failed);
10920
10921 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10922 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
10923 let response = handler
10924 .handle_control_frame(
10925 &ctx,
10926 route_open_frame(305, "failed", unique_project_root("failed")),
10927 )
10928 .await
10929 .unwrap();
10930
10931 assert_eq!(response[0].header.ty, FrameType::Error);
10932 let error = parse_error(&response[0]);
10933 assert_eq!(error["code"], "target_unavailable");
10934 assert!(error["message"]
10935 .as_str()
10936 .unwrap()
10937 .contains("state=failed, enabled=true, live=false"));
10938 }
10939
10940 #[tokio::test]
10941 async fn route_open_role_mismatch_remains_target_unavailable() {
10942 let registry = Arc::new(Registry::default());
10943 let handler = ControlHandler::new(Arc::clone(®istry));
10944 handler
10945 .handle_control(
10946 ConnectionId::new(41),
10947 non_routable_hello_frame_with_control_ops("health-only", 306, None),
10948 )
10949 .unwrap();
10950
10951 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
10952 let response = handler
10953 .handle_control_frame(
10954 &ctx,
10955 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
10956 )
10957 .await
10958 .unwrap();
10959
10960 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10961 assert!(parse_error(&response[0])["message"]
10962 .as_str()
10963 .unwrap()
10964 .contains("does not provide the requested target"));
10965 }
10966
10967 #[tokio::test]
10968 async fn route_open_inactive_registration_remains_target_unavailable() {
10969 let registry = Arc::new(Registry::default());
10970 let handler = ControlHandler::new(Arc::clone(®istry));
10971 handler
10972 .handle_control(
10973 ConnectionId::new(43),
10974 hello_frame("inactive", PROTOCOL_VERSION, 308),
10975 )
10976 .unwrap();
10977 assert!(registry
10978 .set_module_state_for_test("inactive", ChannelState::Closed)
10979 .unwrap());
10980
10981 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
10982 let response = handler
10983 .handle_control_frame(
10984 &ctx,
10985 route_open_frame(309, "inactive", unique_project_root("inactive")),
10986 )
10987 .await
10988 .unwrap();
10989
10990 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10991 assert!(parse_error(&response[0])["message"]
10992 .as_str()
10993 .unwrap()
10994 .contains("is not active"));
10995 }
10996
10997 #[tokio::test]
10998 async fn late_health_reply_is_recorded_through_the_module_response_path() {
10999 let registry = Arc::new(Registry::default());
11000 let forwarding = Arc::new(ForwardingTable::default());
11001 let supervisor_handle = SupervisorHandle::new();
11002 let supervisor =
11003 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
11004 .with_forwarding(Arc::clone(&forwarding))
11005 .with_handle(supervisor_handle.clone());
11006 let module = supervisor
11007 .supervise_configured(
11008 crate::ModuleSpec {
11009 module_id: "late-health-response".to_string(),
11010 program: PathBuf::from("disabled-module"),
11011 args: Vec::new(),
11012 env: Vec::new(),
11013 reserved: false,
11014 reserved_prefixes: Vec::new(),
11015 protocol: ModuleProtocol::Subc,
11016 overlap: Default::default(),
11017 },
11018 false,
11019 )
11020 .unwrap();
11021 let handler =
11022 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11023 .with_supervisor(supervisor_handle);
11024 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
11025 handler
11026 .handle_control_frame(
11027 &module_ctx,
11028 hello_frame_with_control_ops(
11029 "late-health-response",
11030 PROTOCOL_VERSION,
11031 7,
11032 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11033 ),
11034 )
11035 .await
11036 .unwrap();
11037 let probe_started_at = Instant::now() - Duration::from_millis(80);
11038 let pending = forwarding
11039 .begin_health_probe_rpc_for(
11040 "late-health-response",
11041 MODULE_CONTROL_OP_HEALTH_CHECK,
11042 probe_started_at,
11043 Instant::now() - Duration::from_millis(1),
11044 )
11045 .unwrap();
11046 assert!(forwarding
11047 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
11048 .unwrap());
11049
11050 let responses = handler
11051 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
11052 .await
11053 .unwrap();
11054
11055 assert!(responses.is_empty());
11056 let health = module.status().unwrap().health;
11057 assert_eq!(health.late_answer_count, 1);
11058 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
11059 }
11060
11061 #[tokio::test]
11062 async fn health_probe_timeout_and_module_death_are_typed() {
11063 let registry = Arc::new(Registry::default());
11064 let forwarding = Arc::new(ForwardingTable::default());
11065 let handler =
11066 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11067 .with_health_probe_timeout(Duration::from_millis(50));
11068 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11069 hello_via_sink(
11070 &handler,
11071 &module_ctx,
11072 &mut module_rx,
11073 hello_frame_with_control_ops(
11074 "aft",
11075 PROTOCOL_VERSION,
11076 7,
11077 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11078 ),
11079 )
11080 .await;
11081
11082 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11083 let responses = handler
11084 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
11085 .await
11086 .unwrap();
11087 assert_eq!(responses[0].header.ty, FrameType::Error);
11088 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
11089 let _ = module_rx.try_recv();
11090
11091 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
11092 let health_handler = handler.clone();
11093 let death_task = tokio::spawn(async move {
11094 health_handler
11095 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
11096 .await
11097 .unwrap()
11098 });
11099 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11100 .await
11101 .unwrap()
11102 .unwrap();
11103 handler
11104 .cleanup_connection(module_ctx.connection_id)
11105 .unwrap();
11106 let responses = death_task.await.unwrap();
11107 assert_eq!(responses[0].header.ty, FrameType::Error);
11108 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
11109 }
11110
11111 #[test]
11112 fn hello_requires_exact_protocol_version() {
11113 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
11114 let registry = Arc::new(Registry::default());
11115 let handler = ControlHandler::new(Arc::clone(®istry));
11116 let responses = handler
11117 .handle_control(
11118 ConnectionId::new(connection),
11119 hello_frame("aft", offered, 9),
11120 )
11121 .unwrap();
11122
11123 assert_eq!(responses.len(), 1);
11124 assert_eq!(responses[0].header.ty, FrameType::Error);
11125 let error = parse_error(&responses[0]);
11126 assert_eq!(error["code"], "version_unsupported");
11127 assert!(registry.get_module("aft").unwrap().is_none());
11128 assert_eq!(registry.active_registration_count().unwrap(), 0);
11129 }
11130 }
11131
11132 #[test]
11133 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
11134 let registry = Arc::new(Registry::default());
11135 let forwarding = Arc::new(ForwardingTable::default());
11136 let handler =
11137 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11138 let module_connection = ConnectionId::new(301);
11139 let registration = registry
11140 .register_with_control_ops(
11141 manifest("aft-push", PROTOCOL_VERSION),
11142 PROTOCOL_VERSION,
11143 module_connection,
11144 module_baseline_control_ops(),
11145 )
11146 .unwrap();
11147 let (module_tx, _module_rx) = mpsc::channel(8);
11148 let endpoint = forwarding
11149 .register_module_connection(
11150 module_connection,
11151 "aft-push".to_string(),
11152 PROTOCOL_VERSION,
11153 manifest_concurrency(®istration.manifest),
11154 FrameSink::new(module_tx),
11155 )
11156 .unwrap();
11157
11158 let unknown = Frame::build(
11160 FrameType::Push,
11161 control_flags(),
11162 0,
11163 0,
11164 5,
11165 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
11166 )
11167 .unwrap();
11168 let out = handler.handle_status_update(endpoint, unknown).unwrap();
11169 assert!(
11170 out.is_empty(),
11171 "unknown push op must be ignored, got {out:?}"
11172 );
11173
11174 let malformed = Frame::build(
11176 FrameType::Push,
11177 control_flags(),
11178 0,
11179 0,
11180 6,
11181 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
11182 )
11183 .unwrap();
11184 let out = handler.handle_status_update(endpoint, malformed).unwrap();
11185 assert_eq!(out.len(), 1);
11186 assert_eq!(out[0].header.ty, FrameType::Error);
11187 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
11188 }
11189
11190 #[test]
11191 fn hello_rejected_when_connection_already_owns_client_routes() {
11192 let registry = Arc::new(Registry::default());
11193 let forwarding = Arc::new(ForwardingTable::default());
11194 let handler =
11195 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11196 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
11198 let client_connection = ConnectionId::new(202);
11199
11200 let responses = handler
11203 .handle_control(
11204 client_connection,
11205 hello_frame("aft-second", PROTOCOL_VERSION, 9),
11206 )
11207 .unwrap();
11208 assert_eq!(responses[0].header.ty, FrameType::Error);
11209 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
11210 assert!(registry.get_module("aft-second").unwrap().is_none());
11211 }
11212
11213 #[tokio::test]
11214 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
11215 let registry = Arc::new(Registry::default());
11216 let forwarding = Arc::new(ForwardingTable::default());
11217 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
11218 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
11219 hello_via_sink(
11220 &handler,
11221 &module_ctx,
11222 &mut module_rx,
11223 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
11224 )
11225 .await;
11226 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
11227 let pending = forwarding
11228 .begin_route_bind_relay_for_test(
11229 client_ctx.connection_id,
11230 client_ctx.egress.clone(),
11231 2,
11232 "alpha",
11233 )
11234 .unwrap();
11235 forwarding
11236 .complete_pending_relay(
11237 module_ctx.connection_id,
11238 pending.corr,
11239 RouteBindRelayOutcome::Accepted,
11240 )
11241 .unwrap();
11242 client_rx.try_recv().unwrap();
11243 for module_id in ["beta", "alpha"] {
11244 let replies = handler
11245 .handle_control_frame(
11246 &module_ctx,
11247 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
11248 )
11249 .await
11250 .unwrap();
11251 assert_eq!(replies.len(), 1, "second HELLO must be refused");
11252 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
11253 }
11254 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
11255 assert!(registry.get_module("beta").unwrap().is_none());
11256 assert!(matches!(
11257 forwarding
11258 .lookup_data_route(
11259 client_ctx.connection_id,
11260 pending.client_channel,
11261 pending.client_epoch,
11262 )
11263 .unwrap(),
11264 DataRoute::Client(DataRouteState::Bound(_))
11265 ));
11266 assert!(handler
11267 .hello_launch_nonces
11268 .lock()
11269 .unwrap()
11270 .presented(module_ctx.connection_id, Some("alpha-nonce")));
11271 assert!(module_rx.try_recv().is_err());
11272 }
11273
11274 #[test]
11275 fn reserved_module_hello_requires_matching_launch_nonce() {
11276 let registry = Arc::new(Registry::default());
11277 let supervisor = SupervisorHandle::new();
11278 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
11281 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11282
11283 let no_nonce = handler
11285 .handle_control(
11286 ConnectionId::new(1),
11287 hello_frame("vault", PROTOCOL_VERSION, 1),
11288 )
11289 .unwrap();
11290 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
11291 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
11292 assert!(registry.get_module("vault").unwrap().is_none());
11293
11294 let wrong = handler
11296 .handle_control(
11297 ConnectionId::new(2),
11298 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
11299 )
11300 .unwrap();
11301 assert_eq!(wrong[0].header.ty, FrameType::Error);
11302 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
11303 assert!(registry.get_module("vault").unwrap().is_none());
11304
11305 let ok = handler
11307 .handle_control(
11308 ConnectionId::new(3),
11309 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
11310 )
11311 .unwrap();
11312 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
11313 assert!(registry.get_module("vault").unwrap().is_some());
11314 }
11315
11316 #[test]
11317 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
11318 let registry = Arc::new(Registry::default());
11319 let supervisor = SupervisorHandle::new();
11320 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11321 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11322 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11323
11324 let squat = handler
11325 .handle_control(
11326 ConnectionId::new(1),
11327 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
11328 )
11329 .unwrap();
11330 assert_eq!(squat[0].header.ty, FrameType::Error);
11331 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
11332 assert!(parse_error(&squat[0])["message"]
11333 .as_str()
11334 .unwrap()
11335 .contains("fed:"));
11336
11337 let accepted_peer = handler
11338 .handle_control(
11339 ConnectionId::new(2),
11340 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
11341 )
11342 .unwrap();
11343 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
11344
11345 let accepted_short = handler
11346 .handle_control(
11347 ConnectionId::new(3),
11348 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
11349 )
11350 .unwrap();
11351 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
11352
11353 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
11354 let response = handler
11355 .handle_control(
11356 ConnectionId::new(conn),
11357 hello_frame(module_id, PROTOCOL_VERSION, conn),
11358 )
11359 .unwrap();
11360 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
11361 }
11362 }
11363
11364 #[test]
11365 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
11366 let registry = Arc::new(Registry::default());
11367 let supervisor = SupervisorHandle::new();
11368 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11369 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11370 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
11371 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11372
11373 let owner_nonce = handler
11374 .handle_control(
11375 ConnectionId::new(1),
11376 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
11377 )
11378 .unwrap();
11379 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
11380 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
11381 assert!(registry.get_module("fed:special").unwrap().is_none());
11382
11383 let exact_nonce = handler
11384 .handle_control(
11385 ConnectionId::new(2),
11386 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
11387 )
11388 .unwrap();
11389 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
11390 assert!(registry.get_module("fed:special").unwrap().is_some());
11391 }
11392
11393 #[test]
11394 fn non_reserved_module_ignores_launch_nonce() {
11395 let registry = Arc::new(Registry::default());
11396 let handler = ControlHandler::new(Arc::clone(®istry));
11399 let no_nonce = handler
11400 .handle_control(
11401 ConnectionId::new(1),
11402 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
11403 )
11404 .unwrap();
11405 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
11406 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
11407
11408 let echoed_nonce = handler
11409 .handle_control(
11410 ConnectionId::new(2),
11411 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
11412 )
11413 .unwrap();
11414 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
11415 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
11416 }
11417
11418 #[test]
11419 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
11420 let handler = ControlHandler::default();
11421 let conn = ConnectionId::new(1);
11422 let malformed = Frame::build(
11423 FrameType::Hello,
11424 control_flags(),
11425 0,
11426 0,
11427 3,
11428 b"{not json".to_vec(),
11429 )
11430 .unwrap();
11431
11432 let error = handler.handle_control(conn, malformed).unwrap();
11433 assert_eq!(error[0].header.ty, FrameType::Error);
11434 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
11435
11436 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11437 let pong = handler.handle_control(conn, ping).unwrap();
11438 assert_eq!(pong[0].header.ty, FrameType::Pong);
11439 assert_eq!(pong[0].header.corr, 4);
11440 }
11441
11442 #[test]
11443 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
11444 let registry = Arc::new(Registry::default());
11445 let handler = ControlHandler::new(Arc::clone(®istry));
11446
11447 handler
11448 .handle_control(
11449 ConnectionId::new(1),
11450 hello_frame("aft", PROTOCOL_VERSION, 1),
11451 )
11452 .unwrap();
11453 let duplicate = handler
11454 .handle_control(
11455 ConnectionId::new(2),
11456 hello_frame("aft", PROTOCOL_VERSION, 2),
11457 )
11458 .unwrap();
11459
11460 assert_eq!(duplicate[0].header.ty, FrameType::Error);
11461 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11462 let registration = registry.get_module("aft").unwrap().unwrap();
11463 assert_eq!(registration.connection_id, ConnectionId::new(1));
11464 }
11465
11466 #[test]
11467 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
11468 let registry = Arc::new(Registry::default());
11469 let forwarding = Arc::new(ForwardingTable::default());
11470 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
11471 let handler =
11472 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11473 .with_process_liveness(process_liveness);
11474 let (ctx, route_channel, route_epoch) =
11475 bind_liveness_route(®istry, &forwarding, "aft-dead");
11476 let responses = handler
11477 .handle_route_poll(
11478 &ctx,
11479 route_poll_frame(41, PollKind::Liveness, route_channel),
11480 route_channel,
11481 route_epoch,
11482 PollKind::Liveness,
11483 )
11484 .unwrap();
11485
11486 assert_eq!(responses.len(), 1);
11487 assert_eq!(responses[0].header.ty, FrameType::Response);
11488 assert_route_poll_liveness(&responses[0], false);
11489 }
11490
11491 #[test]
11492 fn liveness_poll_without_process_source_uses_bound_route() {
11493 let registry = Arc::new(Registry::default());
11494 let forwarding = Arc::new(ForwardingTable::default());
11495 let handler =
11496 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11497 let (ctx, route_channel, route_epoch) =
11498 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
11499 let responses = handler
11500 .handle_route_poll(
11501 &ctx,
11502 route_poll_frame(42, PollKind::Liveness, route_channel),
11503 route_channel,
11504 route_epoch,
11505 PollKind::Liveness,
11506 )
11507 .unwrap();
11508
11509 assert_route_poll_liveness(&responses[0], true);
11510 }
11511
11512 #[test]
11513 fn liveness_poll_untracked_process_source_uses_bound_route() {
11514 let registry = Arc::new(Registry::default());
11515 let forwarding = Arc::new(ForwardingTable::default());
11516 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
11517 let handler =
11518 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11519 .with_process_liveness(process_liveness);
11520 let (ctx, route_channel, route_epoch) =
11521 bind_liveness_route(®istry, &forwarding, "aft-untracked");
11522 let responses = handler
11523 .handle_route_poll(
11524 &ctx,
11525 route_poll_frame(43, PollKind::Liveness, route_channel),
11526 route_channel,
11527 route_epoch,
11528 PollKind::Liveness,
11529 )
11530 .unwrap();
11531
11532 assert_route_poll_liveness(&responses[0], true);
11533 }
11534
11535 #[tokio::test]
11536 async fn unknown_op_returns_unknown_control_op() {
11537 let handler = ControlHandler::default();
11538 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
11539 let request = Frame::build(
11540 FrameType::Request,
11541 control_flags(),
11542 0,
11543 0,
11544 55,
11545 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
11546 )
11547 .unwrap();
11548
11549 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11550
11551 assert_eq!(response.len(), 1);
11552 assert_eq!(response[0].header.ty, FrameType::Error);
11553 assert_eq!(response[0].header.corr, 55);
11554 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
11555 }
11556
11557 #[tokio::test]
11558 async fn supervisor_provenance_rejects_unknown_exact_module() {
11559 let handler = ControlHandler::default();
11560 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
11561 let request = Frame::build(
11562 FrameType::Request,
11563 control_flags(),
11564 0,
11565 0,
11566 57,
11567 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
11568 )
11569 .unwrap();
11570
11571 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11572
11573 assert_eq!(response.len(), 1);
11574 assert_eq!(response[0].header.ty, FrameType::Error);
11575 assert_eq!(response[0].header.corr, 57);
11576 let error = parse_error(&response[0]);
11577 assert_eq!(error["code"], "unknown_module");
11578 assert_eq!(error["message"], "module_id 'missing' is not supervised");
11579 }
11580
11581 #[test]
11582 fn provenance_probe_override_keeps_handler_tests_deterministic() {
11583 let expected = subc_control::RunningImageAgreement::Unavailable {
11584 reason: subc_control::RunningImageUnavailableReason::HashFailed,
11585 };
11586 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
11587 assert_eq!(handler.provenance_probe_override, Some(expected));
11588 }
11589
11590 #[test]
11591 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
11592 let verdict = reload_verdict(
11593 std::path::Path::new("/bin/new"),
11594 Some(std::path::Path::new("/bin/old")),
11595 subc_control::RunningImageAgreement::Unavailable {
11596 reason: subc_control::RunningImageUnavailableReason::HashFailed,
11597 },
11598 );
11599 assert!(matches!(
11600 verdict.path,
11601 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
11602 if configured == std::path::Path::new("/bin/new")
11603 && spawned_from == std::path::Path::new("/bin/old")
11604 ));
11605 }
11606
11607 #[test]
11608 fn reload_verdict_detects_replaced_image_at_same_path() {
11609 let image = subc_control::RunningImageAgreement::Mismatch {
11610 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
11611 digest: "old".into(),
11612 },
11613 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
11614 digest: "new".into(),
11615 },
11616 };
11617 let verdict = reload_verdict(
11618 std::path::Path::new("/bin/same"),
11619 Some(std::path::Path::new("/bin/same")),
11620 image.clone(),
11621 );
11622 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11623 assert_eq!(verdict.image, image);
11624 }
11625
11626 #[test]
11627 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
11628 let image = subc_control::RunningImageAgreement::Unavailable {
11629 reason: subc_control::RunningImageUnavailableReason::NotRunning,
11630 };
11631 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
11632 assert_eq!(
11633 verdict.path,
11634 subc_control::ReloadPathAgreement::Unavailable {
11635 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
11636 }
11637 );
11638 assert_eq!(verdict.image, image);
11639
11640 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
11641 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11642 };
11643 let verdict = reload_verdict(
11644 std::path::Path::new("/bin/same"),
11645 Some(std::path::Path::new("/bin/same")),
11646 unconfirmed.clone(),
11647 );
11648 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11649 assert_eq!(verdict.image, unconfirmed);
11650 }
11651
11652 #[test]
11653 fn reload_verdict_preserves_each_image_unavailability_reason() {
11654 use subc_control::RunningImageUnavailableReason as Reason;
11655
11656 for reason in [
11657 Reason::NotRunning,
11658 Reason::UnsupportedPlatform,
11659 Reason::RunningExecutableUnreadable,
11660 Reason::SpawnedPathUnreadable,
11661 Reason::HashFailed,
11662 Reason::ProcessIdentityUnconfirmed,
11663 Reason::Unknown("future_probe_reason".to_string()),
11664 ] {
11665 let image = subc_control::RunningImageAgreement::Unavailable {
11666 reason: reason.clone(),
11667 };
11668 let verdict = reload_verdict(
11669 std::path::Path::new("/bin/same"),
11670 Some(std::path::Path::new("/bin/same")),
11671 image.clone(),
11672 );
11673 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11674 assert_eq!(verdict.image, image, "{reason:?}");
11675 }
11676 }
11677
11678 #[tokio::test]
11679 async fn malformed_control_bodies_return_invalid_control_body() {
11680 let handler = ControlHandler::default();
11681 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
11682
11683 for (corr, body) in [
11684 (56, br#"{"route_channel":1}"#.as_slice()),
11685 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
11686 (
11687 58,
11688 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
11689 ),
11690 ] {
11691 let request = Frame::build(
11692 FrameType::Request,
11693 control_flags(),
11694 0,
11695 0,
11696 corr,
11697 body.to_vec(),
11698 )
11699 .unwrap();
11700 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11701
11702 assert_eq!(response.len(), 1);
11703 assert_eq!(response[0].header.ty, FrameType::Error);
11704 assert_eq!(response[0].header.corr, corr);
11705 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
11706 }
11707 }
11708
11709 #[tokio::test]
11710 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
11711 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11712 let registry = Arc::new(Registry::default());
11713 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11714 let router = Router::with_control_handler(Arc::clone(&control));
11715 let connection = router.begin_connection();
11716 let (ctx, mut rx) = route_ctx(connection.id());
11717
11718 router
11719 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
11720 .await
11721 .unwrap();
11722 let response = rx.recv().await.unwrap();
11723 let ack = parse_ack(&response);
11724 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11725 let channel = 1;
11726
11727 let goodbye =
11728 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
11729 router.route_for_connection(&ctx, goodbye).await.unwrap();
11730 assert!(rx.try_recv().is_err());
11731 assert!(registry.get_module("aft").unwrap().is_none());
11732
11733 router
11734 .route_for_connection(&ctx, channel_request(channel, 13))
11735 .await
11736 .unwrap();
11737 let error_frame = rx.recv().await.unwrap();
11738 assert_eq!(error_frame.header.ty, FrameType::Error);
11739 assert_eq!(error_frame.header.channel, channel);
11740 let captured = crate::router::test_log::captured_logs(&logs);
11741 assert_eq!(
11742 captured
11743 .lines()
11744 .filter(|line| {
11745 line.contains("module_id=aft")
11746 && line.contains("reason=explicit_goodbye")
11747 && line.contains("module registration ended")
11748 })
11749 .count(),
11750 1,
11751 "unexpected GOODBYE registry log: {captured}"
11752 );
11753 }
11754
11755 #[tokio::test]
11756 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
11757 let registry = Arc::new(Registry::default());
11758 let handler = ControlHandler::new(registry).with_capability_config(
11759 [("prov".to_string(), true), ("cons".to_string(), true)],
11760 BTreeMap::new(),
11761 );
11762 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
11763 register_capability_manifest(
11764 &handler,
11765 &provider_ctx,
11766 &mut provider_rx,
11767 capability_manifest("prov", &["thing/v1"], &[]),
11768 1,
11769 )
11770 .await;
11771 let mut consumer = capability_manifest("cons", &[], &[]);
11772 consumer.capabilities.as_mut().unwrap().requires.push(
11773 subc_protocol::manifest::CapabilityRequirement {
11774 capability: "thing/v1".to_string(),
11775 need: subc_protocol::manifest::CapabilityNeed::Required,
11776 },
11777 );
11778 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
11779 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
11780 assert_eq!(
11781 handler.capability_evaluator.verdict("cons", "thing/v1"),
11782 Some(CapabilityVerdict::Provided)
11783 );
11784 let (mut client_rx, _) = open_route_for_capability_test(
11785 &handler,
11786 &provider_ctx,
11787 &mut provider_rx,
11788 703,
11789 3,
11790 "prov",
11791 None,
11792 )
11793 .await;
11794 let goodbye =
11795 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11796 handler
11797 .handle_control_frame(&provider_ctx, goodbye)
11798 .await
11799 .unwrap();
11800 assert_eq!(
11801 handler.capability_evaluator.verdict("cons", "thing/v1"),
11802 Some(CapabilityVerdict::NeverProvided)
11803 );
11804 let closed = client_rx
11805 .try_recv()
11806 .expect("GOODBYE pushes route.closed before route GOODBYE");
11807 assert!(
11808 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
11809 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
11810 );
11811 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
11812 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11813 }
11814
11815 #[tokio::test]
11816 async fn dropping_router_connection_releases_registration() {
11817 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11818 let registry = Arc::new(Registry::default());
11819 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11820 let router = Router::with_control_handler(Arc::clone(&control));
11821 let connection = router.begin_connection();
11822 let connection_id = connection.id();
11823 let (ctx, mut rx) = route_ctx(connection_id);
11824
11825 router
11826 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
11827 .await
11828 .unwrap();
11829 let response = rx.recv().await.unwrap();
11830 let ack = parse_ack(&response);
11831 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11832 assert!(registry.get_module("aft").unwrap().is_some());
11833
11834 drop(connection);
11835
11836 assert!(registry.get_module("aft").unwrap().is_none());
11837 assert_eq!(registry.active_registration_count().unwrap(), 0);
11838
11839 control
11840 .cleanup_connection(ConnectionId::new(u64::MAX))
11841 .unwrap();
11842 let captured = crate::router::test_log::captured_logs(&logs);
11843 println!("captured registry lifecycle logs:\n{captured}");
11844 let events: Vec<_> = captured
11845 .lines()
11846 .filter(|line| {
11847 line.contains("module registered module_id=aft ")
11848 || line.contains("module registration ended")
11849 })
11850 .collect();
11851 assert_eq!(
11852 events.len(),
11853 2,
11854 "unexpected registry lifecycle logs: {captured}"
11855 );
11856 assert!(events[0].contains("module registered module_id=aft "));
11857 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
11858 assert!(events[1].contains(&format!(
11859 "module_id=aft connection_id={}",
11860 connection_id.get()
11861 )));
11862 assert!(events[1].contains("reason=connection_closed"));
11863 assert!(events[1].contains("module registration ended"));
11864 assert_eq!(
11865 captured
11866 .lines()
11867 .filter(|line| line.contains("module registered module_id=aft "))
11868 .count(),
11869 1,
11870 "legacy registration admission line must appear once: {captured}"
11871 );
11872 }
11873
11874 #[tokio::test]
11875 async fn hello_registration_keeps_legacy_module_registered_line_once() {
11876 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11877 let registry = Arc::new(Registry::default());
11878 let control = ControlHandler::new(Arc::clone(®istry));
11879 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
11880 hello_via_sink(
11881 &control,
11882 &ctx,
11883 &mut rx,
11884 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
11885 )
11886 .await;
11887
11888 let captured = crate::router::test_log::captured_logs(&logs);
11889 let admissions: Vec<_> = captured
11890 .lines()
11891 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
11892 .collect();
11893 assert_eq!(
11894 admissions.len(),
11895 1,
11896 "legacy admission line must remain exactly once: {captured}"
11897 );
11898 assert!(admissions[0].contains("routable_provider=true"));
11899 assert!(admissions[0].contains("connection_id=777"));
11900 }
11901
11902 fn capability_manifest(
11903 module_id: &str,
11904 provides: &[&str],
11905 must_never_reach: &[&str],
11906 ) -> ModuleManifest {
11907 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11908 manifest.capabilities = Some(CapabilityDeclarations {
11909 provides: provides
11910 .iter()
11911 .map(|capability| (*capability).to_string())
11912 .collect(),
11913 requires: Vec::new(),
11914 must_never_reach: must_never_reach
11915 .iter()
11916 .map(|capability| (*capability).to_string())
11917 .collect(),
11918 });
11919 manifest
11920 }
11921
11922 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
11923 Frame::build(
11924 FrameType::Hello,
11925 control_flags(),
11926 0,
11927 0,
11928 corr,
11929 serde_json::to_vec(&ModuleHelloBody {
11930 protocol_ver: manifest.protocol_ver,
11931 manifest,
11932 control_ops: None,
11933 launch_nonce: None,
11934 })
11935 .expect("capability test HELLO serializes"),
11936 )
11937 .expect("capability test HELLO frame builds")
11938 }
11939
11940 fn catalog_update_with_capabilities_frame(
11941 corr: u64,
11942 capabilities: CapabilityDeclarations,
11943 ) -> Frame {
11944 Frame::build(
11945 FrameType::Request,
11946 control_flags(),
11947 0,
11948 0,
11949 corr,
11950 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11951 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
11952 capabilities: Some(capabilities),
11953 ready: None,
11954 })
11955 .expect("capability catalog.update serializes"),
11956 )
11957 .expect("capability catalog.update frame builds")
11958 }
11959
11960 async fn register_capability_manifest(
11961 handler: &ControlHandler,
11962 ctx: &RouteCtx,
11963 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11964 manifest: ModuleManifest,
11965 corr: u64,
11966 ) {
11967 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
11968 }
11969
11970 async fn open_route_for_capability_test(
11971 handler: &ControlHandler,
11972 target_ctx: &RouteCtx,
11973 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11974 client_connection_id: u64,
11975 corr: u64,
11976 target_module_id: &str,
11977 consumer_identity: Option<ConsumerIdentity>,
11978 ) -> (
11979 mpsc::Receiver<crate::router::OutboundFrame>,
11980 ModuleControlRequest,
11981 ) {
11982 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
11983 let route_handler = handler.clone();
11984 let target_module_id = target_module_id.to_string();
11985 let route_task = tokio::spawn(async move {
11986 route_handler
11987 .handle_control_frame(
11988 &client_ctx,
11989 route_open_frame_with_admission_facts(
11990 corr,
11991 &target_module_id,
11992 unique_project_root("admission-facts"),
11993 consumer_identity,
11994 None,
11995 ),
11996 )
11997 .await
11998 .expect("capability test route.open succeeds")
11999 });
12000 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
12001 .await
12002 .expect("capability test route.open must reach route.bind")
12003 .expect("target control receiver stays open");
12004 let bind_request: ModuleControlRequest =
12005 serde_json::from_slice(&bind.body).expect("route.bind decodes");
12006 handler
12007 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
12008 .await
12009 .expect("capability test route.bind ACK succeeds");
12010 assert!(route_task.await.expect("route.open task joins").is_empty());
12011 let opened = client_rx
12012 .recv()
12013 .await
12014 .expect("successful route.open publishes a response");
12015 assert!(matches!(
12016 serde_json::from_slice::<ClientControlResponse>(&opened.body),
12017 Ok(ClientControlResponse::RouteOpen { .. })
12018 ));
12019 (client_rx, bind_request)
12020 }
12021
12022 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
12023 assert_eq!(frame.header.ty, FrameType::Push);
12024 assert_eq!(frame.header.channel, 0);
12025 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
12026 .expect("route.closed control push decodes");
12027 let ClientControlPush::RouteClosed { channels, .. } = &push else {
12028 panic!("expected route.closed");
12029 };
12030 assert_eq!(channels.len(), 1, "exactly one violating route closed");
12031 let channels = channels.clone();
12032 assert_eq!(
12033 push,
12034 ClientControlPush::RouteClosed {
12035 module_id: target_module_id.to_string(),
12036 channels,
12037 reason: RouteCloseReason::CapabilityDenied,
12038 drained: false,
12039 abandoned: 0,
12040 excluded_subscriptions: 0,
12041 terminal: Some(false),
12042 }
12043 );
12044 }
12045
12046 #[tokio::test]
12047 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
12048 let registry = Arc::new(Registry::default());
12049 let forwarding = Arc::new(ForwardingTable::default());
12050 let supervisor = SupervisorHandle::new();
12051 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12052 let handler =
12053 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12054 .with_supervisor(supervisor);
12055 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
12056 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
12057 register_capability_manifest(
12058 &handler,
12059 &target_ctx,
12060 &mut target_rx,
12061 capability_manifest("target", &["credentials-provider/v1"], &[]),
12062 1,
12063 )
12064 .await;
12065 register_capability_manifest(
12066 &handler,
12067 &opener_ctx,
12068 &mut opener_rx,
12069 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12070 2,
12071 )
12072 .await;
12073
12074 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
12075 let replies = handler
12076 .handle_control_frame(
12077 &client_ctx,
12078 route_open_frame_with_admission_facts(
12079 3,
12080 "target",
12081 unique_project_root("admission-facts"),
12082 Some(ConsumerIdentity {
12083 module_id: "opener".to_string(),
12084 launch_nonce: "opener-nonce".to_string(),
12085 }),
12086 None,
12087 ),
12088 )
12089 .await
12090 .expect("denied route.open returns a typed frame");
12091 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
12092 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12093 assert!(
12094 target_rx.try_recv().is_err(),
12095 "forbidden route.open must not relay route.bind"
12096 );
12097 }
12098
12099 #[tokio::test]
12100 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
12101 let registry = Arc::new(Registry::default());
12102 let forwarding = Arc::new(ForwardingTable::default());
12103 let supervisor = SupervisorHandle::new();
12104 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12105 let handler =
12106 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12107 .with_supervisor(supervisor);
12108 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
12109 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
12110 register_capability_manifest(
12111 &handler,
12112 &target_ctx,
12113 &mut target_rx,
12114 capability_manifest("target", &["credentials-provider/v1"], &[]),
12115 1,
12116 )
12117 .await;
12118 register_capability_manifest(
12119 &handler,
12120 &old_opener_ctx,
12121 &mut old_opener_rx,
12122 capability_manifest("opener", &[], &[]),
12123 2,
12124 )
12125 .await;
12126 let (mut client_rx, _) = open_route_for_capability_test(
12127 &handler,
12128 &target_ctx,
12129 &mut target_rx,
12130 712,
12131 3,
12132 "target",
12133 Some(ConsumerIdentity {
12134 module_id: "opener".to_string(),
12135 launch_nonce: "opener-nonce".to_string(),
12136 }),
12137 )
12138 .await;
12139 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12140
12141 handler
12142 .cleanup_connection(old_opener_ctx.connection_id)
12143 .expect("old opener registration cleans up");
12144 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
12145 register_capability_manifest(
12146 &handler,
12147 &new_opener_ctx,
12148 &mut new_opener_rx,
12149 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12150 4,
12151 )
12152 .await;
12153
12154 assert_capability_denied_push(
12155 client_rx
12156 .try_recv()
12157 .expect("HELLO deny addition must emit route.closed")
12158 .frame,
12159 "target",
12160 );
12161 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12162 assert!(matches!(
12163 target_rx.try_recv(),
12164 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
12165 ));
12166 }
12167
12168 #[tokio::test]
12169 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
12170 let registry = Arc::new(Registry::default());
12171 let forwarding = Arc::new(ForwardingTable::default());
12172 let supervisor = SupervisorHandle::new();
12173 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12174 let handler =
12175 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12176 .with_supervisor(supervisor);
12177 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
12178 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
12179 register_capability_manifest(
12180 &handler,
12181 &target_ctx,
12182 &mut target_rx,
12183 capability_manifest("target", &[], &[]),
12184 1,
12185 )
12186 .await;
12187 register_capability_manifest(
12188 &handler,
12189 &opener_ctx,
12190 &mut opener_rx,
12191 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12192 2,
12193 )
12194 .await;
12195 let (mut client_rx, _) = open_route_for_capability_test(
12196 &handler,
12197 &target_ctx,
12198 &mut target_rx,
12199 722,
12200 3,
12201 "target",
12202 Some(ConsumerIdentity {
12203 module_id: "opener".to_string(),
12204 launch_nonce: "opener-nonce".to_string(),
12205 }),
12206 )
12207 .await;
12208 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12209
12210 let replies = handler
12211 .handle_control_frame(
12212 &target_ctx,
12213 catalog_update_with_capabilities_frame(
12214 4,
12215 CapabilityDeclarations {
12216 provides: vec!["credentials-provider/v1".to_string()],
12217 requires: Vec::new(),
12218 must_never_reach: Vec::new(),
12219 },
12220 ),
12221 )
12222 .await
12223 .expect("claim catalog.update succeeds");
12224 assert!(matches!(
12225 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
12226 Ok(ModuleControlResponseToModule::CatalogUpdate {})
12227 ));
12228 assert_capability_denied_push(
12229 client_rx
12230 .try_recv()
12231 .expect("claim addition must emit route.closed")
12232 .frame,
12233 "target",
12234 );
12235 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12236 assert!(matches!(
12237 target_rx.try_recv(),
12238 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
12239 ));
12240 }
12241
12242 #[tokio::test]
12243 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
12244 let registry = Arc::new(Registry::default());
12245 let forwarding = Arc::new(ForwardingTable::default());
12246 let supervisor = SupervisorHandle::new();
12247 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12248 let handler =
12249 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12250 .with_supervisor(supervisor);
12251 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
12252 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
12253 register_capability_manifest(
12254 &handler,
12255 &target_ctx,
12256 &mut target_rx,
12257 capability_manifest("target", &["credentials-provider/v1"], &[]),
12258 1,
12259 )
12260 .await;
12261 register_capability_manifest(
12262 &handler,
12263 &opener_ctx,
12264 &mut opener_rx,
12265 capability_manifest("opener", &[], &[]),
12266 2,
12267 )
12268 .await;
12269 let (mut client_rx, _) = open_route_for_capability_test(
12270 &handler,
12271 &target_ctx,
12272 &mut target_rx,
12273 732,
12274 3,
12275 "target",
12276 Some(ConsumerIdentity {
12277 module_id: "opener".to_string(),
12278 launch_nonce: "opener-nonce".to_string(),
12279 }),
12280 )
12281 .await;
12282 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12283
12284 handler
12285 .handle_control_frame(
12286 &target_ctx,
12287 catalog_update_with_capabilities_frame(
12288 4,
12289 CapabilityDeclarations {
12290 provides: Vec::new(),
12291 requires: Vec::new(),
12292 must_never_reach: Vec::new(),
12293 },
12294 ),
12295 )
12296 .await
12297 .expect("claim removal catalog.update succeeds");
12298 assert_eq!(
12299 forwarding.active_binding_count().unwrap(),
12300 1,
12301 "removing an attested target claim must leave the route census unchanged"
12302 );
12303 assert!(
12304 client_rx.try_recv().is_err(),
12305 "claim removal must not emit route.closed capability_denied"
12306 );
12307 assert!(
12308 target_rx.try_recv().is_err(),
12309 "claim removal must not send the target a route GOODBYE"
12310 );
12311 }
12312
12313 #[tokio::test]
12316 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
12317 let registry = Arc::new(Registry::default());
12318 let forwarding = Arc::new(ForwardingTable::default());
12319 let supervisor = SupervisorHandle::new();
12320 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12321 let handler =
12322 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12323 .with_supervisor(supervisor);
12324 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
12325 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
12326 register_capability_manifest(
12327 &handler,
12328 &target_ctx,
12329 &mut target_rx,
12330 capability_manifest("target", &["credentials-provider/v1"], &[]),
12331 1,
12332 )
12333 .await;
12334 register_capability_manifest(
12335 &handler,
12336 &opener_ctx,
12337 &mut opener_rx,
12338 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12339 2,
12340 )
12341 .await;
12342
12343 let (_client_rx, bind) = open_route_for_capability_test(
12344 &handler,
12345 &target_ctx,
12346 &mut target_rx,
12347 742,
12348 3,
12349 "target",
12350 None,
12351 )
12352 .await;
12353 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12354 panic!("direct scope-honesty route must bind");
12355 };
12356 assert_eq!(principal, Some(Principal::Direct));
12357 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12358 }
12359
12360 #[tokio::test]
12363 async fn must_never_reach_self_route_is_capability_forbidden() {
12364 let registry = Arc::new(Registry::default());
12365 let forwarding = Arc::new(ForwardingTable::default());
12366 let supervisor = SupervisorHandle::new();
12367 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
12368 let handler =
12369 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12370 .with_supervisor(supervisor);
12371 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
12372 register_capability_manifest(
12373 &handler,
12374 &self_ctx,
12375 &mut self_rx,
12376 capability_manifest(
12377 "self-provider",
12378 &["credentials-provider/v1"],
12379 &["credentials-provider/v1"],
12380 ),
12381 1,
12382 )
12383 .await;
12384
12385 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
12386 let replies = handler
12387 .handle_control_frame(
12388 &client_ctx,
12389 route_open_frame_with_admission_facts(
12390 2,
12391 "self-provider",
12392 unique_project_root("admission-facts"),
12393 Some(ConsumerIdentity {
12394 module_id: "self-provider".to_string(),
12395 launch_nonce: "self-nonce".to_string(),
12396 }),
12397 None,
12398 ),
12399 )
12400 .await
12401 .expect("self-route refusal returns a typed frame");
12402 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
12403 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12404 assert!(
12405 self_rx.try_recv().is_err(),
12406 "self denial must not relay route.bind"
12407 );
12408 }
12409
12410 #[test]
12411 fn unsupported_channel_zero_frame_returns_error() {
12412 let handler = ControlHandler::default();
12413 let request = Frame::build(
12414 FrameType::Request,
12415 control_flags(),
12416 0,
12417 0,
12418 21,
12419 b"opaque".to_vec(),
12420 )
12421 .unwrap();
12422
12423 let response = handler
12424 .handle_control(ConnectionId::new(1), request)
12425 .unwrap();
12426
12427 assert_eq!(response[0].header.ty, FrameType::Error);
12428 assert_eq!(
12429 parse_error(&response[0])["code"],
12430 "unsupported_control_frame"
12431 );
12432 }
12433
12434 mod swap {
12439 use super::*;
12440
12441 const INCUMBENT: ConnectionId = ConnectionId::new(30);
12442 const CANDIDATE: ConnectionId = ConnectionId::new(40);
12443
12444 struct Swap {
12445 registry: Arc<Registry>,
12446 forwarding: Arc<ForwardingTable>,
12447 handler: ControlHandler,
12448 incumbent_ctx: RouteCtx,
12449 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12450 candidate_ctx: RouteCtx,
12451 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12452 }
12453
12454 async fn swap_with_incumbent() -> Swap {
12455 let registry = Arc::new(Registry::default());
12456 let forwarding = Arc::new(ForwardingTable::default());
12457 let handler =
12458 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12459 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
12460 hello_via_sink(
12461 &handler,
12462 &incumbent_ctx,
12463 &mut incumbent_rx,
12464 hello_frame("aft", PROTOCOL_VERSION, 7),
12465 )
12466 .await;
12467 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
12468 Swap {
12469 registry,
12470 forwarding,
12471 handler,
12472 incumbent_ctx,
12473 incumbent_rx,
12474 candidate_ctx,
12475 candidate_rx,
12476 }
12477 }
12478
12479 fn register_candidate(swap: &Swap, ready: Option<bool>) {
12480 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
12481 candidate_manifest.ready = ready;
12482 let registration = swap
12483 .registry
12484 .register_candidate_with_control_ops(
12485 candidate_manifest,
12486 PROTOCOL_VERSION,
12487 CANDIDATE,
12488 module_baseline_control_ops(),
12489 )
12490 .unwrap();
12491 swap.forwarding
12492 .register_candidate_module_connection(
12493 CANDIDATE,
12494 "aft".to_string(),
12495 PROTOCOL_VERSION,
12496 manifest_concurrency(®istration.manifest),
12497 swap.candidate_ctx.egress.clone(),
12498 )
12499 .unwrap();
12500 }
12501
12502 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
12503 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
12504 swap.registry.promote_candidate("aft").unwrap().unwrap();
12505 cutover.incumbent.unwrap()
12506 }
12507
12508 fn keyed_total(counters: &Value, key: &str) -> u64 {
12509 counters[key]
12510 .as_object()
12511 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
12512 .unwrap_or(0)
12513 }
12514
12515 #[tokio::test]
12516 async fn replacement_logs_old_end_and_new_admission_once() {
12517 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12518 let mut swap = swap_with_incumbent().await;
12519 swap.handler
12520 .supervisor
12521 .open_swap("aft", "candidate-nonce".to_string());
12522 hello_via_sink(
12523 &swap.handler,
12524 &swap.candidate_ctx,
12525 &mut swap.candidate_rx,
12526 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
12527 )
12528 .await;
12529 cutover(&swap);
12530 swap.handler.cleanup_connection(INCUMBENT).unwrap();
12531
12532 let captured = crate::router::test_log::captured_logs(&logs);
12533 println!("captured replacement registry logs:\n{captured}");
12534 let new_admissions: Vec<_> = captured
12535 .lines()
12536 .filter(|line| {
12537 line.contains("connection_id=40")
12538 && line.contains("swap candidate registered; not routable until cutover")
12539 })
12540 .collect();
12541 assert_eq!(
12542 new_admissions.len(),
12543 1,
12544 "unexpected admission logs: {captured}"
12545 );
12546 assert!(new_admissions[0].contains("module_id=aft"));
12547 assert!(new_admissions[0].contains("connection_id=40"));
12548 assert!(
12549 new_admissions[0].contains("swap candidate registered; not routable until cutover")
12550 );
12551 assert!(new_admissions[0].contains("ready=true"));
12552
12553 let old_admissions: Vec<_> = captured
12554 .lines()
12555 .filter(|line| {
12556 line.contains("module registered module_id=aft ")
12557 && line.contains("connection_id=30")
12558 })
12559 .collect();
12560 assert_eq!(
12561 old_admissions.len(),
12562 1,
12563 "unexpected incumbent admission: {captured}"
12564 );
12565
12566 let old_ends: Vec<_> = captured
12567 .lines()
12568 .filter(|line| {
12569 line.contains("connection_id=30") && line.contains("module registration ended")
12570 })
12571 .collect();
12572 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
12573 assert!(old_ends[0].contains("module_id=aft"));
12574 assert!(old_ends[0].contains("connection_id=30"));
12575 assert!(old_ends[0].contains("reason=replaced"));
12576 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
12577 assert_eq!(
12578 captured
12579 .lines()
12580 .filter(|line| line.contains("module registration promoted"))
12581 .count(),
12582 1,
12583 "unexpected promotion logs: {captured}"
12584 );
12585 }
12586
12587 #[tokio::test]
12593 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
12594 let mut swap = swap_with_incumbent().await;
12595 let handler = swap.handler.clone();
12596
12597 let cotenant = ConnectionId::new(31);
12599 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
12600 let (cotenant_task, cotenant_bind) = relay_route_open(
12601 &handler,
12602 cotenant,
12603 &cotenant_ctx.egress,
12604 &mut swap.incumbent_rx,
12605 100,
12606 "aft",
12607 "swap-cotenant",
12608 )
12609 .await;
12610 handler
12611 .handle_control_frame(
12612 &swap.incumbent_ctx,
12613 route_bind_ack(cotenant_bind.header.corr),
12614 )
12615 .await
12616 .unwrap();
12617 assert!(cotenant_task.await.unwrap().is_empty());
12618 let (cotenant_channel, cotenant_epoch) =
12619 published_route(&cotenant_rx.recv().await.unwrap());
12620
12621 let caller = ConnectionId::new(32);
12623 let (caller_ctx, mut caller_rx) = route_ctx(caller);
12624 let (caller_task, caller_bind) = relay_route_open(
12625 &handler,
12626 caller,
12627 &caller_ctx.egress,
12628 &mut swap.incumbent_rx,
12629 101,
12630 "aft",
12631 "swap-caller",
12632 )
12633 .await;
12634 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
12635
12636 register_candidate(&swap, None);
12637 cutover(&swap);
12638
12639 let ack = handler
12641 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
12642 .await;
12643 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
12644 if module_loop_error.is_some() {
12645 handler.cleanup_connection(INCUMBENT).unwrap();
12648 }
12649
12650 assert!(
12652 cotenant_rx.try_recv().is_err(),
12653 "the co-tenant route on the incumbent was torn down by one late ack: \
12654 {module_loop_error:?}"
12655 );
12656 assert!(matches!(
12657 swap.forwarding
12658 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
12659 .unwrap(),
12660 DataRoute::Client(DataRouteState::Bound(_))
12661 ));
12662 assert_eq!(module_loop_error, None);
12663 assert!(swap
12664 .registry
12665 .get_module_by_connection(INCUMBENT)
12666 .unwrap()
12667 .is_some());
12668
12669 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
12671 .await
12672 .expect("the incumbent is told to drop the abandoned binding")
12673 .unwrap()
12674 .frame;
12675 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12676 assert_eq!(goodbye.header.channel, abandoned_channel);
12677 assert_eq!(goodbye.header.epoch, abandoned_epoch);
12678 assert!(swap.incumbent_rx.try_recv().is_err());
12679
12680 let response = caller_task.await.unwrap();
12682 assert_eq!(response.len(), 1);
12683 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
12684 assert!(caller_rx.try_recv().is_err());
12685
12686 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
12690 let counters = handler.counters().snapshot();
12691 assert_eq!(
12692 keyed_total(&counters, "route_open_accepted_by_principal"),
12693 1
12694 );
12695 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
12696 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
12697 }
12698
12699 #[tokio::test]
12703 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
12704 let mut swap = swap_with_incumbent().await;
12705 register_candidate(&swap, None);
12706 let incumbent = cutover(&swap);
12707 swap.forwarding
12708 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
12709 .unwrap()
12710 .expect("the incumbent is still registered");
12711
12712 let client = ConnectionId::new(33);
12713 let (client_ctx, mut client_rx) = route_ctx(client);
12714 let route_handler = swap.handler.clone();
12715 let open_ctx = RouteCtx {
12716 connection_id: client,
12717 egress: client_ctx.egress.clone(),
12718 };
12719 let mut route_task = tokio::spawn(async move {
12720 route_handler
12721 .handle_control_frame(
12722 &open_ctx,
12723 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
12724 )
12725 .await
12726 .unwrap()
12727 });
12728 let bind = tokio::select! {
12729 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
12730 response = &mut route_task => {
12731 let response = response.unwrap();
12732 panic!(
12733 "post-cutover route.open was refused instead of relayed to the candidate: {}",
12734 parse_error(&response[0])["code"]
12735 );
12736 }
12737 };
12738 swap.handler
12739 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
12740 .await
12741 .unwrap();
12742 assert!(route_task.await.unwrap().is_empty());
12743 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
12744 match swap
12745 .forwarding
12746 .lookup_data_route(client, channel, epoch)
12747 .unwrap()
12748 {
12749 DataRoute::Client(DataRouteState::Bound(route)) => {
12750 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
12751 }
12752 other => panic!("expected a bound route on the candidate, got {other:?}"),
12753 }
12754 assert!(swap.incumbent_rx.try_recv().is_err());
12755 }
12756
12757 #[tokio::test]
12762 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
12763 let swap = swap_with_incumbent().await;
12764 register_candidate(&swap, Some(false));
12765 let update = Frame::build(
12766 FrameType::Request,
12767 control_flags(),
12768 0,
12769 0,
12770 55,
12771 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
12772 provides: manifest("aft", PROTOCOL_VERSION).provides,
12773 capabilities: None,
12774 ready: Some(true),
12775 })
12776 .unwrap(),
12777 )
12778 .unwrap();
12779
12780 let replies = swap
12781 .handler
12782 .handle_control_frame(&swap.candidate_ctx, update)
12783 .await
12784 .unwrap();
12785
12786 assert_eq!(replies.len(), 1);
12787 assert_eq!(
12788 replies[0].header.ty,
12789 FrameType::Response,
12790 "candidate catalog.update was refused: {:?}",
12791 serde_json::from_slice::<Value>(&replies[0].body).ok()
12792 );
12793 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
12794 assert_eq!(
12795 swap.registry
12796 .get_module("aft")
12797 .unwrap()
12798 .unwrap()
12799 .connection_id,
12800 INCUMBENT
12801 );
12802 }
12803 }
12804
12805 mod swap_admission {
12809 use super::*;
12810
12811 const INCUMBENT_NONCE: &str = "incumbent-nonce";
12812 const CANDIDATE_NONCE: &str = "candidate-nonce";
12813
12814 fn handler_with_incumbent(
12815 module_id: &str,
12816 reserved: bool,
12817 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
12818 let registry = Arc::new(Registry::default());
12819 let supervisor = SupervisorHandle::new();
12820 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
12821 if reserved {
12822 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
12823 }
12824 let handler =
12825 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
12826 let incumbent = handler
12827 .handle_control(
12828 ConnectionId::new(1),
12829 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
12830 )
12831 .unwrap();
12832 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
12833 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
12834 (registry, supervisor, handler)
12835 }
12836
12837 #[test]
12843 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
12844 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
12845
12846 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
12847 let replies = handler
12848 .handle_control(
12849 ConnectionId::new(connection),
12850 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
12851 )
12852 .unwrap();
12853 assert_eq!(replies[0].header.ty, FrameType::Error);
12854 assert_eq!(
12855 parse_error(&replies[0])["code"],
12856 "swap_token_invalid",
12857 "nonce {nonce:?}"
12858 );
12859 }
12860 assert!(registry.get_candidate("aft").unwrap().is_none());
12861 assert_eq!(
12862 registry.get_module("aft").unwrap().unwrap().connection_id,
12863 ConnectionId::new(1)
12864 );
12865
12866 let admitted = handler
12869 .handle_control(
12870 ConnectionId::new(4),
12871 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
12872 )
12873 .unwrap();
12874 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
12875 assert_eq!(
12876 registry
12877 .get_candidate("aft")
12878 .unwrap()
12879 .unwrap()
12880 .connection_id,
12881 ConnectionId::new(4)
12882 );
12883 assert_eq!(
12884 registry.get_module("aft").unwrap().unwrap().connection_id,
12885 ConnectionId::new(1),
12886 "the candidate must not take the active slot"
12887 );
12888 let replayed = handler
12889 .handle_control(
12890 ConnectionId::new(5),
12891 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
12892 )
12893 .unwrap();
12894 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
12895
12896 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
12900 let squatter = handler
12901 .handle_control(
12902 ConnectionId::new(6),
12903 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
12904 )
12905 .unwrap();
12906 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
12907 assert!(
12908 registry.get_module("aft").unwrap().is_none(),
12909 "a squatter took the active slot of an id being swapped"
12910 );
12911 }
12912
12913 #[test]
12918 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
12919 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
12920
12921 let replies = handler
12922 .handle_control(
12923 ConnectionId::new(2),
12924 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12925 )
12926 .unwrap();
12927
12928 assert_eq!(
12929 replies[0].header.ty,
12930 FrameType::HelloAck,
12931 "reserved candidate refused: {:?}",
12932 serde_json::from_slice::<Value>(&replies[0].body).ok()
12933 );
12934 assert_eq!(
12935 registry
12936 .get_candidate("vault")
12937 .unwrap()
12938 .unwrap()
12939 .connection_id,
12940 ConnectionId::new(2)
12941 );
12942 }
12943
12944 #[test]
12947 fn without_an_open_swap_the_ordinary_gates_decide() {
12948 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
12949 supervisor.close_swap("vault");
12950
12951 let candidate = handler
12952 .handle_control(
12953 ConnectionId::new(2),
12954 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12955 )
12956 .unwrap();
12957 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
12958 let duplicate = handler
12959 .handle_control(
12960 ConnectionId::new(3),
12961 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
12962 )
12963 .unwrap();
12964 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12965 assert!(registry.get_candidate("vault").unwrap().is_none());
12966 }
12967 }
12968
12969 mod scopes {
12973 mod delta {
12974 include!("scope_handler_delta_tests.rs");
12975 }
12976
12977 use subc_protocol::scope::{
12978 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
12979 ScopeStatus,
12980 };
12981
12982 use super::*;
12983
12984 const OWNER: &str = "prefrontal-core";
12985
12986 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
12987 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
12988 }
12989
12990 async fn call(
12991 handler: &ControlHandler,
12992 ctx: &RouteCtx,
12993 request: &ModuleControlRequestFromModule,
12994 ) -> Frame {
12995 let body = serde_json::to_vec(request).unwrap();
12996 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
12997 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
12998 assert_eq!(replies.len(), 1, "{replies:?}");
12999 replies.pop().unwrap()
13000 }
13001
13002 async fn sync(
13003 handler: &ControlHandler,
13004 ctx: &RouteCtx,
13005 generation: u64,
13006 scopes: Vec<ScopeRecord>,
13007 ) -> Result<ModuleControlResponseToModule, String> {
13008 let reply = call(
13009 handler,
13010 ctx,
13011 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
13012 )
13013 .await;
13014 match reply.header.ty {
13015 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
13016 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
13017 }
13018 }
13019
13020 async fn describe(
13021 handler: &ControlHandler,
13022 ctx: &RouteCtx,
13023 owner: &str,
13024 scope_ref: &str,
13025 ) -> ModuleControlResponseToModule {
13026 let reply = call(
13027 handler,
13028 ctx,
13029 &ModuleControlRequestFromModule::ScopeDescribe {
13030 owner: Principal::Reserved {
13031 module_id: owner.to_string(),
13032 },
13033 scope_ref: scope_ref.to_string(),
13034 },
13035 )
13036 .await;
13037 assert_eq!(
13038 reply.header.ty,
13039 FrameType::Response,
13040 "{:?}",
13041 parse_error(&reply)
13042 );
13043 serde_json::from_slice(&reply.body).unwrap()
13044 }
13045
13046 async fn module(
13048 handler: &ControlHandler,
13049 connection: u64,
13050 module_id: &str,
13051 nonce: Option<&str>,
13052 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
13053 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
13054 hello_via_sink(
13055 handler,
13056 &ctx,
13057 &mut rx,
13058 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
13059 )
13060 .await;
13061 (ctx, rx)
13062 }
13063
13064 #[tokio::test]
13067 async fn a_client_connection_cannot_sync_or_describe() {
13068 let handler = ControlHandler::new(Arc::new(Registry::default()));
13069 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
13070 for request in [
13071 ModuleControlRequestFromModule::ScopeSync {
13072 generation: 1,
13073 scopes: vec![head("s", 1)],
13074 },
13075 ModuleControlRequestFromModule::ScopeDescribe {
13076 owner: Principal::Direct,
13077 scope_ref: "s".to_string(),
13078 },
13079 ] {
13080 let reply = call(&handler, &ctx, &request).await;
13081 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
13082 }
13083 assert!(
13084 !handler
13085 .scopes
13086 .read()
13087 .unwrap()
13088 .describe(
13089 &Principal::Reserved {
13090 module_id: OWNER.to_string()
13091 },
13092 "s"
13093 )
13094 .owner_synced
13095 );
13096 }
13097
13098 #[tokio::test]
13101 async fn a_module_without_a_supervised_launch_cannot_sync() {
13102 let handler = ControlHandler::new(Arc::new(Registry::default()));
13103 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
13104 assert_eq!(
13105 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
13106 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13107 );
13108 }
13109
13110 #[tokio::test]
13111 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
13112 let supervisor = SupervisorHandle::new();
13113 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13114 let handler = ControlHandler::new(Arc::new(Registry::default()))
13115 .with_supervisor(supervisor.clone());
13116 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
13117 sync(&handler, &incumbent, 1, vec![head("s", 1)])
13118 .await
13119 .expect("the current launch syncs");
13120
13121 supervisor.open_swap(OWNER, "n2".to_string());
13124 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
13125 assert_eq!(
13126 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
13127 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13128 );
13129 sync(&handler, &incumbent, 2, vec![head("s", 1)])
13130 .await
13131 .expect("the serving owner syncs during the swap");
13132
13133 supervisor.close_swap(OWNER);
13136 assert_eq!(
13137 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
13138 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13139 );
13140 sync(&handler, &incumbent, 3, vec![head("s", 1)])
13141 .await
13142 .expect("the serving owner syncs after the rollback");
13143 handler.cleanup_connection(candidate.connection_id).unwrap();
13144
13145 supervisor.open_swap(OWNER, "n3".to_string());
13150 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
13151 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
13152 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
13153 .await
13154 .expect("the promoted launch takes authority");
13155 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
13156 panic!("unexpected reply {reply:?}");
13157 };
13158 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
13159 assert_eq!(
13160 sync(&handler, &incumbent, 4, Vec::new()).await,
13161 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13162 );
13163 }
13164
13165 #[tokio::test]
13168 async fn closing_the_authority_connection_frees_sync_authority() {
13169 let supervisor = SupervisorHandle::new();
13170 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13171 let handler = ControlHandler::new(Arc::new(Registry::default()))
13172 .with_supervisor(supervisor.clone());
13173 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
13174 sync(&handler, &first, 10, vec![head("s", 1)])
13175 .await
13176 .unwrap();
13177 handler.cleanup_connection(first.connection_id).unwrap();
13178
13179 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
13180 sync(&handler, &second, 1, vec![head("s", 1)])
13181 .await
13182 .expect("the next connection takes the released authority");
13183 }
13184
13185 #[tokio::test]
13186 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
13187 let supervisor = SupervisorHandle::new();
13188 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13189 let handler =
13190 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
13191 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
13192 sync(&handler, &first, 10, vec![head("s", 1)])
13193 .await
13194 .unwrap();
13195 handler
13196 .handle_control_frame(
13197 &first,
13198 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
13199 )
13200 .await
13201 .unwrap();
13202 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
13203 sync(&handler, &second, 1, vec![head("s", 1)])
13204 .await
13205 .expect("GOODBYE releases authority even if the old socket remains open");
13206 }
13207
13208 #[tokio::test]
13209 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
13210 let registry = Arc::new(Registry::default());
13211 let supervisor_handle = SupervisorHandle::new();
13212 let supervisor =
13213 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
13214 .with_handle(supervisor_handle.clone())
13215 .with_daemon_incarnation("incarnation-7".to_string());
13216 supervisor
13219 .supervise_configured(
13220 ModuleSpec {
13221 module_id: OWNER.to_string(),
13222 program: PathBuf::from("/nonexistent/prefrontal-core"),
13223 args: Vec::new(),
13224 env: Vec::new(),
13225 reserved: false,
13226 reserved_prefixes: Vec::new(),
13227 protocol: ModuleProtocol::Subc,
13228 overlap: Default::default(),
13229 },
13230 false,
13231 )
13232 .unwrap();
13233 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
13234 let handler =
13235 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13236 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
13237
13238 let ModuleControlResponseToModule::ScopeDescribe {
13240 status,
13241 daemon_incarnation,
13242 owner_synced,
13243 owner_configured,
13244 scope,
13245 ..
13246 } = describe(&handler, &reader, OWNER, "s").await
13247 else {
13248 panic!("not a describe reply");
13249 };
13250 assert_eq!(status, ScopeStatus::NotLive);
13251 assert_eq!(daemon_incarnation, "incarnation-7");
13252 assert!(!owner_synced);
13253 assert!(owner_configured);
13254 assert!(scope.is_none());
13255
13256 let ModuleControlResponseToModule::ScopeDescribe {
13259 status,
13260 owner_configured,
13261 ..
13262 } = describe(&handler, &reader, "ghost", "s").await
13263 else {
13264 panic!("not a describe reply");
13265 };
13266 assert_eq!(status, ScopeStatus::NotLive);
13267 assert!(!owner_configured);
13268
13269 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
13271 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
13272 let ModuleControlResponseToModule::ScopeDescribe {
13273 status,
13274 scope_epoch,
13275 owner_synced,
13276 scope,
13277 ..
13278 } = describe(&handler, &reader, OWNER, "s").await
13279 else {
13280 panic!("not a describe reply");
13281 };
13282 assert_eq!(status, ScopeStatus::Live);
13283 assert_eq!(scope_epoch, Some(4));
13284 assert!(owner_synced);
13285 let stamp = scope.expect("a live scope carries its stamp");
13286 assert!(
13287 stamp.owner_authorized,
13288 "prefrontal-core is the default authority"
13289 );
13290 assert_eq!(stamp.kind, ScopeKind::Head);
13291 }
13292
13293 #[tokio::test]
13294 async fn scope_authority_owners_decides_owner_authorized() {
13295 let supervisor = SupervisorHandle::new();
13296 supervisor.set_spawn_nonce("broca", "b1".to_string());
13297 let handler = ControlHandler::new(Arc::new(Registry::default()))
13298 .with_supervisor(supervisor)
13299 .with_scope_authority_owners(vec!["broca".to_string()]);
13300 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
13301 let mut gated = head("s", 1);
13302 gated.attributes.agent_id = Some("agent".to_string());
13303 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
13304 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
13305 describe(&handler, &broca, "broca", "s").await
13306 else {
13307 panic!("not a describe reply");
13308 };
13309 assert!(scope.unwrap().owner_authorized);
13310 }
13311
13312 #[tokio::test]
13316 async fn scope_ops_and_the_scopes_capability_are_advertised() {
13317 let handler = ControlHandler::new(Arc::new(Registry::default()));
13318 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
13319 let ack = hello_via_sink(
13320 &handler,
13321 &ctx,
13322 &mut rx,
13323 hello_frame("m", PROTOCOL_VERSION, 1),
13324 )
13325 .await;
13326 let ack = parse_ack(&ack);
13327 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
13328 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
13329 }
13330 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
13331
13332 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
13333 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13334 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13335 let reply = handler
13336 .handle_control_frame(&client, frame)
13337 .await
13338 .unwrap()
13339 .pop()
13340 .unwrap();
13341 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13342 serde_json::from_slice(&reply.body).unwrap()
13343 else {
13344 panic!("not a server.describe reply");
13345 };
13346 assert!(
13347 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
13348 "{capabilities:?}"
13349 );
13350 }
13351
13352 const PLEXUS: &str = "plexus";
13355 const OTHER: &str = "other";
13356 const AFT: &str = "aft";
13357 const BROCA: &str = "broca";
13358 const MAGIC: &str = "magic-context";
13359
13360 fn nonce(module_id: &str) -> String {
13361 format!("nonce-{module_id}")
13362 }
13363
13364 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
13365 let (tx, rx) = mpsc::channel(64);
13366 (
13367 RouteCtx {
13368 connection_id: ConnectionId::new(connection),
13369 egress: FrameSink::new(tx),
13370 },
13371 rx,
13372 )
13373 }
13374
13375 struct Rig {
13379 handler: ControlHandler,
13380 forwarding: Arc<ForwardingTable>,
13381 owner: RouteCtx,
13382 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13383 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
13384 generation: u64,
13385 next_connection: u64,
13386 _supervisor: Supervisor,
13387 }
13388
13389 async fn rig() -> Rig {
13390 rig_with_flow_support(true).await
13391 }
13392
13393 async fn rig_with_flow_support(flow_support: bool) -> Rig {
13394 rig_with_scope_capabilities(flow_support, false).await
13395 }
13396
13397 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
13398 let registry = Arc::new(Registry::default());
13399 let forwarding = Arc::new(ForwardingTable::default());
13400 let supervisor_handle = SupervisorHandle::new();
13401 let supervisor =
13402 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
13403 .with_handle(supervisor_handle.clone());
13404 supervisor
13405 .supervise_configured(
13406 ModuleSpec {
13407 module_id: OWNER.to_string(),
13408 program: PathBuf::from("/nonexistent/prefrontal-core"),
13409 args: Vec::new(),
13410 env: Vec::new(),
13411 reserved: false,
13412 reserved_prefixes: Vec::new(),
13413 protocol: ModuleProtocol::Subc,
13414 overlap: Default::default(),
13415 },
13416 false,
13417 )
13418 .unwrap();
13419 for module_id in [OWNER, AFT, BROCA, MAGIC] {
13420 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
13421 }
13422 let handler =
13423 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13424 .with_supervisor(supervisor_handle);
13425 let (owner, mut owner_rx) = wide_ctx(1);
13426 hello_via_sink(
13427 &handler,
13428 &owner,
13429 &mut owner_rx,
13430 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
13431 )
13432 .await;
13433 let mut modules = BTreeMap::new();
13434 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
13435 let (ctx, mut rx) = wide_ctx(connection);
13436 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
13437 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
13438 body["manifest"]["provenance"] =
13442 serde_json::json!({"wire_crate_version": "0.29.0"});
13443 if flow_support || run_support {
13444 let mut provides = Vec::new();
13445 if flow_support {
13446 provides.push("flow-scopes/v1");
13447 }
13448 if run_support {
13449 provides.push("agent-run-scopes/v1");
13450 }
13451 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
13452 }
13453 let hello = Frame::build(
13454 FrameType::Hello,
13455 control_flags(),
13456 0,
13457 0,
13458 connection,
13459 serde_json::to_vec(&body).unwrap(),
13460 )
13461 .unwrap();
13462 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
13463 modules.insert(module_id.to_string(), (ctx, rx));
13464 }
13465 Rig {
13466 handler,
13467 forwarding,
13468 owner,
13469 _owner_rx: owner_rx,
13470 modules,
13471 generation: 0,
13472 next_connection: 100,
13473 _supervisor: supervisor,
13474 }
13475 }
13476
13477 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
13478 ScopeCarrier::new(Principal::Reserved {
13479 module_id: module_id.to_string(),
13480 })
13481 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
13482 }
13483
13484 fn session(scope_epoch: u64) -> ScopeRecord {
13487 let mut record = head("s", scope_epoch);
13488 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
13489 record.attributes.agent_id = Some("agent-1".to_string());
13490 record.attributes.delegates = true;
13491 record
13492 }
13493
13494 impl Rig {
13495 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
13496 self.generation += 1;
13497 sync(&self.handler, &self.owner, self.generation, scopes)
13498 .await
13499 .expect("the owner's sync is accepted");
13500 }
13501
13502 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
13503 ScopeSelector {
13504 owner: Principal::Reserved {
13505 module_id: OWNER.to_string(),
13506 },
13507 scope_ref: scope_ref.to_string(),
13508 scope_epoch,
13509 }
13510 }
13511
13512 fn open_frame(
13513 &mut self,
13514 opener: Option<&str>,
13515 target: &str,
13516 scope: Option<ScopeSelector>,
13517 ) -> (
13518 RouteCtx,
13519 mpsc::Receiver<crate::router::OutboundFrame>,
13520 Frame,
13521 ) {
13522 self.next_connection += 1;
13523 let (ctx, rx) = wide_ctx(self.next_connection);
13524 let root = unique_project_root("scoped-open");
13525 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
13526 target: RouteTarget::ToolProvider {
13527 module_id: target.to_string(),
13528 },
13529 identity: BindIdentity::new(
13530 root.path().to_path_buf(),
13531 "unit".to_string(),
13532 "session".to_string(),
13533 ),
13534 consumer_identity: opener.map(|module_id| ConsumerIdentity {
13535 module_id: module_id.to_string(),
13536 launch_nonce: nonce(module_id),
13537 }),
13538 consumer_capabilities: None,
13539 role_versions: None,
13540 admission_facts: None,
13541 scope,
13542 })
13543 .unwrap();
13544 let frame = Frame::build(
13545 FrameType::Request,
13546 control_flags(),
13547 0,
13548 0,
13549 self.next_connection,
13550 body,
13551 )
13552 .unwrap();
13553 (ctx, rx, frame)
13554 }
13555
13556 async fn refused(
13558 &mut self,
13559 opener: Option<&str>,
13560 target: &str,
13561 scope: Option<ScopeSelector>,
13562 ) -> String {
13563 self.refusal_body(opener, target, scope).await["code"]
13564 .as_str()
13565 .unwrap()
13566 .to_string()
13567 }
13568
13569 async fn refusal_body(
13570 &mut self,
13571 opener: Option<&str>,
13572 target: &str,
13573 scope: Option<ScopeSelector>,
13574 ) -> Value {
13575 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
13576 let replies = tokio::time::timeout(
13577 Duration::from_secs(2),
13578 self.handler.handle_control_frame(&ctx, frame),
13579 )
13580 .await
13581 .expect("the open must be refused before waiting for a bind ack")
13582 .unwrap();
13583 assert_eq!(replies.len(), 1, "{replies:?}");
13584 assert_eq!(replies[0].header.ty, FrameType::Error);
13585 let (_, module_rx) = self.modules.get_mut(target).unwrap();
13586 assert!(
13587 module_rx.try_recv().is_err(),
13588 "a refused open relays nothing"
13589 );
13590 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
13591 parse_error(&replies[0])
13592 }
13593
13594 async fn relayed(
13596 &mut self,
13597 opener: Option<&str>,
13598 target: &str,
13599 scope: Option<ScopeSelector>,
13600 ) -> Relayed {
13601 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
13602 let handler = self.handler.clone();
13603 let task_ctx = ctx.clone();
13604 let task = tokio::spawn(async move {
13605 handler
13606 .handle_control_frame(&task_ctx, frame)
13607 .await
13608 .unwrap()
13609 });
13610 let (_, module_rx) = self.modules.get_mut(target).unwrap();
13611 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
13612 .await
13613 .expect("the target receives the relayed route.bind")
13614 .unwrap()
13615 .frame;
13616 Relayed {
13617 target: target.to_string(),
13618 client: ctx,
13619 client_rx: rx,
13620 task,
13621 bind,
13622 }
13623 }
13624
13625 async fn ack(&self, relayed: &Relayed) {
13626 let (module, _) = &self.modules[&relayed.target];
13627 self.handler
13628 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
13629 .await
13630 .unwrap();
13631 }
13632
13633 async fn bound(
13635 &mut self,
13636 opener: Option<&str>,
13637 target: &str,
13638 scope: Option<ScopeSelector>,
13639 ) -> Bound {
13640 let relayed = self.relayed(opener, target, scope).await;
13641 self.ack(&relayed).await;
13642 let Relayed {
13643 target,
13644 client,
13645 mut client_rx,
13646 task,
13647 bind,
13648 } = relayed;
13649 assert!(
13650 task.await.unwrap().is_empty(),
13651 "the open is answered by commit"
13652 );
13653 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
13654 Bound {
13655 target,
13656 client,
13657 client_rx,
13658 channel,
13659 epoch,
13660 bind,
13661 }
13662 }
13663
13664 fn live(&self, route: &Bound) -> bool {
13665 matches!(
13666 self.forwarding
13667 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
13668 .unwrap(),
13669 DataRoute::Client(DataRouteState::Bound(_))
13670 )
13671 }
13672 }
13673
13674 struct Relayed {
13675 target: String,
13676 client: RouteCtx,
13677 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13678 task: tokio::task::JoinHandle<Vec<Frame>>,
13679 bind: Frame,
13680 }
13681
13682 struct Bound {
13683 target: String,
13684 client: RouteCtx,
13685 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13686 channel: u16,
13687 epoch: u32,
13688 bind: Frame,
13689 }
13690
13691 impl Bound {
13692 fn closed_reason(&mut self) -> RouteCloseReason {
13695 let mut reason = None;
13696 let mut goodbye = false;
13697 while let Ok(outbound) = self.client_rx.try_recv() {
13698 let frame = outbound.frame;
13699 match frame.header.ty {
13700 FrameType::Goodbye => {
13701 assert_eq!(
13702 (frame.header.channel, frame.header.epoch),
13703 (self.channel, self.epoch)
13704 );
13705 goodbye = true;
13706 }
13707 FrameType::Push => {
13708 let ClientControlPush::RouteClosed {
13709 reason: r,
13710 module_id,
13711 ..
13712 } = serde_json::from_slice(&frame.body).unwrap()
13713 else {
13714 panic!("unexpected push");
13715 };
13716 assert_eq!(module_id, self.target);
13717 reason = Some(r);
13718 }
13719 other => panic!("unexpected frame {other:?}"),
13720 }
13721 }
13722 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
13723 reason.expect("the client is told why the route closed")
13724 }
13725
13726 fn untouched(&mut self) -> bool {
13727 self.client_rx.try_recv().is_err()
13728 }
13729
13730 fn stamp(&self) -> Option<ScopeStamp> {
13731 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
13732 ModuleControlRequest::RouteBind { scope, .. } => scope,
13733 other => panic!("expected a route.bind, got {other:?}"),
13734 }
13735 }
13736 }
13737
13738 #[tokio::test]
13739 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
13740 ) {
13741 let mut rig = rig().await;
13742 let mut record = session(1);
13743 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
13744 record.child_owners = vec![Principal::Reserved {
13745 module_id: MAGIC.to_string(),
13746 }];
13747 rig.sync(vec![record]).await;
13748 let scope = || Some(rig_selector("s", Some(1)));
13749
13750 rig.bound(Some(OWNER), PLEXUS, scope()).await;
13753 rig.bound(Some(AFT), OTHER, scope()).await;
13754 rig.bound(Some(BROCA), PLEXUS, scope()).await;
13755
13756 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
13760 assert_eq!(
13761 rig.refused(opener, target, scope()).await,
13762 error_codes::SCOPE_NOT_CARRIER,
13763 "{opener:?} -> {target}"
13764 );
13765 }
13766 }
13767
13768 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
13769 ScopeSelector {
13770 owner: Principal::Reserved {
13771 module_id: OWNER.to_string(),
13772 },
13773 scope_ref: scope_ref.to_string(),
13774 scope_epoch,
13775 }
13776 }
13777
13778 #[tokio::test]
13779 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
13780 let mut rig = rig().await;
13781 rig.sync(vec![session(1)]).await;
13782 for opener in [OWNER, AFT] {
13783 assert_eq!(
13784 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
13785 .await,
13786 error_codes::SCOPE_EPOCH_REQUIRED,
13787 "{opener}"
13788 );
13789 }
13790 }
13791
13792 #[tokio::test]
13793 async fn admission_separates_not_synced_not_live_and_ended() {
13794 let mut rig = rig().await;
13795 let code = rig
13797 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13798 .await;
13799 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
13800 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
13801
13802 let ghost = ScopeSelector {
13804 owner: Principal::Reserved {
13805 module_id: "ghost".to_string(),
13806 },
13807 scope_ref: "s".to_string(),
13808 scope_epoch: Some(1),
13809 };
13810 assert_eq!(
13811 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
13812 error_codes::SCOPE_NOT_LIVE
13813 );
13814
13815 rig.sync(vec![session(2)]).await;
13816 assert_eq!(
13817 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
13818 .await,
13819 error_codes::SCOPE_NOT_LIVE
13820 );
13821 for epoch in [1, 3] {
13822 assert_eq!(
13823 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
13824 .await,
13825 error_codes::SCOPE_ENDED,
13826 "epoch {epoch}"
13827 );
13828 }
13829 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
13831 .await;
13832 }
13833
13834 #[tokio::test]
13835 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
13836 let mut rig = rig().await;
13837 rig.sync(vec![session(1)]).await;
13838 let route = rig
13839 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13840 .await;
13841 let stamp = route.stamp().expect("a scoped bind carries the stamp");
13842 assert_eq!(stamp.scope_ref, "s");
13843 assert_eq!(stamp.scope_epoch, 1);
13844 assert_eq!(stamp.kind, ScopeKind::Head);
13845 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
13846 assert!(stamp.attributes.delegates);
13847 assert!(stamp.owner_authorized);
13848 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13849 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
13850
13851 let (broca, mut broca_rx) = wide_ctx(50);
13854 hello_via_sink(
13855 &rig.handler,
13856 &broca,
13857 &mut broca_rx,
13858 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
13859 )
13860 .await;
13861 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
13862 .await
13863 .unwrap();
13864 let own = ScopeSelector {
13865 owner: Principal::Reserved {
13866 module_id: BROCA.to_string(),
13867 },
13868 scope_ref: "b".to_string(),
13869 scope_epoch: Some(1),
13870 };
13871 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
13872 assert!(!route.stamp().unwrap().owner_authorized);
13873 }
13874
13875 #[tokio::test]
13876 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
13877 let mut rig = rig().await;
13878 let mut record = head("s", 1);
13879 record.carriers = vec![carrier(AFT, None)];
13880 let flow_id = "Flow:run-7/step_2!~";
13881 record.attributes.flow_id = Some(flow_id.to_string());
13882 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
13883 .await
13884 .unwrap();
13885 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
13886 panic!("not a sync reply");
13887 };
13888 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
13889 let route = rig
13890 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13891 .await;
13892 assert!(rig.live(&route), "the stamped bind committed");
13893 let stamp = route.stamp().expect("a flow scope carries a stamp");
13894 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
13895 assert_eq!(stamp.attributes.agent_id, None);
13896 assert!(!stamp.attributes.delegates);
13897 assert!(stamp.owner_authorized);
13898 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13899 assert_eq!(unscoped.stamp(), None);
13900 }
13901
13902 #[tokio::test]
13903 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
13904 let mut rig = rig_with_flow_support(false).await;
13905 let mut record = session(1);
13906 record.attributes.flow_id = Some("flow:7".to_string());
13907 rig.sync(vec![record]).await;
13908 for opener in [OWNER, AFT] {
13909 let body = rig
13910 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13911 .await;
13912 assert_eq!(body["code"], "target_flow_unsupported");
13913 let message = body["message"].as_str().unwrap();
13914 for required in [PLEXUS, "flow-scopes/v1"] {
13915 assert!(message.contains(required), "{message}");
13916 }
13917 }
13918 }
13919
13920 #[tokio::test]
13921 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
13922 let mut rig = rig_with_flow_support(true).await;
13923 let mut record = session(1);
13924 record.attributes.flow_id = Some("flow:7".to_string());
13925 rig.sync(vec![record]).await;
13926 for opener in [OWNER, AFT] {
13927 let route = rig
13928 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13929 .await;
13930 assert!(rig.live(&route));
13931 assert_eq!(
13932 route.stamp().unwrap().attributes.flow_id.as_deref(),
13933 Some("flow:7")
13934 );
13935 }
13936 }
13937
13938 #[tokio::test]
13939 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
13940 use std::future::Future;
13941
13942 let mut rig = rig_with_flow_support(true).await;
13943 let mut record = session(1);
13944 record.attributes.flow_id = Some("flow:7".to_string());
13945 rig.sync(vec![record]).await;
13946 let (client, mut client_rx, frame) =
13947 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
13948 for _ in 0..64 {
13951 client.egress.try_send(route_bind_ack(1)).unwrap();
13952 }
13953 let handler = rig.handler.clone();
13954 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
13955 std::future::poll_fn(|cx| {
13956 assert!(open.as_mut().poll(cx).is_pending());
13957 std::task::Poll::Ready(())
13958 })
13959 .await;
13960 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13961
13962 let old_connection = rig.modules[PLEXUS].0.connection_id;
13963 rig.handler.cleanup_connection(old_connection).unwrap();
13964 let (replacement, mut replacement_rx) = wide_ctx(200);
13965 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
13966 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
13967 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
13968 let hello = Frame::build(
13969 FrameType::Hello,
13970 control_flags(),
13971 0,
13972 0,
13973 200,
13974 serde_json::to_vec(&body).unwrap(),
13975 )
13976 .unwrap();
13977 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
13978
13979 client_rx.try_recv().unwrap();
13980 let replies = tokio::time::timeout(Duration::from_secs(2), open)
13981 .await
13982 .expect("the replacement is refused without waiting for a bind ack")
13983 .unwrap();
13984 assert_eq!(replies.len(), 1);
13985 let body = parse_error(&replies[0]);
13986 assert_eq!(body["code"], "target_flow_unsupported");
13987 for required in [PLEXUS, "flow-scopes/v1"] {
13988 assert!(body["message"].as_str().unwrap().contains(required));
13989 }
13990 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
13991 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
13992 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13993 assert!(rig
13994 .handler
13995 .registry
13996 .get_module_by_connection(replacement.connection_id)
13997 .unwrap()
13998 .is_some());
13999 }
14000
14001 #[tokio::test]
14002 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
14003 {
14004 let mut rig = rig_with_flow_support(false).await;
14005 rig.sync(vec![session(1)]).await;
14006 let route = rig
14007 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14008 .await;
14009 assert!(rig.live(&route));
14010 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
14011 let mut record = session(1);
14012 record.attributes.flow_id = Some("flow:7".to_string());
14013 rig.sync(vec![record]).await;
14014 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
14015 assert!(rig.live(&unscoped));
14016 assert_eq!(unscoped.stamp(), None);
14017 }
14018
14019 #[tokio::test]
14020 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
14021 let mut rig = rig().await;
14022 let mut record = session(1);
14023 record.attributes.flow_id = Some("flow:7".to_string());
14024 rig.sync(vec![record.clone()]).await;
14025 let before = rig.forwarding.published_scope_tag(OWNER, "s");
14026 let mut owner_route = rig
14027 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
14028 .await;
14029 let mut carrier_route = rig
14030 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
14031 .await;
14032 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14033 rig.sync(vec![record.clone()]).await;
14034 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
14035 assert!(rig.live(&owner_route) && owner_route.untouched());
14036 assert!(rig.live(&carrier_route) && carrier_route.untouched());
14037
14038 record.attributes.flow_id = Some("flow:8".to_string());
14039 rig.sync(vec![record]).await;
14040 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
14041 let before = before.unwrap();
14042 assert_eq!(after.scope_epoch, before.scope_epoch);
14043 assert!(after.version > before.version);
14044 for route in [&mut owner_route, &mut carrier_route] {
14045 assert!(!rig.live(route));
14046 assert_eq!(
14047 route.closed_reason(),
14048 RouteCloseReason::ScopeDelegationChanged
14049 );
14050 }
14051 assert!(rig.live(&unscoped) && unscoped.untouched());
14052 for target in [PLEXUS, OTHER] {
14055 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
14056 let goodbye = module_rx
14057 .try_recv()
14058 .expect("the provider sees the drain")
14059 .frame;
14060 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14061 assert!(module_rx.try_recv().is_err());
14062 }
14063 let rebound = rig
14064 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14065 .await;
14066 assert_eq!(
14067 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
14068 Some("flow:8")
14069 );
14070 }
14071
14072 #[tokio::test]
14077 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
14078 let mut rig = rig().await;
14079 rig.sync(vec![session(1)]).await;
14080 let mut cotenant = rig
14081 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
14082 .await;
14083
14084 let mut changed = session(1);
14085 changed.child_owners.push(Principal::Reserved {
14086 module_id: MAGIC.to_string(),
14087 });
14088 let mut ended = None;
14089 for (code, next) in [
14090 (error_codes::SCOPE_CHANGED, vec![changed]),
14091 (error_codes::SCOPE_ENDED, Vec::new()),
14092 ] {
14093 let relayed = rig
14094 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14095 .await;
14096 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
14097 ended = Some(next.is_empty());
14098 rig.sync(next).await;
14099 rig.ack(&relayed).await;
14100 let replies = relayed.task.await.unwrap();
14101 assert_eq!(replies.len(), 1, "{replies:?}");
14102 assert_eq!(parse_error(&replies[0])["code"], code);
14103 assert_eq!(
14104 subc_protocol::error_codes::is_retryable_route_open(code),
14105 code == error_codes::SCOPE_CHANGED
14106 );
14107 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14108 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14110 let mut goodbyes = Vec::new();
14113 while let Ok(outbound) = plexus_rx.try_recv() {
14114 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
14115 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
14116 }
14117 assert!(
14118 goodbyes.contains(&(bind_channel, bind_epoch)),
14119 "{goodbyes:?}"
14120 );
14121 assert!(rig
14122 .handler
14123 .registry
14124 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
14125 .unwrap()
14126 .is_some());
14127 }
14128 assert_eq!(ended, Some(true));
14129 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
14132 }
14133
14134 #[tokio::test]
14137 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
14138 struct Case {
14139 name: &'static str,
14140 change: fn(&mut ScopeRecord),
14141 closed: [Option<RouteCloseReason>; 4],
14144 }
14145 use RouteCloseReason::*;
14146 let cases = [
14147 Case {
14148 name: "a carrier entry removed",
14149 change: |r| {
14150 r.carriers.retain(|c| {
14151 c.principal
14152 != Principal::Reserved {
14153 module_id: AFT.to_string(),
14154 }
14155 })
14156 },
14157 closed: [None, Some(ScopeCarrierRemoved), None, None],
14158 },
14159 Case {
14160 name: "a target removed from a carrier",
14161 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
14162 closed: [None, None, None, Some(ScopeCarrierRemoved)],
14163 },
14164 Case {
14165 name: "a bare carrier narrowed to targets",
14166 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
14167 closed: [None, Some(ScopeCarrierRemoved), None, None],
14168 },
14169 Case {
14170 name: "delegates turned off",
14171 change: |r| r.attributes.delegates = false,
14172 closed: [Some(ScopeDelegationChanged); 4],
14173 },
14174 Case {
14175 name: "agent_id changed",
14176 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
14177 closed: [Some(ScopeDelegationChanged); 4],
14178 },
14179 Case {
14180 name: "a carrier added, child owners changed, the record re-sent",
14181 change: |r| {
14182 r.carriers.push(carrier(MAGIC, None));
14183 r.child_owners.push(Principal::Reserved {
14184 module_id: MAGIC.to_string(),
14185 });
14186 },
14187 closed: [None; 4],
14188 },
14189 Case {
14190 name: "a target added",
14191 change: |r| {
14192 r.carriers[1]
14193 .targets
14194 .as_mut()
14195 .unwrap()
14196 .push("third".to_string())
14197 },
14198 closed: [None; 4],
14199 },
14200 Case {
14201 name: "delegates turned on",
14202 change: |r| r.attributes.delegates = true,
14203 closed: [None; 4],
14204 },
14205 ];
14206 for case in cases {
14207 let mut rig = rig().await;
14208 rig.sync(vec![session(1)]).await;
14209 let scope = || Some(rig_selector("s", Some(1)));
14210 let mut routes = [
14211 rig.bound(Some(OWNER), PLEXUS, scope()).await,
14212 rig.bound(Some(AFT), PLEXUS, scope()).await,
14213 rig.bound(Some(BROCA), PLEXUS, scope()).await,
14214 rig.bound(Some(BROCA), OTHER, scope()).await,
14215 ];
14216 let mut record = session(1);
14217 (case.change)(&mut record);
14218 rig.sync(vec![record]).await;
14219 for (index, expected) in case.closed.iter().enumerate() {
14220 let route = &mut routes[index];
14221 match expected {
14222 Some(reason) => {
14223 assert!(!rig.live(route), "{}: route {index} still live", case.name);
14224 assert_eq!(
14225 route.closed_reason(),
14226 *reason,
14227 "{}: route {index}",
14228 case.name
14229 );
14230 }
14231 None => {
14232 assert!(rig.live(route), "{}: route {index} closed", case.name);
14233 assert!(
14234 route.untouched(),
14235 "{}: route {index} was told something",
14236 case.name
14237 );
14238 }
14239 }
14240 }
14241 }
14242 }
14243
14244 #[tokio::test]
14245 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
14246 for next in [Vec::new(), vec![session(2)]] {
14248 let mut rig = rig().await;
14249 rig.sync(vec![session(1)]).await;
14250 let mut route = rig
14251 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14252 .await;
14253 rig.sync(next).await;
14254 assert!(!rig.live(&route));
14255 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
14256 }
14257
14258 let mut rig = rig().await;
14261 let mut child = session(1);
14262 child.scope_ref = "child".to_string();
14263 child.kind = ScopeKind::Worker;
14264 child.parent = Some(ScopeParent::new(
14265 Principal::Reserved {
14266 module_id: OWNER.to_string(),
14267 },
14268 "s".to_string(),
14269 1,
14270 ));
14271 rig.sync(vec![session(1), child.clone()]).await;
14272 let mut child_route = rig
14273 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
14274 .await;
14275 assert_eq!(
14276 child_route.stamp().unwrap().parent_state,
14277 Some(ParentState::Linked)
14278 );
14279 rig.sync(vec![child]).await;
14280 assert!(!rig.live(&child_route));
14281 assert_eq!(
14282 child_route.closed_reason(),
14283 RouteCloseReason::ScopeParentEnded
14284 );
14285 }
14286
14287 #[tokio::test]
14288 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
14289 ) {
14290 let mut rig = rig().await;
14291 rig.sync(vec![session(1)]).await;
14292 let mut route = rig
14293 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14294 .await;
14295 let before = rig.forwarding.published_scope_tag(OWNER, "s");
14296 rig.sync(vec![session(1)]).await;
14297 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
14298 assert!(rig.live(&route) && route.untouched());
14299
14300 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
14304 .forwarding
14305 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
14306 .unwrap()
14307 else {
14308 panic!("the route is bound");
14309 };
14310 binding.flow.acquire_tagged(9, false).await.unwrap();
14311 let mut widened = session(1);
14312 widened.carriers.push(carrier(MAGIC, None));
14313 rig.sync(vec![widened]).await;
14314 assert!(rig.live(&route) && route.untouched());
14315 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14316 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
14317 assert_eq!(binding.flow.in_flight(), 1);
14320 binding
14321 .flow
14322 .acquire_tagged(10, false)
14323 .await
14324 .expect("the flow is still open");
14325 }
14326
14327 #[tokio::test]
14330 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
14331 let mut rig = rig().await;
14332 rig.sync(vec![session(1)]).await;
14333 let mut on_incumbent = rig
14334 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14335 .await;
14336
14337 let (candidate, _candidate_rx) = wide_ctx(9);
14340 let registration = rig
14341 .handler
14342 .registry
14343 .register_candidate_with_control_ops(
14344 manifest(PLEXUS, PROTOCOL_VERSION),
14345 PROTOCOL_VERSION,
14346 candidate.connection_id,
14347 module_baseline_control_ops(),
14348 )
14349 .unwrap();
14350 rig.forwarding
14351 .register_candidate_module_connection(
14352 candidate.connection_id,
14353 PLEXUS.to_string(),
14354 PROTOCOL_VERSION,
14355 manifest_concurrency(®istration.manifest),
14356 candidate.egress.clone(),
14357 )
14358 .unwrap();
14359 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
14360 rig.handler
14361 .registry
14362 .promote_candidate(PLEXUS)
14363 .unwrap()
14364 .unwrap();
14365 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
14366
14367 rig.sync(Vec::new()).await;
14368 assert!(!rig.live(&on_incumbent));
14369 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
14370 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14371 let goodbye = incumbent_rx
14372 .try_recv()
14373 .expect("the superseded endpoint is told")
14374 .frame;
14375 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14376 }
14377 }
14378}
14379
14380#[cfg(test)]
14381mod concurrency_default_exposure_tests {
14382 use super::*;
14383
14384 fn hello_body(role_json: &str) -> Vec<u8> {
14385 format!(
14386 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
14387 )
14388 .into_bytes()
14389 }
14390
14391 fn manifest_from(body: &[u8]) -> ModuleManifest {
14392 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
14393 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
14394 .expect("manifest parses")
14395 }
14396
14397 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
14398
14399 #[test]
14400 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
14401 let body = hello_body(&format!(
14402 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
14403 ));
14404 let manifest = manifest_from(&body);
14405 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
14408 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
14409 }
14410
14411 #[test]
14412 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
14413 let body = hello_body(&format!(
14414 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
14415 ));
14416 let manifest = manifest_from(&body);
14417 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
14418 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
14419 }
14420
14421 #[test]
14422 fn non_management_roles_are_never_reported() {
14423 let body = hello_body(
14424 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
14425 );
14426 let manifest = manifest_from(&body);
14427 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
14428 }
14429}