1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_event_declarations,
23 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
24 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
25 },
26 scope::{
27 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
29 SCOPE_SYNC_OP,
30 },
31 session::{
32 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
33 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
34 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
35 ROLE_VERSIONS_FIELD,
36 },
37 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
38 Priority, RouteTarget, PROTOCOL_VERSION,
39};
40use tokio::time::{timeout_at, Instant};
41use tracing::{debug, info, warn};
42
43use crate::{
44 capability_requirements::{
45 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
46 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
47 RuntimeModule,
48 },
49 daemon_config::RestartRequiredSection,
50 forwarding::{
51 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
52 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
53 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
54 },
55 observability::{
56 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
57 },
58 provenance::{
59 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
60 },
61 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
62 router::{RouteCtx, RouterError},
63 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
64 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
65 stderr_tail::{CaptureState, TailEntry},
66 supervise::{
67 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
68 SupervisorHandle, SwapHelloAdmission,
69 },
70 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
71};
72
73pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
79
80const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
81const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
82const CAP_PING_PONG: &str = "ping_pong_v1";
83const CAP_SESSION_ATTACH: &str = "session_attach_v1";
84const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
85
86const SUBC_CONTROL_OPS: &[&str] = &[
87 ops::SERVER_DESCRIBE,
88 ops::CATALOG_LIST,
89 ops::ROUTE_OPEN,
90 ops::ROUTE_POLL,
91 ops::ROUTE_CLOSING,
92 ops::ROUTE_CLOSED,
93 ops::SUPERVISOR_LIST,
94 ops::SUPERVISOR_RESTART,
95 ops::SUPERVISOR_SWAP,
96 ops::SUPERVISOR_RELOAD,
97 ops::SUPERVISOR_RESCAN,
98 ops::SUPERVISOR_RELEASE_RESERVED,
99 ops::SUPERVISOR_SET_ENABLED,
100 ops::SUPERVISOR_HEALTH_PROBE,
101 ops::SUPERVISOR_HEALTH,
102 ops::SUPERVISOR_STDERR_TAIL,
103 ops::SUPERVISOR_TERMINALS,
104 ops::SUPERVISOR_ROUTES,
105 ops::SUPERVISOR_PROVENANCE,
106 ops::SUPERVISOR_SPAWN_SNAPSHOT,
107 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
108];
109
110const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
111 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
112 "supervisor.live_roots",
113 SCOPE_SYNC_OP,
114 SCOPE_APPLY_OP,
115 SCOPE_DESCRIBE_OP,
116 "operator.confirm",
117];
118
119const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
123
124const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
125
126pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
134
135pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
150
151pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
169
170const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
171const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
172
173fn reload_verdict(
174 configured: &Path,
175 spawned_from: Option<&Path>,
176 image: subc_control::RunningImageAgreement,
177) -> PendingReloadVerdict {
178 let path = match spawned_from {
179 Some(spawned_from) if configured == spawned_from => ReloadPathAgreement::Match,
180 Some(spawned_from) => ReloadPathAgreement::Mismatch {
181 configured: configured.to_path_buf(),
182 spawned_from: spawned_from.to_path_buf(),
183 },
184 None => ReloadPathAgreement::Unavailable {
185 reason: if matches!(
186 image,
187 subc_control::RunningImageAgreement::Unavailable {
188 reason: subc_control::RunningImageUnavailableReason::NotRunning
189 }
190 ) {
191 ReloadPathUnavailableReason::NotRunning
192 } else {
193 ReloadPathUnavailableReason::SpawnedPathUnavailable
194 },
195 },
196 };
197 PendingReloadVerdict { path, image }
198}
199
200#[derive(Clone)]
201struct DaemonProvenanceFacts {
202 build: DaemonBuildProvenance,
203 pid: Option<u32>,
204 started_at_ms: Option<u64>,
205 start_clock: Option<crate::clock::StartClock>,
206 executable_path: Option<PathBuf>,
207 executable_identity: Option<SpawnedFileIdentity>,
208 process_start_time: Option<u64>,
209 probe: ExecutableIdentityProbe,
210}
211
212impl Default for DaemonProvenanceFacts {
213 fn default() -> Self {
214 Self {
215 build: DaemonBuildProvenance {
216 build_git_sha: None,
217 build_lock_digest: None,
218 },
219 pid: None,
220 started_at_ms: None,
221 start_clock: None,
222 executable_path: None,
223 executable_identity: None,
224 process_start_time: None,
225 probe: ExecutableIdentityProbe::default(),
226 }
227 }
228}
229
230#[derive(Debug, Clone)]
231struct SupervisorRescanContext {
232 supervisor: Supervisor,
233 config_path: PathBuf,
234 configured_port: Option<u16>,
235 storage_config: Option<crate::daemon_config::StorageConfig>,
236 admission_facts_carrier_module_id: Option<String>,
237 admission_facts_targets: Option<Vec<String>>,
238 scope_authority_owners: Vec<String>,
239}
240
241const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
262 "reloading",
263 "supervisor_not_live",
264 "registration_not_active",
265 "no_forwarding_connection",
266 "relay_send_failed",
267];
268
269#[derive(Clone)]
271pub struct ControlHandler {
272 registry: Arc<Registry>,
273 forwarding: Arc<ForwardingTable>,
274 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
275 supervisor: SupervisorHandle,
276 subc_capabilities: Arc<[String]>,
277 route_bind_relay_timeout: Duration,
281 route_bind_relay_timeouts: BTreeMap<String, Duration>,
285 route_bind_breakers: RouteBindBreakers,
288 route_bind_concurrency: RouteBindConcurrency,
291 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
296 route_bind_breaker_threshold: u32,
298 route_bind_breaker_cooldown: Duration,
300 health_probe_timeout: Duration,
301 storage_config: Option<crate::daemon_config::StorageConfig>,
304 machine_id: Option<crate::machine_id::MachineId>,
308 admission_facts_carrier_module_id: Option<String>,
309 admission_facts_targets: Option<Vec<String>>,
310 scopes: Arc<RwLock<ScopeTable>>,
314 scope_deadline_added: Arc<tokio::sync::Notify>,
318 scope_authority_owners: Vec<String>,
321 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
324 rescan: Option<SupervisorRescanContext>,
325 connected_clients: ConnectedClients,
326 counters: DaemonCounters,
327 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
328 daemon_provenance: DaemonProvenanceFacts,
329 #[cfg(test)]
330 control_dispatch_delay: Option<Duration>,
331 #[cfg(test)]
332 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
333 #[cfg(test)]
337 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
338}
339
340impl fmt::Debug for ControlHandler {
341 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
342 f.debug_struct("ControlHandler")
343 .field("registry", &self.registry)
344 .field("forwarding", &self.forwarding)
345 .field("process_liveness", &self.process_liveness.is_some())
346 .field("supervisor", &self.supervisor)
347 .field("subc_capabilities", &self.subc_capabilities)
348 .finish()
349 }
350}
351
352struct RouteOpenRequest {
353 target: RouteTarget,
354 identity: BindIdentity,
355 consumer_identity: Option<ConsumerIdentity>,
356 consumer_capabilities: Option<Vec<String>>,
357 role_versions: Option<BTreeMap<String, String>>,
358 admission_facts: Option<serde_json::Value>,
359 scope: Option<ScopeSelector>,
360}
361
362struct RouteBindReservationGuard {
363 forwarding: Arc<ForwardingTable>,
364 endpoint: ModuleEndpointId,
365 relay_corr: u64,
366 armed: bool,
367}
368
369struct ModuleControlRpcGuard {
370 forwarding: Arc<ForwardingTable>,
371 endpoint: ModuleEndpointId,
372 corr: u64,
373 armed: bool,
374}
375
376impl ModuleControlRpcGuard {
377 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
378 Self {
379 forwarding,
380 endpoint,
381 corr,
382 armed: true,
383 }
384 }
385
386 fn disarm(&mut self) {
387 self.armed = false;
388 }
389}
390
391impl Drop for ModuleControlRpcGuard {
392 fn drop(&mut self) {
393 if self.armed {
394 let _ = self
395 .forwarding
396 .cancel_module_control_rpc(self.endpoint, self.corr);
397 }
398 }
399}
400
401impl RouteBindReservationGuard {
402 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
403 Self {
404 forwarding,
405 endpoint,
406 relay_corr,
407 armed: true,
408 }
409 }
410
411 fn release_and_disarm(&mut self) {
412 if !self.armed {
413 return;
414 }
415 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
416 self.endpoint,
417 self.relay_corr,
418 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
419 ) {
420 send_goodbye_target_best_effort(
421 &self.forwarding.counters(),
422 &target,
423 "canceled route.bind",
424 );
425 }
426 self.armed = false;
427 }
428
429 fn disarm(&mut self) {
430 self.armed = false;
431 }
432}
433
434impl Drop for RouteBindReservationGuard {
435 fn drop(&mut self) {
436 self.release_and_disarm();
437 }
438}
439
440#[derive(Debug, Clone, Default)]
470pub(crate) struct RouteBindBreakers {
471 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
472}
473
474#[derive(Debug, Clone, Default)]
475pub(crate) struct RouteBindConcurrency {
476 modules: Arc<Mutex<HashMap<String, usize>>>,
477}
478
479struct RouteBindConcurrencyGuard {
480 concurrency: RouteBindConcurrency,
481 module_id: String,
482}
483
484impl RouteBindConcurrency {
485 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
488 let mut modules = self
489 .modules
490 .lock()
491 .expect("route.bind concurrency mutex poisoned");
492 let in_flight = modules.entry(module_id.to_string()).or_default();
493 if *in_flight >= limit {
494 return Err(*in_flight);
495 }
496 *in_flight += 1;
497 Ok(RouteBindConcurrencyGuard {
498 concurrency: self.clone(),
499 module_id: module_id.to_string(),
500 })
501 }
502}
503
504impl Drop for RouteBindConcurrencyGuard {
505 fn drop(&mut self) {
506 let mut modules = self
507 .concurrency
508 .modules
509 .lock()
510 .expect("route.bind concurrency mutex poisoned");
511 let remove = {
512 let in_flight = modules
513 .get_mut(&self.module_id)
514 .expect("admitted route.bind has a concurrency entry");
515 *in_flight -= 1;
516 *in_flight == 0
517 };
518 if remove {
519 modules.remove(&self.module_id);
520 }
521 }
522}
523
524#[derive(Debug, Default)]
525struct ModuleBreakerState {
526 consecutive_timeouts: u32,
528 cooldown_until: Option<Instant>,
531 probe_in_flight: Option<Arc<()>>,
536}
537
538enum RouteBindAdmission<'a> {
540 Admitted {
541 guard: RouteBindBreakerGuard<'a>,
542 probe: bool,
545 },
546 Refused {
547 consecutive_timeouts: u32,
548 retry_in: Duration,
552 probe_in_flight: bool,
553 },
554}
555
556struct RouteBindBreakerGuard<'a> {
563 breakers: RouteBindBreakers,
564 module_id: &'a str,
565 probe_token: Option<Arc<()>>,
566 settled: bool,
567}
568
569impl RouteBindBreakerGuard<'_> {
570 fn record_accepted(&mut self) -> bool {
574 self.settled = true;
575 self.breakers.record_accepted(self.module_id)
576 }
577
578 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
581 self.settled = true;
582 self.breakers.record_timeout(
583 self.module_id,
584 self.probe_token.as_ref(),
585 threshold,
586 cooldown,
587 )
588 }
589
590 fn record_inconclusive(&mut self) {
599 self.settled = true;
600 self.breakers
601 .record_inconclusive(self.module_id, self.probe_token.as_ref());
602 }
603}
604
605impl Drop for RouteBindBreakerGuard<'_> {
606 fn drop(&mut self) {
607 if !self.settled {
608 self.breakers
609 .record_inconclusive(self.module_id, self.probe_token.as_ref());
610 }
611 }
612}
613
614struct BreakerOpened {
618 consecutive_timeouts: u32,
619 reopened_after_probe: bool,
622}
623
624impl RouteBindBreakers {
625 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
626 self.modules
627 .lock()
628 .expect("route.bind breaker mutex poisoned")
629 }
630
631 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
634 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
635 probe: probe_token.is_some(),
636 guard: RouteBindBreakerGuard {
637 breakers: self.clone(),
638 module_id,
639 probe_token,
640 settled: false,
641 },
642 };
643
644 let mut modules = self.lock();
645 let Some(state) = modules.get_mut(module_id) else {
646 return admitted(None);
647 };
648 let Some(cooldown_until) = state.cooldown_until else {
649 return admitted(None);
650 };
651 if state.probe_in_flight.is_some() {
652 return RouteBindAdmission::Refused {
653 consecutive_timeouts: state.consecutive_timeouts,
654 retry_in: Duration::ZERO,
655 probe_in_flight: true,
656 };
657 }
658 let now = Instant::now();
659 if now < cooldown_until {
660 return RouteBindAdmission::Refused {
661 consecutive_timeouts: state.consecutive_timeouts,
662 retry_in: cooldown_until - now,
663 probe_in_flight: false,
664 };
665 }
666 let token = Arc::new(());
667 state.probe_in_flight = Some(Arc::clone(&token));
668 admitted(Some(token))
669 }
670
671 fn record_accepted(&self, module_id: &str) -> bool {
672 self.lock()
673 .remove(module_id)
674 .is_some_and(|state| state.cooldown_until.is_some())
675 }
676
677 fn record_timeout(
678 &self,
679 module_id: &str,
680 probe_token: Option<&Arc<()>>,
681 threshold: u32,
682 cooldown: Duration,
683 ) -> Option<BreakerOpened> {
684 let mut modules = self.lock();
685 let state = modules.entry(module_id.to_string()).or_default();
686 let was_open = state.cooldown_until.is_some();
687 let was_probe = Self::owns_probe(state, probe_token);
688 if was_probe {
689 state.probe_in_flight = None;
690 }
691 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
692 if state.consecutive_timeouts < threshold {
693 return None;
694 }
695 state.cooldown_until = Some(Instant::now() + cooldown);
696 Some(BreakerOpened {
697 consecutive_timeouts: state.consecutive_timeouts,
698 reopened_after_probe: was_open && was_probe,
699 })
700 }
701
702 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
703 state
707 .probe_in_flight
708 .as_ref()
709 .zip(token)
710 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
711 }
712
713 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
714 if let Some(state) = self.lock().get_mut(module_id) {
715 if Self::owns_probe(state, probe_token) {
716 state.probe_in_flight = None;
717 }
718 }
719 }
720
721 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
736 self.lock()
737 .remove(module_id)
738 .map(|state| state.consecutive_timeouts)
739 .filter(|discarded| *discarded > 0)
740 }
741
742 fn open_snapshot(&self) -> Option<serde_json::Value> {
749 let now = Instant::now();
750 let modules = self.lock();
751 let open = modules
752 .iter()
753 .filter_map(|(module_id, state)| {
754 let cooldown_until = state.cooldown_until?;
755 Some((
756 module_id.clone(),
757 serde_json::json!({
758 "consecutive_timeouts": state.consecutive_timeouts,
759 "cooldown_remaining_ms":
760 cooldown_until.saturating_duration_since(now).as_millis() as u64,
761 "probe_in_flight": state.probe_in_flight.is_some(),
762 }),
763 ))
764 })
765 .collect::<serde_json::Map<String, serde_json::Value>>();
766 (!open.is_empty()).then_some(serde_json::Value::Object(open))
767 }
768}
769
770impl ControlHandler {
771 pub fn new(registry: Arc<Registry>) -> Self {
772 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
773 }
774
775 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
776 let counters = forwarding.counters();
777 let route_bind_breakers = forwarding.route_bind_breakers();
781 let route_bind_concurrency = forwarding.route_bind_concurrency();
782 let route_outages = forwarding.route_outages();
783 Self {
784 registry,
785 forwarding,
786 process_liveness: None,
787 supervisor: SupervisorHandle::new(),
788 subc_capabilities: Arc::from([
789 CAP_MANIFEST_REGISTRATION.to_string(),
790 CAP_CHANNEL_LIFECYCLE.to_string(),
791 CAP_PING_PONG.to_string(),
792 CAP_SESSION_ATTACH.to_string(),
793 CAP_ADMISSION_FACTS_RELAY.to_string(),
794 CAP_SCOPES_V1.to_string(),
795 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
796 ]),
797 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
798 route_bind_relay_timeouts: BTreeMap::new(),
799 route_bind_breakers,
800 route_bind_concurrency,
801 route_outages,
802 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
803 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
804 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
805 storage_config: None,
806 machine_id: None,
807 admission_facts_carrier_module_id: None,
808 admission_facts_targets: None,
809 scopes: Arc::new(RwLock::new(ScopeTable::new(
810 crate::daemon_config::default_scope_authority_owners(),
811 ))),
812 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
813 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
814 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
815 rescan: None,
816 connected_clients: ConnectedClients::new(),
817 counters,
818 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
819 daemon_provenance: DaemonProvenanceFacts::default(),
820 #[cfg(test)]
821 control_dispatch_delay: None,
822 #[cfg(test)]
823 provenance_probe_override: None,
824 #[cfg(test)]
825 route_bind_relay_sent_hold: None,
826 }
827 }
828
829 pub fn with_storage_config(
832 mut self,
833 storage_config: Option<crate::daemon_config::StorageConfig>,
834 ) -> Self {
835 self.storage_config = storage_config;
836 self
837 }
838
839 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
842 self.machine_id = machine_id;
843 self
844 }
845
846 pub fn with_admission_facts_config(
850 mut self,
851 carrier_module_id: Option<String>,
852 targets: Option<Vec<String>>,
853 ) -> Self {
854 self.admission_facts_carrier_module_id = carrier_module_id;
855 self.admission_facts_targets = targets;
856 self
857 }
858
859 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
863 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
864 self.scope_authority_owners = owners;
865 self
866 }
867
868 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
871 self.route_bind_relay_timeout = timeout;
872 self
873 }
874
875 pub fn with_route_bind_relay_timeouts(
881 mut self,
882 timeouts: impl IntoIterator<Item = (String, Duration)>,
883 ) -> Self {
884 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
885 self
886 }
887
888 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
894 self.route_bind_relay_timeouts
895 .get(module_id)
896 .copied()
897 .unwrap_or(self.route_bind_relay_timeout)
898 }
899
900 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
908 self.route_bind_breaker_threshold = threshold.max(1);
909 self.route_bind_breaker_cooldown = cooldown;
910 self
911 }
912
913 #[cfg(test)]
914 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
915 self.health_probe_timeout = timeout;
916 self
917 }
918
919 #[cfg(test)]
920 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
921 self.control_dispatch_delay = Some(delay);
922 self
923 }
924
925 #[cfg(test)]
929 pub(crate) fn with_route_bind_relay_sent_hold(
930 mut self,
931 hold: Arc<tokio::sync::Barrier>,
932 ) -> Self {
933 self.route_bind_relay_sent_hold = Some(hold);
934 self
935 }
936
937 pub fn with_process_liveness(
938 mut self,
939 process_liveness: Arc<dyn ModuleProcessLiveness>,
940 ) -> Self {
941 self.process_liveness = Some(process_liveness);
942 self
943 }
944
945 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
946 self.supervisor = supervisor;
947 self
948 }
949
950 pub fn with_daemon_provenance(
951 mut self,
952 pid: u32,
953 started_at_ms: u64,
954 executable_path: Option<PathBuf>,
955 build_git_sha: Option<String>,
956 build_lock_digest: Option<String>,
957 ) -> Self {
958 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
959 let process_start_time = process_start_time(pid);
960 self.daemon_provenance = DaemonProvenanceFacts {
961 build: DaemonBuildProvenance {
962 build_git_sha,
963 build_lock_digest,
964 },
965 pid: Some(pid),
966 started_at_ms: Some(started_at_ms),
967 start_clock: None,
968 executable_path,
969 executable_identity,
970 process_start_time,
971 probe: ExecutableIdentityProbe::default(),
972 };
973 self
974 }
975
976 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
977 self.daemon_provenance.start_clock = Some(clock);
978 self
979 }
980
981 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
983 self.scopes
984 .write()
985 .unwrap_or_else(|poisoned| poisoned.into_inner())
986 .set_wall_clock(Arc::new(clock));
987 self
988 }
989
990 #[cfg(test)]
991 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
992 self.provenance_probe_override = Some(result);
993 self
994 }
995
996 pub fn with_capability_config(
1000 self,
1001 modules: impl IntoIterator<Item = (String, bool)>,
1002 reserved_capabilities: BTreeMap<String, String>,
1003 ) -> Self {
1004 self.capability_evaluator
1005 .configure(modules, reserved_capabilities);
1006 self
1007 }
1008
1009 pub fn with_supervisor_rescan(
1010 mut self,
1011 supervisor: Supervisor,
1012 config_path: impl Into<PathBuf>,
1013 configured_port: Option<u16>,
1014 ) -> Self {
1015 self.rescan = Some(SupervisorRescanContext {
1016 supervisor,
1017 config_path: config_path.into(),
1018 configured_port,
1019 storage_config: self.storage_config.clone(),
1020 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1021 admission_facts_targets: self.admission_facts_targets.clone(),
1022 scope_authority_owners: self.scope_authority_owners.clone(),
1023 });
1024 self
1025 }
1026
1027 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1028 self.connected_clients = connected_clients;
1029 self
1030 }
1031
1032 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1033 Arc::clone(&self.forwarding)
1034 }
1035
1036 pub(crate) fn counters(&self) -> DaemonCounters {
1037 self.counters.clone()
1038 }
1039
1040 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1043 tokio::spawn(async move {
1044 loop {
1045 self.capability_evaluator
1046 .wait_for_change_or_deadline()
1047 .await;
1048 self.refresh_capability_requirements();
1049 }
1050 });
1051 }
1052
1053 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1059 tokio::spawn(async move {
1060 loop {
1061 let has_deadlines = self
1062 .scopes
1063 .read()
1064 .unwrap_or_else(|poisoned| poisoned.into_inner())
1065 .has_deadlines();
1066 if !has_deadlines {
1067 self.scope_deadline_added.notified().await;
1068 continue;
1069 }
1070 tokio::time::sleep(Duration::from_secs(1)).await;
1071 if let Err(error) = self.sweep_expired_scopes() {
1072 warn!(%error, "scope expiry sweep failed");
1073 }
1074 }
1075 });
1076 }
1077
1078 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1079 let mut table = self
1080 .scopes
1081 .write()
1082 .unwrap_or_else(|poisoned| poisoned.into_inner());
1083 let swept = table.sweep_expired();
1084 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1085 drop(table);
1086 self.close_scope_drained_routes(drained);
1087 Ok(())
1088 }
1089
1090 fn publish_scope_changes(
1091 &self,
1092 changes: &[crate::scopes::ScopeTagChange],
1093 expired: &[crate::scopes::ScopeExpired],
1094 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1095 if changes.is_empty() {
1096 return Ok(Vec::new());
1097 }
1098 let drained = self
1099 .forwarding
1100 .publish_scope_changes(changes)
1101 .map_err(RouterError::Forwarding)?;
1102 let mut counts = HashMap::new();
1103 for route in &drained {
1104 *counts
1105 .entry((
1106 route.scope.owner.as_str(),
1107 route.scope.scope_ref.as_str(),
1108 route.scope.tag.scope_epoch,
1109 ))
1110 .or_insert(0usize) += 1;
1111 }
1112 for scope in expired {
1113 let routes_closed = counts
1114 .get(&(
1115 scope.owner.as_str(),
1116 scope.scope_ref.as_str(),
1117 scope.scope_epoch,
1118 ))
1119 .copied()
1120 .unwrap_or(0);
1121 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1122 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1123 routes_closed, "scope expired");
1124 }
1125 Ok(drained)
1126 }
1127
1128 fn runtime_capability_snapshot(
1129 &self,
1130 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1131 let runtime = self
1132 .supervisor
1133 .list()
1134 .into_iter()
1135 .map(|module| {
1136 let status = module.status().map_err(|err| {
1137 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1138 })?;
1139 Ok(RuntimeModule {
1140 module_id: status.module_id,
1141 state: status.state,
1142 enabled: status.enabled,
1143 })
1144 })
1145 .collect::<Result<Vec<_>, RouterError>>()?;
1146 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1147 RouterError::backend(
1148 0,
1149 0,
1150 format!("failed to list capability registrations: {err}"),
1151 )
1152 })?;
1153 let registrations = registrations
1154 .into_iter()
1155 .map(|registration| RegisteredModule {
1156 module_id: registration.manifest.module_id,
1157 module_version: registration.manifest.module_version,
1158 capabilities: registration.manifest.capabilities,
1159 })
1160 .collect();
1161 Ok((runtime, registrations))
1162 }
1163
1164 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1171 let cached_registration = RegisteredModule {
1172 module_id: registration.manifest.module_id.clone(),
1173 module_version: registration.manifest.module_version.clone(),
1174 capabilities: registration.manifest.capabilities.clone(),
1175 };
1176 if self.capability_evaluator.record_hello(&cached_registration) {
1177 warn!(
1178 module_id = %cached_registration.module_id,
1179 "capability claims drifted from the cached manifest"
1180 );
1181 }
1182 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1183 self.enforce_capability_denies();
1184 }
1185 self.refresh_capability_requirements();
1186 }
1187
1188 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1192 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1193 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1194 self.supervisor.set_swap_promotion_observer(observer);
1195 }
1196
1197 pub fn refresh_capability_requirements(&self) {
1198 match self.runtime_capability_snapshot() {
1199 Ok((runtime, registrations)) => {
1200 log_requirement_events(
1201 self.capability_evaluator
1202 .evaluate_now(&runtime, ®istrations),
1203 );
1204 }
1205 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1206 }
1207 }
1208
1209 fn enforce_capability_denies(&self) {
1213 let (_, registrations) = match self.registry.list_modules() {
1214 Ok(snapshot) => snapshot,
1215 Err(err) => {
1216 warn!(error = %err, "failed to read registrations for capability deny census");
1217 return;
1218 }
1219 };
1220 let manifests = registrations
1221 .into_iter()
1222 .map(|registration| {
1223 (
1224 registration.manifest.module_id.clone(),
1225 registration.manifest,
1226 )
1227 })
1228 .collect::<BTreeMap<_, _>>();
1229 let census = match self.forwarding.route_census(None) {
1230 Ok(census) => census,
1231 Err(err) => {
1232 warn!(error = %err, "failed to read route census for capability deny enforcement");
1233 return;
1234 }
1235 };
1236
1237 for (target_module_id, routes) in census {
1238 let Some(target_manifest) = manifests.get(&target_module_id) else {
1239 continue;
1240 };
1241 let mut closed_routes = Vec::new();
1242 let mut module_goodbyes = Vec::new();
1243 for route in routes {
1244 let Principal::Reserved {
1245 module_id: opening_module_id,
1246 } = &route.principal
1247 else {
1248 continue;
1249 };
1250 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1251 continue;
1252 };
1253 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1254 continue;
1255 };
1256
1257 match self.forwarding.release_client_route(
1258 route.goodbye_target.connection_id,
1259 route.goodbye_target.channel,
1260 route.goodbye_target.epoch,
1261 ) {
1262 Ok(RouteRelease::Removed(module_goodbye)) => {
1263 warn!(
1264 opening_module_id,
1265 target_module_id,
1266 capability,
1267 "force-closing route because an attested capability deny edge now matches"
1268 );
1269 closed_routes.push(route);
1270 module_goodbyes.push(module_goodbye);
1271 }
1272 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1273 Err(err) => warn!(
1274 opening_module_id,
1275 target_module_id,
1276 capability,
1277 error = %err,
1278 "failed to force-close capability-denied route"
1279 ),
1280 }
1281 }
1282
1283 if closed_routes.is_empty() {
1284 continue;
1285 }
1286 send_route_control_pushes(
1287 &self.forwarding,
1288 closed_routes,
1289 ClientControlPush::RouteClosed {
1290 module_id: target_module_id,
1291 channels: Vec::new(),
1292 reason: RouteCloseReason::CapabilityDenied,
1293 drained: false,
1294 abandoned: 0,
1295 excluded_subscriptions: 0,
1296 terminal: Some(false),
1297 },
1298 );
1299 self.emit_route_goodbyes(module_goodbyes);
1300 }
1301 }
1302
1303 fn not_ready_reason(
1309 &self,
1310 registration: &crate::registry::ModuleRegistration,
1311 ) -> Option<NotReadyReason> {
1312 if !registration.ready {
1313 return Some(NotReadyReason {
1314 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1315 capability: None,
1316 });
1317 }
1318 self.first_unprovided_required_capability(registration)
1319 .map(|capability| NotReadyReason {
1320 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1321 capability: Some(capability),
1322 })
1323 }
1324
1325 fn first_unprovided_required_capability(
1343 &self,
1344 registration: &crate::registry::ModuleRegistration,
1345 ) -> Option<String> {
1346 let required = registration
1347 .manifest
1348 .capabilities
1349 .iter()
1350 .flat_map(|declarations| declarations.requires.iter())
1351 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1352 .map(|requirement| requirement.capability.as_str())
1353 .collect::<BTreeSet<_>>();
1354 if required.is_empty() {
1355 return None;
1356 }
1357 let module_id = registration.manifest.module_id.as_str();
1358 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1359 if required
1360 .iter()
1361 .any(|capability| verdict(capability).is_none())
1362 {
1363 self.refresh_capability_requirements();
1364 }
1365 required
1366 .into_iter()
1367 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1368 .map(str::to_string)
1369 }
1370
1371 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1372 self.capability_evaluator
1373 .statuses()
1374 .into_iter()
1375 .map(capability_requirement_status)
1376 .collect()
1377 }
1378
1379 fn deregister_connection(
1386 &self,
1387 connection_id: ConnectionId,
1388 reason: RegistrationEndReason,
1389 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1390 self.registry
1391 .deregister_connection_with_reason(connection_id, reason)
1392 }
1393
1394 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1395 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1396 return None;
1397 }
1398 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1399 parse_client_control_request(&frame.body)
1400 else {
1401 return None;
1402 };
1403 Some(target_module_id(&target).to_string())
1404 }
1405
1406 pub(crate) fn route_open_capacity_refusal(
1407 &self,
1408 ctx: &RouteCtx,
1409 frame: &Frame,
1410 target_module_id: &str,
1411 in_flight: usize,
1412 limit: usize,
1413 ) -> Result<Frame, RouterError> {
1414 self.route_open_admission_refusal_frame(
1415 ctx,
1416 frame,
1417 target_module_id,
1418 "open_admission_full",
1419 (in_flight, limit),
1420 format!(
1421 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1422 ),
1423 )
1424 }
1425
1426 fn route_open_target_capacity_refusal(
1427 &self,
1428 ctx: &RouteCtx,
1429 frame: &Frame,
1430 target_module_id: &str,
1431 in_flight: usize,
1432 ) -> Result<Frame, RouterError> {
1433 self.route_open_admission_refusal_frame(
1434 ctx,
1435 frame,
1436 target_module_id,
1437 "target_binds_full",
1438 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1439 format!(
1440 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1441 ),
1442 )
1443 }
1444
1445 fn route_open_admission_refusal_frame(
1453 &self,
1454 ctx: &RouteCtx,
1455 frame: &Frame,
1456 target_module_id: &str,
1457 reason: &'static str,
1458 (in_flight, limit): (usize, usize),
1459 message: impl Into<String>,
1460 ) -> Result<Frame, RouterError> {
1461 let code = error_codes::TARGET_UNAVAILABLE;
1462 self.counters.increment_route_open_refused(code);
1463 info!(
1464 target: "control",
1465 code,
1466 reason,
1467 module_id = ?target_module_id,
1468 connection_id = ctx.connection_id.get(),
1469 in_flight,
1470 limit,
1471 "route.open refused"
1472 );
1473 control_error_frame(frame, code, message.into())
1474 }
1475
1476 #[cfg(test)]
1482 pub fn handle_control(
1483 &self,
1484 connection_id: ConnectionId,
1485 frame: Frame,
1486 ) -> Result<Vec<Frame>, RouterError> {
1487 match frame.header.ty {
1488 FrameType::Ping => Ok(vec![pong(&frame)?]),
1489 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1490 FrameType::Goodbye => self.handle_goodbye(connection_id),
1491 ty => Ok(vec![control_error_frame(
1492 &frame,
1493 "unsupported_control_frame",
1494 format!("unsupported channel-0 frame {ty:?}"),
1495 )?]),
1496 }
1497 }
1498
1499 pub async fn handle_control_frame(
1500 &self,
1501 ctx: &RouteCtx,
1502 frame: Frame,
1503 ) -> Result<Vec<Frame>, RouterError> {
1504 self.handle_control_frame_timed(ctx, frame, None).await
1505 }
1506
1507 pub(crate) async fn handle_control_frame_timed(
1508 &self,
1509 ctx: &RouteCtx,
1510 frame: Frame,
1511 dispatch_started_at: Option<StdInstant>,
1512 ) -> Result<Vec<Frame>, RouterError> {
1513 match frame.header.ty {
1514 FrameType::Ping => Ok(vec![pong(&frame)?]),
1515 FrameType::Hello => {
1516 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1517 }
1518 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1519 FrameType::Cancel => {
1520 if self
1523 .forwarding
1524 .operator_confirms()
1525 .cancel(ctx.connection_id, frame.header.corr)
1526 || self
1527 .supervisor
1528 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1529 {
1530 Ok(Vec::new())
1531 } else {
1532 Ok(vec![control_error_frame(
1533 &frame,
1534 "unknown_subscription",
1535 "no supervisor spawn subscription has this correlation id",
1536 )?])
1537 }
1538 }
1539 FrameType::Request => {
1540 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1543 if op
1544 .as_ref()
1545 .is_some_and(|probe| probe.op == "operator.confirm")
1546 {
1547 return self.handle_operator_confirm(ctx, frame);
1548 }
1549 if self
1550 .forwarding
1551 .module_endpoint_for_connection(ctx.connection_id)
1552 .map_err(RouterError::Forwarding)?
1553 .is_some()
1554 {
1555 if !is_known_module_request_op(&frame.body) {
1556 return Ok(vec![control_error_frame(
1557 &frame,
1558 "unsupported_control_frame",
1559 "module-originated channel-0 REQUEST is not supported",
1560 )?]);
1561 }
1562 let request = match parse_module_control_request_from_module(&frame.body) {
1563 Ok(request) => request,
1564 Err((err, ControlRequestBodyError::UnknownOp)) => {
1565 return Ok(vec![control_error_frame(
1566 &frame,
1567 "unsupported_control_frame",
1568 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1569 )?])
1570 }
1571 Err((err, ControlRequestBodyError::InvalidBody)) => {
1572 return Ok(vec![control_error_frame(
1573 &frame,
1574 "invalid_control_body",
1575 format!("malformed module control body: {err}"),
1576 )?])
1577 }
1578 };
1579 let op = module_control_request_op(&request);
1580 let corr = frame.header.corr;
1581 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1582 let result =
1583 self.handle_module_control_request(ctx.connection_id, frame, request);
1584 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1585 return result;
1586 }
1587
1588 if is_known_module_request_op(&frame.body) {
1589 return Ok(vec![control_error_frame(
1590 &frame,
1591 "not_registered",
1592 "catalog.update requires an active module registration owned by this connection",
1593 )?]);
1594 }
1595
1596 let request = match parse_client_control_request(&frame.body) {
1597 Ok(request) => request,
1598 Err((err, ControlRequestBodyError::UnknownOp)) => {
1599 return Ok(vec![control_error_frame(
1600 &frame,
1601 "unknown_control_op",
1602 format!("unknown client control op: {err}"),
1603 )?])
1604 }
1605 Err((err, ControlRequestBodyError::InvalidBody)) => {
1606 return Ok(vec![control_error_frame(
1607 &frame,
1608 "invalid_control_body",
1609 format!("malformed client control body: {err}"),
1610 )?])
1611 }
1612 };
1613 let op = client_control_request_op(&request);
1614 let corr = frame.header.corr;
1615 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1616 #[cfg(test)]
1617 if let Some(delay) = self.control_dispatch_delay {
1618 tokio::time::sleep(delay).await;
1619 }
1620 let result = self
1621 .handle_client_control_request(ctx, frame, request)
1622 .await;
1623 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1624 result
1625 }
1626 FrameType::Push => {
1627 let Some(endpoint) = self
1628 .forwarding
1629 .module_endpoint_for_connection(ctx.connection_id)
1630 .map_err(RouterError::Forwarding)?
1631 else {
1632 return Ok(vec![control_error_frame(
1633 &frame,
1634 "unsupported_control_frame",
1635 "client-originated channel-0 PUSH is not supported",
1636 )?]);
1637 };
1638 self.handle_status_update(endpoint, frame)
1639 }
1640 FrameType::Response | FrameType::Error
1641 if self
1642 .forwarding
1643 .module_endpoint_for_connection(ctx.connection_id)
1644 .map_err(RouterError::Forwarding)?
1645 .is_some() =>
1646 {
1647 self.handle_module_relay_response(ctx.connection_id, frame)
1648 }
1649 ty => Ok(vec![control_error_frame(
1650 &frame,
1651 "unsupported_control_frame",
1652 format!("unsupported channel-0 frame {ty:?}"),
1653 )?]),
1654 }
1655 }
1656
1657 pub fn cleanup_connection(
1658 &self,
1659 connection_id: ConnectionId,
1660 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1661 self.cleanup_connection_with_end_reason(
1662 connection_id,
1663 RegistrationEndReason::ConnectionClosed,
1664 )
1665 }
1666
1667 fn cleanup_connection_with_end_reason(
1668 &self,
1669 connection_id: ConnectionId,
1670 requested_reason: RegistrationEndReason,
1671 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1672 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1673 self.registry
1674 .get_module_by_connection(connection_id)?
1675 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1676 .and_then(|module| module.registration_end_reason().ok().flatten())
1677 .unwrap_or(requested_reason)
1678 } else {
1679 requested_reason
1680 };
1681 let crash_closed = self
1682 .registry
1683 .get_module_by_connection(connection_id)?
1684 .and_then(|registration| {
1685 self.forwarding
1686 .module_endpoint_for_connection(connection_id)
1687 .ok()
1688 .flatten()
1689 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1690 .map(|routes| (registration.manifest.module_id, routes))
1691 });
1692 let crash_closed = crash_closed.map(|(module_id, routes)| {
1693 let terminal = match self.supervisor.get(&module_id) {
1694 None => false,
1695 Some(module) => match module.will_recover_after_connection_loss() {
1696 Ok(will_recover) => !will_recover,
1697 Err(err) => {
1698 warn!(
1699 %module_id,
1700 error = %err,
1701 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1702 );
1703 false
1704 }
1705 },
1706 };
1707 let reason = match self.forwarding.is_daemon_draining() {
1711 Ok(true) => RouteCloseReason::Restart,
1712 Ok(false) => RouteCloseReason::Crash,
1713 Err(err) => {
1714 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1715 RouteCloseReason::Crash
1716 }
1717 };
1718 (module_id, routes, reason, terminal)
1719 });
1720 let registrations = self.deregister_connection(connection_id, end_reason);
1721 let cleanup = if crash_closed.is_some() {
1722 self.forwarding.cleanup_connection_counted(connection_id)
1723 } else {
1724 self.forwarding
1730 .cleanup_connection(connection_id)
1731 .map(|released| crate::forwarding::ConnectionCleanup {
1732 released,
1733 abandoned_relays: 0,
1734 })
1735 };
1736 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1741 let abandoned = cleanup
1742 .as_ref()
1743 .map_or(0, |cleanup| cleanup.abandoned_relays);
1744 send_route_control_pushes(
1745 &self.forwarding,
1746 routes,
1747 ClientControlPush::RouteClosed {
1748 module_id,
1749 channels: Vec::new(),
1750 reason,
1751 drained: false,
1752 abandoned,
1753 excluded_subscriptions: 0,
1754 terminal: Some(terminal),
1755 },
1756 );
1757 }
1758 if let Ok(cleanup) = cleanup {
1759 self.emit_route_goodbyes(cleanup.released);
1760 }
1761 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1765 crate::supervise::notify_registration_release();
1766 self.capability_evaluator.wake_deadline_loop();
1767 self.refresh_capability_requirements();
1768 }
1769 self.supervisor.remove_spawn_subscribers(connection_id);
1770 self.hello_launch_nonces
1773 .lock()
1774 .unwrap_or_else(|poisoned| poisoned.into_inner())
1775 .forget(connection_id);
1776 self.scopes
1777 .write()
1778 .unwrap_or_else(|poisoned| poisoned.into_inner())
1779 .release_connection(connection_id);
1780 registrations
1781 }
1782
1783 pub(crate) fn handle_route_goodbye(
1784 &self,
1785 connection_id: ConnectionId,
1786 route_channel: u16,
1787 route_epoch: u32,
1788 ) -> Result<bool, RouterError> {
1789 debug!(
1790 connection_id = connection_id.get(),
1791 route_channel, route_epoch, "handling route GOODBYE"
1792 );
1793 let RouteRelease::Removed(released_route) = self
1794 .forwarding
1795 .release_client_route(connection_id, route_channel, route_epoch)
1796 .map_err(RouterError::Forwarding)?
1797 else {
1798 return Ok(false);
1799 };
1800 self.emit_route_goodbyes(vec![released_route]);
1801 Ok(true)
1802 }
1803
1804 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1805 for released in released_routes {
1806 let frame = match Frame::build_with_version(
1807 released.negotiated_ver,
1808 FrameType::Goodbye,
1809 control_flags(),
1810 released.channel,
1811 released.epoch,
1812 0,
1813 Vec::new(),
1814 ) {
1815 Ok(frame) => frame,
1816 Err(err) => {
1817 warn!(
1818 route_channel = released.channel,
1819 error = %err,
1820 "failed to build route GOODBYE frame"
1821 );
1822 continue;
1823 }
1824 };
1825 if !released.close_on_delivery_failure() {
1826 crate::forwarding::send_module_route_goodbye(
1827 &self.counters,
1828 &released.sink,
1829 frame,
1830 released.module_id.as_deref(),
1831 "client route released",
1832 );
1833 continue;
1834 }
1835 if let Err(err) = released.sink.try_send(frame) {
1836 warn!(
1837 target_connection_id = released.connection_id.get(),
1838 route_channel = released.channel,
1839 error = %err,
1840 "route GOODBYE was not delivered to client; closing target connection"
1841 );
1842 if self
1843 .forwarding
1844 .escalate_client_delivery_failure(
1845 released.connection_id,
1846 released.channel,
1847 released.epoch,
1848 CloseReason::new(
1849 "route_goodbye_delivery_failed",
1850 format!(
1851 "failed to enqueue route GOODBYE for channel {}: {err}",
1852 released.channel
1853 ),
1854 ),
1855 crate::forwarding::UndeliveredFrame {
1856 module_id: released.module_id.as_deref(),
1857 sink: &released.sink,
1858 },
1859 )
1860 .unwrap_or(false)
1861 {
1862 self.counters.increment_goodbye_relay_client_failed();
1863 }
1864 }
1865 }
1866 }
1867
1868 fn send_abandoned_route_bind_goodbye(
1880 &self,
1881 module_sink: &crate::FrameSink,
1882 negotiated_ver: u8,
1883 module_channel: u16,
1884 module_epoch: u32,
1885 ) {
1886 let frame = match Frame::build_with_version(
1887 negotiated_ver,
1888 FrameType::Goodbye,
1889 control_flags(),
1890 module_channel,
1891 module_epoch,
1892 0,
1893 Vec::new(),
1894 ) {
1895 Ok(frame) => frame,
1896 Err(err) => {
1897 warn!(
1898 route_channel = module_channel,
1899 error = %err,
1900 "failed to build GOODBYE for abandoned route.bind"
1901 );
1902 return;
1903 }
1904 };
1905 crate::forwarding::send_module_route_goodbye(
1906 &self.counters,
1907 module_sink,
1908 frame,
1909 None,
1910 "abandoned route.bind",
1911 );
1912 }
1913
1914 fn handle_hello(
1915 &self,
1916 connection_id: ConnectionId,
1917 sink: Option<crate::FrameSink>,
1918 frame: Frame,
1919 ) -> Result<Vec<Frame>, RouterError> {
1920 debug!(
1921 connection_id = connection_id.get(),
1922 corr = frame.header.corr,
1923 "handling HELLO"
1924 );
1925 if self
1929 .registry
1930 .get_module_by_connection(connection_id)
1931 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
1932 .is_some()
1933 {
1934 return Ok(vec![control_error_frame(
1935 &frame,
1936 "invalid_hello",
1937 "connection is already registered as a module",
1938 )?]);
1939 }
1940 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1941 Ok(value) => value,
1942 Err(err) => {
1943 return Ok(vec![control_error_frame(
1944 &frame,
1945 "invalid_hello",
1946 format!("malformed HELLO body: {err}"),
1947 )?])
1948 }
1949 };
1950 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1951 return Ok(vec![control_error_frame(
1952 &frame,
1953 "invalid_capability_grammar",
1954 err.to_string(),
1955 )?]);
1956 }
1957 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1958 return Ok(vec![control_error_frame(
1959 &frame,
1960 "invalid_manifest",
1961 err.to_string(),
1962 )?]);
1963 }
1964 if let Err(err) = validate_hello_event_declarations(&hello_value) {
1965 return Ok(vec![control_error_frame(
1966 &frame,
1967 "invalid_event_declaration",
1968 err.to_string(),
1969 )?]);
1970 }
1971 if let Some(provenance) = hello_value
1972 .get("manifest")
1973 .and_then(|manifest| manifest.get("provenance"))
1974 {
1975 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
1976 return Ok(vec![control_error_frame(
1977 &frame,
1978 "invalid_manifest",
1979 format!("malformed manifest provenance: {err}"),
1980 )?]);
1981 }
1982 }
1983 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
1984 Ok(hello) => hello,
1985 Err(err) => {
1986 return Ok(vec![control_error_frame(
1987 &frame,
1988 "invalid_hello",
1989 format!("malformed HELLO body: {err}"),
1990 )?])
1991 }
1992 };
1993
1994 if hello.protocol_ver != hello.manifest.protocol_ver {
1995 return Ok(vec![control_error_frame(
1996 &frame,
1997 "invalid_manifest",
1998 format!(
1999 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2000 hello.protocol_ver, hello.manifest.protocol_ver
2001 ),
2002 )?]);
2003 }
2004
2005 if hello.manifest.module_id.trim().is_empty() {
2006 return Ok(vec![control_error_frame(
2007 &frame,
2008 "invalid_manifest",
2009 "manifest module_id must not be empty",
2010 )?]);
2011 }
2012
2013 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2014 Ok(negotiated_ver) => negotiated_ver,
2015 Err(message) => {
2016 return Ok(vec![control_error_frame(
2017 &frame,
2018 "version_unsupported",
2019 message,
2020 )?])
2021 }
2022 };
2023
2024 let swap_admission = self
2032 .supervisor
2033 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2034 if swap_admission == SwapHelloAdmission::Refused {
2035 warn!(
2036 module_id = %hello.manifest.module_id,
2037 connection_id = connection_id.get(),
2038 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2039 );
2040 return Ok(vec![control_error_frame(
2041 &frame,
2042 "swap_token_invalid",
2043 format!(
2044 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2045 hello.manifest.module_id
2046 ),
2047 )?]);
2048 }
2049 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2050
2051 if let Some(rejection) = (!swap_candidate)
2059 .then(|| {
2060 self.supervisor.reserved_hello_rejection(
2061 &hello.manifest.module_id,
2062 hello.launch_nonce.as_deref(),
2063 )
2064 })
2065 .flatten()
2066 {
2067 let message = match rejection {
2068 ReservedHelloRejection::Exact { module_id } => format!(
2069 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2070 ),
2071 ReservedHelloRejection::Prefix {
2072 prefix,
2073 owner_module_id,
2074 } => format!(
2075 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2076 hello.manifest.module_id
2077 ),
2078 };
2079 return Ok(vec![control_error_frame(
2080 &frame,
2081 "reserved_module",
2082 message,
2083 )?]);
2084 }
2085
2086 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2087 &hello.manifest.module_id,
2088 hello.manifest.capabilities.as_ref(),
2089 );
2090 if let Some(refusal) = reserved_capability_refusals.first() {
2091 let capability = refusal.capability.clone();
2092 let bound_module = refusal.claimants[0].clone();
2093 log_duplicate_claim_events(reserved_capability_refusals);
2094 return Ok(vec![control_error_frame(
2095 &frame,
2096 "reserved_capability",
2097 format!(
2098 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2099 capability, bound_module, hello.manifest.module_id
2100 ),
2101 )?]);
2102 }
2103
2104 if self
2107 .forwarding
2108 .connection_has_client_routes(connection_id)
2109 .map_err(RouterError::Forwarding)?
2110 {
2111 return Ok(vec![control_error_frame(
2112 &frame,
2113 "invalid_hello",
2114 "connection has open client routes and cannot also register as a module",
2115 )?]);
2116 }
2117
2118 self.hello_launch_nonces
2123 .lock()
2124 .unwrap_or_else(|poisoned| poisoned.into_inner())
2125 .record(connection_id, hello.launch_nonce.as_deref());
2126 let control_ops = effective_module_control_ops(hello.control_ops);
2127 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2130 if swap_candidate {
2131 return self.register_swap_candidate(
2132 connection_id,
2133 sink,
2134 &frame,
2135 hello.manifest,
2136 negotiated_ver,
2137 control_ops,
2138 hello_ack,
2139 );
2140 }
2141 let registration = match self.registry.register_with_control_ops(
2142 hello.manifest,
2143 negotiated_ver,
2144 connection_id,
2145 control_ops,
2146 ) {
2147 Ok(registration) => registration,
2148 Err(RegistryError::DuplicateModuleId { module_id }) => {
2149 return Ok(vec![control_error_frame(
2150 &frame,
2151 "duplicate_module_id",
2152 format!(
2153 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2154 ),
2155 )?])
2156 }
2157 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2158 return Ok(vec![control_error_frame(
2159 &frame,
2160 "invalid_module_id",
2161 err.to_string(),
2162 )?])
2163 }
2164 Err(err) => {
2165 return Ok(vec![control_error_frame(
2166 &frame,
2167 "registry_error",
2168 err.to_string(),
2169 )?])
2170 }
2171 };
2172
2173 let reply = if let Some(sink) = sink {
2174 let concurrency = manifest_concurrency(®istration.manifest);
2190 if let Err(err) = self.forwarding.register_module_connection_acked(
2191 connection_id,
2192 registration.manifest.module_id.clone(),
2193 negotiated_ver,
2194 concurrency,
2195 sink,
2196 hello_ack,
2197 ) {
2198 if matches!(
2202 self.deregister_connection(
2203 connection_id,
2204 RegistrationEndReason::RegistrationFailed,
2205 ),
2206 Ok(r) if !r.is_empty()
2207 ) {
2208 crate::supervise::notify_registration_release();
2209 }
2210 return Ok(vec![control_error_frame(
2211 &frame,
2212 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2213 "invalid_hello"
2214 } else {
2215 forwarding_error_code(&err)
2216 },
2217 err.to_string(),
2218 )?]);
2219 }
2220 Vec::new()
2221 } else {
2222 vec![hello_ack]
2225 };
2226
2227 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2236 info!(
2237 module_id = %registration.manifest.module_id,
2238 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2239 );
2240 }
2241
2242 self.apply_registration_capabilities(®istration);
2243
2244 info!(
2245 module_id = %registration.manifest.module_id,
2246 module_version = %registration.manifest.module_version,
2247 negotiated_ver,
2248 routable_provider = manifest_provides_routable_role(®istration.manifest),
2249 connection_id = connection_id.get(),
2250 "module registered"
2251 );
2252
2253 Ok(reply)
2254 }
2255
2256 #[allow(clippy::too_many_arguments)]
2266 fn register_swap_candidate(
2267 &self,
2268 connection_id: ConnectionId,
2269 sink: Option<crate::FrameSink>,
2270 frame: &Frame,
2271 manifest: ModuleManifest,
2272 negotiated_ver: u8,
2273 control_ops: Vec<String>,
2274 hello_ack: Frame,
2275 ) -> Result<Vec<Frame>, RouterError> {
2276 let module_id = manifest.module_id.clone();
2277 let registration = match self.registry.register_candidate_with_control_ops(
2278 manifest,
2279 negotiated_ver,
2280 connection_id,
2281 control_ops,
2282 ) {
2283 Ok(registration) => registration,
2284 Err(RegistryError::DuplicateModuleId { module_id }) => {
2285 return Ok(vec![control_error_frame(
2286 frame,
2287 "duplicate_module_id",
2288 format!(
2289 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2290 ),
2291 )?])
2292 }
2293 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2294 return Ok(vec![control_error_frame(
2295 frame,
2296 "invalid_module_id",
2297 err.to_string(),
2298 )?])
2299 }
2300 Err(err) => {
2301 return Ok(vec![control_error_frame(
2302 frame,
2303 "registry_error",
2304 err.to_string(),
2305 )?])
2306 }
2307 };
2308 let reply = if let Some(sink) = sink {
2309 let concurrency = manifest_concurrency(®istration.manifest);
2313 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2314 connection_id,
2315 module_id.clone(),
2316 negotiated_ver,
2317 concurrency,
2318 sink,
2319 hello_ack,
2320 ) {
2321 if matches!(
2322 self.deregister_connection(
2323 connection_id,
2324 RegistrationEndReason::RegistrationFailed,
2325 ),
2326 Ok(r) if !r.is_empty()
2327 ) {
2328 crate::supervise::notify_registration_release();
2329 }
2330 return Ok(vec![control_error_frame(
2331 frame,
2332 forwarding_error_code(&err),
2333 err.to_string(),
2334 )?]);
2335 }
2336 Vec::new()
2337 } else {
2338 vec![hello_ack]
2339 };
2340 self.supervisor.mark_swap_candidate_admitted(&module_id);
2341 info!(
2342 module_id = %module_id,
2343 module_version = %registration.manifest.module_version,
2344 negotiated_ver,
2345 ready = registration.ready,
2346 connection_id = connection_id.get(),
2347 "swap candidate registered; not routable until cutover"
2348 );
2349 Ok(reply)
2350 }
2351
2352 fn build_hello_ack(
2353 &self,
2354 frame: &Frame,
2355 negotiated_ver: u8,
2356 module_id: &str,
2357 ) -> Result<Frame, RouterError> {
2358 let ack = ModuleHelloAckBody {
2359 negotiated_ver,
2360 subc_ops: module_subc_ops(),
2361 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2362 storage: self
2363 .storage_config
2364 .as_ref()
2365 .map(|cfg| cfg.descriptor_for(module_id)),
2366 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2367 };
2368 let body = serde_json::to_vec(&ack).map_err(|err| {
2369 RouterError::backend(
2370 0,
2371 frame.header.corr,
2372 format!("failed to encode HELLO_ACK: {err}"),
2373 )
2374 })?;
2375
2376 Frame::build_with_version(
2377 negotiated_ver,
2378 FrameType::HelloAck,
2379 control_flags(),
2380 0,
2381 0,
2382 frame.header.corr,
2383 body,
2384 )
2385 .map_err(RouterError::FrameBuild)
2386 }
2387
2388 async fn handle_client_control_request(
2389 &self,
2390 ctx: &RouteCtx,
2391 frame: Frame,
2392 request: ClientControlRequest,
2393 ) -> Result<Vec<Frame>, RouterError> {
2394 match request {
2395 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2396 ClientControlRequest::CatalogList { module_id } => {
2397 self.handle_catalog_list(frame, module_id)
2398 }
2399 ClientControlRequest::RouteOpen {
2400 target,
2401 identity,
2402 consumer_identity,
2403 consumer_capabilities,
2404 role_versions,
2405 admission_facts,
2406 scope,
2407 } => {
2408 self.handle_route_open(
2409 ctx,
2410 frame,
2411 RouteOpenRequest {
2412 target,
2413 identity,
2414 consumer_identity,
2415 consumer_capabilities,
2416 role_versions,
2417 admission_facts,
2418 scope,
2419 },
2420 )
2421 .await
2422 }
2423 ClientControlRequest::RoutePoll {
2424 route_channel,
2425 route_epoch,
2426 kind,
2427 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2428 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2429 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2430 self.handle_supervisor_spawn_snapshot(frame)
2431 }
2432 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2433 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2434 }
2435 ClientControlRequest::SupervisorRestart {
2436 module_id,
2437 drain_timeout_ms,
2438 } => {
2439 self.log_supervisor_request_received(
2440 ctx,
2441 frame.header.corr,
2442 ops::SUPERVISOR_RESTART,
2443 Some(&module_id),
2444 None,
2445 )?;
2446 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2447 .await
2448 }
2449 ClientControlRequest::SupervisorSwap {
2450 module_id,
2451 ready_timeout_ms,
2452 } => {
2453 self.log_supervisor_request_received(
2454 ctx,
2455 frame.header.corr,
2456 ops::SUPERVISOR_SWAP,
2457 Some(&module_id),
2458 None,
2459 )?;
2460 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2461 .await
2462 }
2463 ClientControlRequest::SupervisorReload { module_id } => {
2464 self.log_supervisor_request_received(
2465 ctx,
2466 frame.header.corr,
2467 ops::SUPERVISOR_RELOAD,
2468 Some(&module_id),
2469 None,
2470 )?;
2471 self.handle_supervisor_reload(frame, module_id).await
2472 }
2473 ClientControlRequest::SupervisorRescan { preview } => {
2474 if !preview {
2475 self.log_supervisor_request_received(
2476 ctx,
2477 frame.header.corr,
2478 ops::SUPERVISOR_RESCAN,
2479 None,
2480 None,
2481 )?;
2482 }
2483 self.handle_supervisor_rescan(frame, preview).await
2484 }
2485 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2486 self.log_supervisor_request_received(
2487 ctx,
2488 frame.header.corr,
2489 ops::SUPERVISOR_RELEASE_RESERVED,
2490 Some(&module_id),
2491 None,
2492 )?;
2493 self.handle_supervisor_release_reserved(frame, module_id)
2494 .await
2495 }
2496 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2497 self.log_supervisor_request_received(
2498 ctx,
2499 frame.header.corr,
2500 ops::SUPERVISOR_SET_ENABLED,
2501 Some(&module_id),
2502 Some(enabled),
2503 )?;
2504 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2505 .await
2506 }
2507 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2508 self.handle_supervisor_health_probe(frame, module_id).await
2509 }
2510 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2511 ClientControlRequest::SupervisorRoutes { module_id } => {
2512 self.handle_supervisor_routes(frame, module_id)
2513 }
2514 ClientControlRequest::SupervisorProvenance { module_id } => {
2515 self.handle_supervisor_provenance(frame, module_id).await
2516 }
2517 ClientControlRequest::SupervisorStderrTail {
2518 module_id,
2519 max_lines,
2520 max_bytes,
2521 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2522 ClientControlRequest::SupervisorTerminals { module_id } => {
2523 self.handle_supervisor_terminals(frame, module_id).await
2524 }
2525 }
2526 }
2527
2528 fn log_supervisor_request_received(
2529 &self,
2530 ctx: &RouteCtx,
2531 corr: u64,
2532 op: &'static str,
2533 module_id: Option<&str>,
2534 enabled: Option<bool>,
2535 ) -> Result<(), RouterError> {
2536 let caller = self
2537 .registry
2538 .get_module_by_connection(ctx.connection_id)
2539 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2540 .map(|registration| Principal::Reserved {
2541 module_id: registration.manifest.module_id,
2542 })
2543 .unwrap_or(Principal::Direct);
2544 let caller = principal_label(&caller);
2545 info!(
2548 target: "control",
2549 op,
2550 module_id,
2551 enabled,
2552 connection_id = ctx.connection_id.get(),
2553 caller = %caller,
2554 "supervisor request received"
2555 );
2556 Ok(())
2557 }
2558
2559 fn handle_module_control_request(
2560 &self,
2561 connection_id: ConnectionId,
2562 frame: Frame,
2563 request: ModuleControlRequestFromModule,
2564 ) -> Result<Vec<Frame>, RouterError> {
2565 match request {
2566 ModuleControlRequestFromModule::CatalogUpdate {
2567 provides,
2568 capabilities,
2569 ready,
2570 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2571 ModuleControlRequestFromModule::LiveRoots {} => {
2572 let registered = self
2573 .registry
2574 .get_module_by_connection(connection_id)
2575 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2576 let Some(registration) = registered else {
2577 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2578 };
2579 let response = self
2580 .forwarding
2581 .live_roots(®istration.manifest.module_id)
2582 .map_err(RouterError::Forwarding)?;
2583 Ok(vec![control_response_body_frame(
2584 &frame,
2585 &response,
2586 "ModuleControlResponseToModule::LiveRoots",
2587 )?])
2588 }
2589 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2590 self.handle_scope_sync(connection_id, frame, generation, scopes)
2591 }
2592 ModuleControlRequestFromModule::ScopeApply {
2593 generation,
2594 upsert,
2595 end,
2596 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2597 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2598 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2599 }
2600 }
2601 }
2602
2603 fn handle_operator_confirm(
2604 &self,
2605 ctx: &RouteCtx,
2606 frame: Frame,
2607 ) -> Result<Vec<Frame>, RouterError> {
2608 use crate::operator_confirm::{audit, Outcome};
2609 let registration = self
2610 .registry
2611 .get_module_by_connection(ctx.connection_id)
2612 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2613 let Some(registration) = registration else {
2614 let outcome = Outcome::refusal("not_registered");
2615 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2616 return Ok(vec![outcome.frame(&frame)]);
2617 };
2618 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2619 Ok(request) => request,
2620 Err(_) => {
2621 let outcome = Outcome::refusal("invalid_control_body");
2622 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2623 return Ok(vec![outcome.frame(&frame)]);
2624 }
2625 };
2626 let module_id = registration.manifest.module_id;
2627 let nonce = self
2631 .hello_launch_nonces
2632 .lock()
2633 .unwrap_or_else(|p| p.into_inner())
2634 .nonce(ctx.connection_id)
2635 .map(str::to_owned);
2636 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2637 self.supervisor
2638 .spawned_consumer_authorized(&module_id, nonce)
2639 });
2640 let confirms = self.forwarding.operator_confirms();
2641 self.forwarding
2642 .with_operator_route(
2643 ctx.connection_id,
2644 request.route_channel,
2645 request.route_epoch,
2646 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
2647 )
2648 .map_err(RouterError::Forwarding)
2649 }
2650
2651 fn handle_scope_sync(
2655 &self,
2656 connection_id: ConnectionId,
2657 frame: Frame,
2658 generation: u64,
2659 scopes: Vec<ScopeRecord>,
2660 ) -> Result<Vec<Frame>, RouterError> {
2661 self.handle_scope_change(connection_id, frame, generation, scopes, None)
2662 }
2663
2664 fn handle_scope_change(
2665 &self,
2666 connection_id: ConnectionId,
2667 frame: Frame,
2668 generation: u64,
2669 scopes: Vec<ScopeRecord>,
2670 end: Option<Vec<ScopeEnd>>,
2671 ) -> Result<Vec<Frame>, RouterError> {
2672 let is_apply = end.is_some();
2673 let Some(registration) = self
2674 .registry
2675 .get_module_by_connection(connection_id)
2676 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2677 else {
2678 return Ok(vec![control_error_frame(
2679 &frame,
2680 "not_registered",
2681 "scope.sync and scope.apply require an active module registration owned by this connection",
2682 )?]);
2683 };
2684 let owner = registration.manifest.module_id;
2685 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2686 let is_current_launch = |connection: ConnectionId| {
2687 self.hello_launch_nonces
2688 .lock()
2689 .unwrap_or_else(|poisoned| poisoned.into_inner())
2690 .presented(connection, current_nonce.as_deref())
2691 };
2692 let mut table = self
2697 .scopes
2698 .write()
2699 .unwrap_or_else(|poisoned| poisoned.into_inner());
2700 let outcome = match end {
2701 Some(end) => table.apply(
2702 &owner,
2703 connection_id,
2704 is_current_launch,
2705 generation,
2706 scopes,
2707 end,
2708 ),
2709 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
2710 };
2711 let drained = match &outcome {
2712 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
2713 Err(_) => Vec::new(),
2714 };
2715 let has_deadlines = outcome.is_ok() && table.has_deadlines();
2716 drop(table);
2717 if has_deadlines {
2718 self.scope_deadline_added.notify_one();
2719 }
2720 match outcome {
2721 Ok(applied) => {
2722 let counts = ScopeOutcomeCounts::of(&applied.results);
2723 info!(
2724 owner = %owner,
2725 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2726 generation,
2727 records = applied.results.len(),
2728 created = counts.created,
2729 replaced = counts.replaced,
2730 updated = counts.updated,
2731 unchanged = counts.unchanged,
2732 refused = counts.refused,
2733 ended = applied.ended.len(),
2734 tag_changes = applied.tag_changes.len(),
2735 routes_closed = drained.len(),
2736 "scope change accepted"
2737 );
2738 for refused in applied
2744 .results
2745 .iter()
2746 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2747 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2748 {
2749 warn!(
2750 owner = %owner,
2751 generation,
2752 scope_ref = %refused.scope_ref,
2753 scope_epoch = refused.scope_epoch,
2754 code = refused.code.as_deref().unwrap_or(""),
2755 "scope record refused"
2756 );
2757 }
2758 self.close_scope_drained_routes(drained);
2759 let response = if is_apply {
2760 ModuleControlResponseToModule::ScopeApply {
2761 generation,
2762 results: applied.results,
2763 end_results: applied.end_results,
2764 ended: applied.ended,
2765 }
2766 } else {
2767 ModuleControlResponseToModule::ScopeSync {
2768 generation,
2769 results: applied.results,
2770 ended: applied.ended,
2771 }
2772 };
2773 Ok(vec![control_response_body_frame(
2774 &frame,
2775 &response,
2776 "scope change response",
2777 )?])
2778 }
2779 Err(refusal) => {
2780 info!(
2781 owner = %owner,
2782 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2783 generation,
2784 code = refusal.code,
2785 "scope change refused"
2786 );
2787 Ok(vec![control_error_frame(
2788 &frame,
2789 refusal.code,
2790 refusal.message,
2791 )?])
2792 }
2793 }
2794 }
2795
2796 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2802 if drained.is_empty() {
2803 return;
2804 }
2805 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2806 BTreeMap::new();
2807 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2808 for route in drained {
2809 warn!(
2810 module_id = %route.module_id,
2811 reason = ?route.reason,
2812 client_connection_id = route.client.connection_id.get(),
2813 route_channel = route.client.channel,
2814 "closing route because its scope changed"
2815 );
2816 pushes
2817 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2818 .or_insert_with(|| (route.reason, Vec::new()))
2819 .1
2820 .push(EndpointRoute {
2821 goodbye_target: route.client.clone(),
2822 principal: Principal::Unverified,
2823 bound_at: Instant::now(),
2824 draining: false,
2825 drain_reason: None,
2826 });
2827 goodbyes.push(route.module);
2828 goodbyes.push(route.client);
2829 }
2830 for ((module_id, _), (reason, routes)) in pushes {
2831 send_route_control_pushes(
2832 &self.forwarding,
2833 routes,
2834 ClientControlPush::RouteClosed {
2835 module_id,
2836 channels: Vec::new(),
2837 reason,
2838 drained: false,
2839 abandoned: 0,
2840 excluded_subscriptions: 0,
2841 terminal: Some(false),
2842 },
2843 );
2844 }
2845 self.emit_route_goodbyes(goodbyes);
2846 }
2847
2848 fn handle_scope_describe(
2851 &self,
2852 connection_id: ConnectionId,
2853 frame: Frame,
2854 owner: Principal,
2855 scope_ref: String,
2856 ) -> Result<Vec<Frame>, RouterError> {
2857 let registered = self
2858 .registry
2859 .get_module_by_connection(connection_id)
2860 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2861 if registered.is_none() {
2862 return Ok(vec![control_error_frame(
2863 &frame,
2864 "not_registered",
2865 "scope.describe requires an active module registration owned by this connection",
2866 )?]);
2867 }
2868 let description = self
2869 .scopes
2870 .read()
2871 .unwrap_or_else(|poisoned| poisoned.into_inner())
2872 .describe(&owner, &scope_ref);
2873 let owner_configured = match &owner {
2874 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
2879 _ => false,
2880 };
2881 let response = ModuleControlResponseToModule::ScopeDescribe {
2882 status: description.status,
2883 scope_epoch: description.scope_epoch,
2884 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2885 owner_synced: description.owner_synced,
2886 owner_configured,
2887 scope: description.stamp,
2888 };
2889 Ok(vec![control_response_body_frame(
2890 &frame,
2891 &response,
2892 "ModuleControlResponseToModule::ScopeDescribe",
2893 )?])
2894 }
2895
2896 fn handle_catalog_update(
2897 &self,
2898 connection_id: ConnectionId,
2899 frame: Frame,
2900 provides: Vec<ProviderRole>,
2901 capabilities: Option<CapabilityDeclarations>,
2902 ready: Option<bool>,
2903 ) -> Result<Vec<Frame>, RouterError> {
2904 self.refresh_capability_requirements();
2905 let Some(registration) = self
2906 .registry
2907 .get_module_by_connection(connection_id)
2908 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2909 else {
2910 return Ok(vec![control_error_frame(
2911 &frame,
2912 "not_registered",
2913 "catalog.update requires an active module registration owned by this connection",
2914 )?]);
2915 };
2916
2917 if let Some(message) =
2918 catalog_update_frozen_field_message(®istration.manifest, &provides)
2919 {
2920 return Ok(vec![control_error_frame(
2921 &frame,
2922 "catalog_update_frozen_field",
2923 message,
2924 )?]);
2925 }
2926
2927 let mut candidate = registration.manifest.clone();
2928 candidate.provides = provides.clone();
2929 candidate.capabilities = capabilities
2930 .clone()
2931 .or_else(|| registration.manifest.capabilities.clone());
2932 if let Err(err) = candidate.validate_capability_grammar() {
2933 return Ok(vec![control_error_frame(
2934 &frame,
2935 "invalid_capability_grammar",
2936 err.to_string(),
2937 )?]);
2938 }
2939
2940 let mut conflicts = self
2943 .capability_evaluator
2944 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
2945 if let Some(conflict) = conflicts.first() {
2946 let message = format!(
2947 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2948 conflict.capability, conflict.claimants[0], candidate.module_id
2949 );
2950 for conflict in &mut conflicts {
2951 conflict.source = DuplicateClaimSource::CatalogUpdate;
2952 }
2953 log_duplicate_claim_events(conflicts);
2954 return Ok(vec![control_error_frame(
2955 &frame,
2956 "reserved_capability",
2957 message,
2958 )?]);
2959 }
2960
2961 let updated = self
2962 .registry
2963 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2964 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2965 if updated.is_none() {
2966 return Ok(vec![control_error_frame(
2967 &frame,
2968 "not_registered",
2969 "catalog.update requires an active module registration owned by this connection",
2970 )?]);
2971 }
2972 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
2973 log_duplicate_claim_events(
2974 self.capability_evaluator
2975 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
2976 );
2977 }
2978 if capability_census_trigger(
2979 registration.manifest.capabilities.as_ref(),
2980 updated
2981 .as_ref()
2982 .and_then(|entry| entry.manifest.capabilities.as_ref()),
2983 ) {
2984 self.enforce_capability_denies();
2985 }
2986 self.refresh_capability_requirements();
2987
2988 let response = ModuleControlResponseToModule::CatalogUpdate {};
2989 control_response_body_frame(
2990 &frame,
2991 &response,
2992 "ModuleControlResponseToModule::CatalogUpdate",
2993 )
2994 .map(|frame| vec![frame])
2995 }
2996
2997 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
2998 self.refresh_capability_requirements();
2999 let mut counters = self.counters.snapshot();
3005 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3006 self.forwarding.client_route_concentration(),
3007 counters.as_object_mut(),
3008 ) {
3009 obj.insert(
3010 "client_connections_with_routes".into(),
3011 connections_with_routes.into(),
3012 );
3013 obj.insert("max_routes_on_one_connection".into(), max.into());
3014 }
3015 if let (Some(open_breakers), Some(obj)) = (
3021 self.route_bind_breakers.open_snapshot(),
3022 counters.as_object_mut(),
3023 ) {
3024 obj.insert("route_bind_breakers_open".into(), open_breakers);
3025 }
3026 let response = ClientControlResponse::ServerDescribe {
3027 protocol_ver: PROTOCOL_VERSION,
3028 subc_ops: subc_ops(),
3029 capabilities: self.subc_capabilities.as_ref().to_vec(),
3030 connected_clients: self.connected_clients.count(),
3031 counters: Some(counters),
3032 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3033 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3034 capability_requirements: self.capability_requirement_statuses(),
3035 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3036 };
3037 Ok(vec![control_response_body_frame(
3038 &frame,
3039 &response,
3040 "ClientControlResponse::ServerDescribe",
3041 )?])
3042 }
3043
3044 fn handle_catalog_list(
3045 &self,
3046 frame: Frame,
3047 module_id: Option<String>,
3048 ) -> Result<Vec<Frame>, RouterError> {
3049 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3050 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3051 })?;
3052 let entries = modules
3053 .into_iter()
3054 .filter(|registration| {
3055 module_id
3056 .as_deref()
3057 .map(|wanted| registration.manifest.module_id == wanted)
3058 .unwrap_or(true)
3059 })
3060 .map(|registration| {
3061 let not_ready = self.not_ready_reason(®istration);
3062 let roles = registration.manifest.provides;
3063 CatalogEntry::new(
3064 registration.manifest.module_id,
3065 roles,
3066 registration.control_ops,
3067 )
3068 .with_ready(not_ready.is_none())
3069 .with_not_ready(not_ready)
3070 .with_module_version(Some(registration.manifest.module_version))
3071 .with_capabilities(registration.manifest.capabilities)
3072 .with_self_signals(registration.manifest.self_signals)
3073 })
3074 .collect();
3075 let response = ClientControlResponse::CatalogList {
3076 generation,
3077 modules: entries,
3078 subc_ops: subc_ops(),
3079 };
3080 Ok(vec![control_response_body_frame(
3081 &frame,
3082 &response,
3083 "ClientControlResponse::CatalogList",
3084 )?])
3085 }
3086
3087 fn route_open_principal(
3088 &self,
3089 frame: &Frame,
3090 consumer_identity: Option<ConsumerIdentity>,
3091 ) -> Result<Result<Principal, Frame>, RouterError> {
3092 let Some(consumer_identity) = consumer_identity else {
3093 return Ok(Ok(Principal::Direct));
3094 };
3095
3096 if self.supervisor.spawned_consumer_authorized(
3097 &consumer_identity.module_id,
3098 &consumer_identity.launch_nonce,
3099 ) {
3100 return Ok(Ok(Principal::Reserved {
3101 module_id: consumer_identity.module_id,
3102 }));
3103 }
3104
3105 Ok(Err(control_error_frame(
3106 frame,
3107 "bad_consumer_identity",
3108 format!(
3109 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3110 consumer_identity.module_id
3111 ),
3112 )?))
3113 }
3114
3115 fn route_open_refusal_frame(
3122 &self,
3123 ctx: &RouteCtx,
3124 frame: &Frame,
3125 module_id: &str,
3126 reason: &'static str,
3127 code: &'static str,
3128 message: impl Into<String>,
3129 ) -> Result<Frame, RouterError> {
3130 self.observe_route_open_refusal(ctx, module_id, reason, code);
3131 control_error_frame(frame, code, message.into())
3132 }
3133
3134 fn route_open_breaker_refusal_frame(
3152 &self,
3153 ctx: &RouteCtx,
3154 frame: &Frame,
3155 module_id: &str,
3156 consecutive_timeouts: u32,
3157 retry_in: Duration,
3158 probe_in_flight: bool,
3159 ) -> Result<Frame, RouterError> {
3160 self.counters
3161 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3162 debug!(
3163 target: "control",
3164 code = "module_timeout",
3165 module_id = ?module_id,
3166 connection_id = ctx.connection_id.get(),
3167 consecutive_timeouts,
3168 retry_in_ms = retry_in.as_millis() as u64,
3169 probe_in_flight,
3170 "route.open refused by open bind-relay breaker"
3171 );
3172 let detail = if probe_in_flight {
3179 "one test route is already being tried; retry once it settles".to_string()
3180 } else {
3181 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3182 };
3183 control_error_frame(
3184 frame,
3185 "module_timeout",
3186 format!(
3187 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3188 timed out in a row); {detail}; its established routes are unaffected"
3189 ),
3190 )
3191 }
3192
3193 fn observe_route_open_refusal(
3206 &self,
3207 ctx: &RouteCtx,
3208 module_id: &str,
3209 reason: &'static str,
3210 code: &'static str,
3211 ) {
3212 self.counters.increment_route_open_refused(code);
3213 info!(
3214 target: "control",
3215 code,
3216 reason,
3217 module_id = ?module_id,
3218 connection_id = ctx.connection_id.get(),
3219 "route.open refused"
3220 );
3221 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3222 self.route_outages.record_not_serving(module_id, reason);
3223 }
3224 }
3225
3226 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3271 self.route_outages.record_accepted(module_id);
3272 self.counters.increment_route_open_accepted(principal);
3273 info!(
3274 target: "control",
3275 principal,
3276 module_id,
3277 connection_id = ctx.connection_id.get(),
3278 "route.open accepted"
3279 );
3280 }
3281
3282 fn supervised_absent_route_open_refusal_frame(
3283 &self,
3284 ctx: &RouteCtx,
3285 frame: &Frame,
3286 module_id: &str,
3287 code: &'static str,
3288 status: &crate::supervise::ModuleStatus,
3289 ) -> Result<Frame, RouterError> {
3290 self.counters.increment_route_open_refused(code);
3291 info!(
3292 target: "control",
3293 code,
3294 reason = "supervised_not_registered",
3295 module_id = ?module_id,
3296 connection_id = ctx.connection_id.get(),
3297 state = %status.state,
3298 enabled = status.enabled,
3299 live = status.live,
3300 "route.open refused"
3301 );
3302 self.route_outages
3306 .record_not_serving(module_id, "supervised_not_registered");
3307 control_error_frame(
3308 frame,
3309 code,
3310 format!(
3311 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3312 status.state, status.enabled, status.live
3313 ),
3314 )
3315 }
3316
3317 async fn handle_route_open(
3318 &self,
3319 ctx: &RouteCtx,
3320 frame: Frame,
3321 request: RouteOpenRequest,
3322 ) -> Result<Vec<Frame>, RouterError> {
3323 let RouteOpenRequest {
3324 target,
3325 mut identity,
3326 consumer_identity,
3327 consumer_capabilities,
3328 role_versions,
3329 admission_facts,
3330 scope,
3331 } = request;
3332 let target_module_id = target_module_id(&target).to_string();
3333 if self
3334 .registry
3335 .get_module_by_connection(ctx.connection_id)
3336 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3337 .is_some()
3338 {
3339 return Ok(vec![control_error_frame(
3340 &frame,
3341 "invalid_request",
3342 "module connections cannot open client routes",
3343 )?]);
3344 }
3345 debug!(
3346 connection_id = ctx.connection_id.get(),
3347 corr = frame.header.corr,
3348 module_id = %target_module_id,
3349 "handling route.open"
3350 );
3351
3352 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3358 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3359 self.observe_route_open_refusal(
3360 ctx,
3361 &target_module_id,
3362 "invalid_role_versions",
3363 error_codes::INVALID_REQUEST,
3364 );
3365 return Ok(vec![control_error_body_frame(
3366 &frame,
3367 ErrorBody {
3368 code: error_codes::INVALID_REQUEST.to_string(),
3369 message: error.to_string(),
3370 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3371 },
3372 )?]);
3373 }
3374
3375 let Some(registration) = self
3394 .registry
3395 .get_module(&target_module_id)
3396 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3397 else {
3398 if let Some((status, warming)) =
3399 self.supervisor_status(&target_module_id, frame.header.corr)?
3400 {
3401 if status.protocol == ModuleProtocol::None {
3408 return Ok(vec![self.route_open_refusal_frame(
3409 ctx,
3410 &frame,
3411 &target_module_id,
3412 "protocol_none",
3413 error_codes::MODULE_NO_PROTOCOL,
3414 format!(
3415 "module_id '{target_module_id}' is declared protocol: none; \
3416 it speaks no subc wire and serves no routes"
3417 ),
3418 )?]);
3419 }
3420 let code = if warming {
3421 "module_warming"
3422 } else {
3423 "target_unavailable"
3424 };
3425 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3426 ctx,
3427 &frame,
3428 &target_module_id,
3429 code,
3430 &status,
3431 )?]);
3432 }
3433 if let Some(removed_ago_ms) =
3434 self.supervisor.removal_tombstone_age_ms(&target_module_id)
3435 {
3436 return Ok(vec![self.route_open_refusal_frame(
3437 ctx,
3438 &frame,
3439 &target_module_id,
3440 "removed",
3441 error_codes::MODULE_REMOVED,
3442 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3443 )?]);
3444 }
3445 return Ok(vec![self.route_open_refusal_frame(
3446 ctx,
3447 &frame,
3448 &target_module_id,
3449 "not_registered",
3450 error_codes::UNKNOWN_MODULE,
3451 format!("module_id '{target_module_id}' is not registered"),
3452 )?]);
3453 };
3454
3455 if !registration.ready {
3459 self.counters
3460 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3461 info!(
3462 target: "control",
3463 code = error_codes::MODULE_WARMING,
3464 module_id = ?target_module_id,
3465 connection_id = ctx.connection_id.get(),
3466 reason = "declared_not_ready",
3467 "route.open refused"
3468 );
3469 self.route_outages
3472 .record_not_serving(&target_module_id, "declared_not_ready");
3473 return Ok(vec![control_error_body_frame(
3474 &frame,
3475 ErrorBody {
3476 code: error_codes::MODULE_WARMING.to_string(),
3477 message: format!(
3478 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3479 ),
3480 detail: Some(serde_json::json!({
3481 "reason": "declared_not_ready"
3482 })),
3483 },
3484 )?]);
3485 }
3486
3487 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3506 self.counters
3507 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3508 info!(
3509 target: "control",
3510 code = error_codes::MODULE_WARMING,
3511 module_id = ?target_module_id,
3512 connection_id = ctx.connection_id.get(),
3513 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3514 capability = %capability,
3515 "route.open refused"
3516 );
3517 return Ok(vec![control_error_body_frame(
3518 &frame,
3519 ErrorBody {
3520 code: error_codes::MODULE_WARMING.to_string(),
3521 message: format!(
3522 "module_id '{target_module_id}' requires capability '{capability}', \
3523 which no registered module provides; retry"
3524 ),
3525 detail: Some(serde_json::json!({
3526 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3527 "capability": capability,
3528 })),
3529 },
3530 )?]);
3531 }
3532
3533 if !target_has_required_role(&target, ®istration.manifest.provides) {
3534 return Ok(vec![self.route_open_refusal_frame(
3535 ctx,
3536 &frame,
3537 &target_module_id,
3538 "role_not_provided",
3539 "target_unavailable",
3540 format!("module_id '{target_module_id}' does not provide the requested target"),
3541 )?]);
3542 }
3543
3544 if registration.state != ChannelState::Active {
3545 return Ok(vec![self.route_open_refusal_frame(
3546 ctx,
3547 &frame,
3548 &target_module_id,
3549 "registration_not_active",
3550 "target_unavailable",
3551 format!("module_id '{target_module_id}' is not active"),
3552 )?]);
3553 }
3554
3555 if self
3556 .forwarding
3557 .module_is_draining(&target_module_id)
3558 .map_err(RouterError::Forwarding)?
3559 {
3560 return Ok(vec![self.route_open_refusal_frame(
3561 ctx,
3562 &frame,
3563 &target_module_id,
3564 "reloading",
3565 "module_reloading",
3566 format!("module_id '{target_module_id}' is reloading"),
3567 )?]);
3568 }
3569
3570 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3571 process_liveness.process_live(&target_module_id) == Some(false)
3572 }) {
3573 if process_liveness.process_replacing(&target_module_id) {
3580 return Ok(vec![self.route_open_refusal_frame(
3581 ctx,
3582 &frame,
3583 &target_module_id,
3584 "reloading",
3585 "module_reloading",
3586 format!("module_id '{target_module_id}' is reloading"),
3587 )?]);
3588 }
3589 return Ok(vec![self.route_open_refusal_frame(
3590 ctx,
3591 &frame,
3592 &target_module_id,
3593 "supervisor_not_live",
3594 "target_unavailable",
3595 format!("module_id '{target_module_id}' is not live"),
3596 )?]);
3597 }
3598
3599 if !self
3600 .forwarding
3601 .has_live_module_connection(&target_module_id)
3602 .map_err(RouterError::Forwarding)?
3603 {
3604 return Ok(vec![self.route_open_refusal_frame(
3605 ctx,
3606 &frame,
3607 &target_module_id,
3608 "no_forwarding_connection",
3609 "target_unavailable",
3610 format!("module_id '{target_module_id}' has no live forwarding connection"),
3611 )?]);
3612 }
3613
3614 if let Some(error) =
3615 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3616 {
3617 self.observe_route_open_refusal(
3618 ctx,
3619 &target_module_id,
3620 "op_not_allowed",
3621 "op_not_allowed",
3622 );
3623 return Ok(vec![error]);
3624 }
3625
3626 let principal = match self.route_open_principal(&frame, consumer_identity)? {
3627 Ok(principal) => principal,
3628 Err(error) => {
3629 self.observe_route_open_refusal(
3630 ctx,
3631 &target_module_id,
3632 "bad_consumer_identity",
3633 "bad_consumer_identity",
3634 );
3635 return Ok(vec![error]);
3636 }
3637 };
3638
3639 if let Principal::Reserved {
3643 module_id: opening_module_id,
3644 } = &principal
3645 {
3646 if let Some(opening_registration) = self
3647 .registry
3648 .get_module(opening_module_id)
3649 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3650 {
3651 if let Some(capability) =
3652 denied_capability(&opening_registration.manifest, ®istration.manifest)
3653 {
3654 warn!(
3655 opening_module_id,
3656 target_module_id,
3657 capability,
3658 "refusing route.open because an attested capability deny edge matches"
3659 );
3660 return Ok(vec![self.route_open_refusal_frame(
3661 ctx,
3662 &frame,
3663 &target_module_id,
3664 "capability_deny_edge",
3665 "capability_forbidden",
3666 format!(
3667 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3668 ),
3669 )?]);
3670 }
3671 }
3672 }
3673
3674 if admission_facts.is_some() {
3675 let carrier_matches = matches!(
3676 &principal,
3677 Principal::Reserved { module_id }
3678 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3679 );
3680 if !carrier_matches {
3681 return Ok(vec![self.route_open_refusal_frame(
3682 ctx,
3683 &frame,
3684 &target_module_id,
3685 "admission_facts_carrier_not_permitted",
3686 "admission_facts_not_permitted",
3687 "admission facts may only be carried by the configured reserved module",
3688 )?]);
3689 }
3690
3691 let target_allowed = self
3692 .admission_facts_targets
3693 .as_ref()
3694 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3695 if !target_allowed {
3696 return Ok(vec![self.route_open_refusal_frame(
3697 ctx,
3698 &frame,
3699 &target_module_id,
3700 "admission_facts_target_not_listed",
3701 "admission_facts_target_not_allowed",
3702 format!(
3703 "admission facts are not permitted for target module_id '{target_module_id}'"
3704 ),
3705 )?]);
3706 }
3707
3708 }
3712
3713 let (bound_scope, scope_stamp) = match scope {
3719 None => (None, None),
3720 Some(selector) => {
3721 let owner_configured = match &selector.owner {
3722 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3728 _ => false,
3729 };
3730 let admitted = self
3731 .scopes
3732 .read()
3733 .unwrap_or_else(|poisoned| poisoned.into_inner())
3734 .admit(&principal, &target_module_id, &selector, owner_configured)
3735 .and_then(|admission| {
3736 crate::scopes::check_target_flow_support(
3737 &admission.stamp,
3738 &target_module_id,
3739 registration.manifest.capabilities.as_ref(),
3740 )?;
3741 crate::scopes::check_target_agent_run_support(
3742 &admission.stamp,
3743 &target_module_id,
3744 registration.manifest.capabilities.as_ref(),
3745 )?;
3746 Ok(admission)
3747 });
3748 match admitted {
3749 Ok(admission) => (
3750 Some(BoundScope {
3751 owner: admission.owner,
3752 scope_ref: admission.stamp.scope_ref.clone(),
3753 tag: admission.tag,
3754 }),
3755 Some(admission.stamp),
3756 ),
3757 Err(refusal) => {
3758 return Ok(vec![self.route_open_refusal_frame(
3759 ctx,
3760 &frame,
3761 &target_module_id,
3762 refusal.code,
3763 refusal.code,
3764 refusal.message,
3765 )?]);
3766 }
3767 }
3768 }
3769 };
3770
3771 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3793 Ok(project_root) => project_root,
3794 Err(err) => {
3795 return Ok(vec![control_error_frame(
3796 &frame,
3797 "invalid_project_root",
3798 err.to_string(),
3799 )?])
3800 }
3801 };
3802 identity.project_root = project_root.as_path().to_path_buf();
3803
3804 let _concurrency_guard = match self
3815 .route_bind_concurrency
3816 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3817 {
3818 Ok(guard) => guard,
3819 Err(in_flight) => {
3820 return Ok(vec![self.route_open_target_capacity_refusal(
3821 ctx,
3822 &frame,
3823 &target_module_id,
3824 in_flight,
3825 )?]);
3826 }
3827 };
3828
3829 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3832 RouteBindAdmission::Admitted { guard, probe } => {
3833 if probe {
3834 info!(
3835 module_id = %target_module_id,
3836 connection_id = ctx.connection_id.get(),
3837 "route.bind breaker half-open: admitting one probe"
3838 );
3839 }
3840 guard
3841 }
3842 RouteBindAdmission::Refused {
3843 consecutive_timeouts,
3844 retry_in,
3845 probe_in_flight,
3846 } => {
3847 return Ok(vec![self.route_open_breaker_refusal_frame(
3848 ctx,
3849 &frame,
3850 &target_module_id,
3851 consecutive_timeouts,
3852 retry_in,
3853 probe_in_flight,
3854 )?]);
3855 }
3856 };
3857
3858 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3863 let relay_deadline = Instant::now() + route_bind_relay_timeout;
3864 let pending = match self
3865 .forwarding
3866 .begin_route_bind_relay_for(
3867 ctx.connection_id,
3868 ctx.egress.clone(),
3869 response_version(&frame),
3870 frame.header.corr,
3871 &target_module_id,
3872 principal.clone(),
3873 bound_scope,
3874 Some(project_root),
3875 relay_deadline,
3876 )
3877 .await
3878 {
3879 Ok(pending) => pending,
3880 Err(err) => {
3881 return Ok(vec![self.route_open_refusal_frame(
3882 ctx,
3883 &frame,
3884 &target_module_id,
3885 "relay_reservation_failed",
3886 forwarding_error_code(&err),
3887 err.to_string(),
3888 )?])
3889 }
3890 };
3891 let crate::forwarding::PendingRouteBindRelay {
3892 endpoint,
3893 module_sink,
3894 negotiated_ver,
3895 client_channel,
3896 client_epoch,
3897 module_channel,
3898 module_epoch,
3899 corr: relay_corr,
3900 receiver,
3901 } = pending;
3902 let mut reservation =
3903 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3904
3905 if let Some(stamp) = scope_stamp
3910 .as_ref()
3911 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
3912 {
3913 let relay_registration = self
3914 .registry
3915 .get_module_by_connection(endpoint.connection_id)
3916 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3917 if let Err(refusal) = crate::scopes::check_target_flow_support(
3918 stamp,
3919 &target_module_id,
3920 relay_registration
3921 .as_ref()
3922 .and_then(|registration| registration.manifest.capabilities.as_ref()),
3923 )
3924 .and_then(|()| {
3925 crate::scopes::check_target_agent_run_support(
3926 stamp,
3927 &target_module_id,
3928 relay_registration
3929 .as_ref()
3930 .and_then(|registration| registration.manifest.capabilities.as_ref()),
3931 )
3932 }) {
3933 reservation.release_and_disarm();
3934 return Ok(vec![self.route_open_refusal_frame(
3935 ctx,
3936 &frame,
3937 &target_module_id,
3938 refusal.code,
3939 refusal.code,
3940 refusal.message,
3941 )?]);
3942 }
3943 }
3944
3945 debug!(
3946 connection_id = ctx.connection_id.get(),
3947 client_channel,
3948 client_epoch,
3949 module_channel,
3950 module_epoch,
3951 "reserved route handle pair"
3952 );
3953 let principal_label = principal_label(&principal);
3956 let relay = ModuleControlRequest::RouteBind {
3957 route_channel: module_channel,
3958 epoch: module_epoch,
3959 target,
3960 identity,
3961 principal: Some(principal),
3962 consumer_capabilities,
3963 role_versions,
3964 admission_facts,
3965 scope: scope_stamp,
3966 };
3967 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3968 RouterError::backend(
3969 0,
3970 frame.header.corr,
3971 format!("failed to encode route.bind request: {err}"),
3972 )
3973 })?;
3974 let relay_frame = Frame::build_with_version(
3975 negotiated_ver,
3976 FrameType::Request,
3977 control_flags(),
3978 0,
3979 0,
3980 relay_corr,
3981 relay_body,
3982 )
3983 .map_err(RouterError::FrameBuild)?;
3984
3985 if let Err(err) = module_sink.send(relay_frame).await {
3986 reservation.release_and_disarm();
3987 return Ok(vec![self.route_open_refusal_frame(
3988 ctx,
3989 &frame,
3990 &target_module_id,
3991 "relay_send_failed",
3992 "target_unavailable",
3993 err.to_string(),
3994 )?]);
3995 }
3996
3997 #[cfg(test)]
3998 if let Some(hold) = &self.route_bind_relay_sent_hold {
3999 hold.wait().await;
4003 hold.wait().await;
4004 }
4005
4006 if self
4011 .forwarding
4012 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4013 .map_err(RouterError::Forwarding)?
4014 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4015 {
4016 self.send_abandoned_route_bind_goodbye(
4017 &module_sink,
4018 negotiated_ver,
4019 module_channel,
4020 module_epoch,
4021 );
4022 }
4023
4024 match timeout_at(relay_deadline, receiver).await {
4025 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4026 reservation.disarm();
4027 if breaker.record_accepted() {
4028 info!(
4029 module_id = %target_module_id,
4030 "route.bind breaker closed: the probe was accepted"
4031 );
4032 }
4033 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4034 Ok(Vec::new())
4035 }
4036 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4037 reservation.release_and_disarm();
4038 breaker.record_inconclusive();
4042 let scope_code = match body.code.as_str() {
4046 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4047 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4048 _ => None,
4049 };
4050 if let Some(code) = scope_code {
4051 self.observe_route_open_refusal(
4052 ctx,
4053 &target_module_id,
4054 "scope_changed_before_commit",
4055 code,
4056 );
4057 return Ok(vec![control_error_body_frame(&frame, body)?]);
4058 }
4059 self.counters
4060 .increment_route_open_refused("module_rejected");
4061 info!(
4062 target: "control",
4063 code = "module_rejected",
4064 module_code = ?body.code,
4065 module_id = ?target_module_id,
4066 connection_id = ctx.connection_id.get(),
4067 "route.open refused"
4068 );
4069 Ok(vec![control_error_body_frame(&frame, body)?])
4070 }
4071 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4072 reservation.release_and_disarm();
4073 breaker.record_inconclusive();
4074 tracing::warn!(
4081 module_id = %target_module_id,
4082 "route.bind relay abandoned: {message}"
4083 );
4084 Ok(vec![self.route_open_refusal_frame(
4085 ctx,
4086 &frame,
4087 &target_module_id,
4088 "relay_abandoned",
4089 "target_unavailable",
4090 message,
4091 )?])
4092 }
4093 Ok(Err(_)) => {
4094 reservation.release_and_disarm();
4095 breaker.record_inconclusive();
4096 Ok(vec![self.route_open_refusal_frame(
4097 ctx,
4098 &frame,
4099 &target_module_id,
4100 "relay_waiter_canceled",
4101 "target_unavailable",
4102 "route.bind relay waiter was canceled before the module responded",
4103 )?])
4104 }
4105 Err(_) => {
4106 reservation.release_and_disarm();
4107 if let Some(opened) = breaker.record_timeout(
4111 self.route_bind_breaker_threshold,
4112 self.route_bind_breaker_cooldown,
4113 ) {
4114 warn!(
4115 module_id = %target_module_id,
4116 consecutive_timeouts = opened.consecutive_timeouts,
4117 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4118 reopened_after_probe = opened.reopened_after_probe,
4119 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4120 );
4121 }
4122 tracing::warn!(
4130 module_id = %target_module_id,
4131 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4132 "route.bind relay timed out: module did not ack within budget"
4133 );
4134 Ok(vec![self.route_open_refusal_frame(
4135 ctx,
4136 &frame,
4137 &target_module_id,
4138 "relay_timed_out",
4139 "module_timeout",
4140 format!(
4141 "module_id '{target_module_id}' did not answer route.bind within {:?}",
4142 route_bind_relay_timeout
4143 ),
4144 )?])
4145 }
4146 }
4147 }
4148
4149 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4150 let response = ClientControlResponse::SupervisorSpawnSnapshot {
4151 snapshot: self.supervisor.spawn_snapshot(),
4152 };
4153 Ok(vec![control_response_body_frame(
4154 &frame,
4155 &response,
4156 "ClientControlResponse::SupervisorSpawnSnapshot",
4157 )?])
4158 }
4159
4160 fn handle_supervisor_spawn_subscribe(
4161 &self,
4162 ctx: &RouteCtx,
4163 frame: Frame,
4164 since: Option<SpawnCursor>,
4165 ) -> Result<Vec<Frame>, RouterError> {
4166 match self.supervisor.subscribe_spawns(
4167 ctx.connection_id,
4168 frame.header.corr,
4169 response_version(&frame),
4170 since,
4171 ctx.egress.clone(),
4172 ) {
4173 Ok(()) => Ok(Vec::new()),
4174 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4175 Ok(vec![control_error_body_frame(
4176 &frame,
4177 ErrorBody {
4178 code: "spawn_cursor_incarnation_mismatch".to_string(),
4179 message: "spawn cursor belongs to a different daemon incarnation"
4180 .to_string(),
4181 detail: Some(serde_json::json!({
4182 "current_daemon_incarnation": current
4183 })),
4184 },
4185 )?])
4186 }
4187 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4188 &frame,
4189 ErrorBody {
4190 code: "spawn_cursor_too_old".to_string(),
4191 message: "spawn cursor predates the retained event ring".to_string(),
4192 detail: Some(serde_json::json!({
4193 "oldest_retained_cursor": oldest
4194 })),
4195 },
4196 )?]),
4197 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4198 0,
4199 frame.header.corr,
4200 format!("failed to open supervisor spawn subscription: {error}"),
4201 )),
4202 }
4203 }
4204
4205 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4206 let generation = self
4207 .registry
4208 .generation()
4209 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4210 let mut modules = Vec::new();
4211 for module in self.supervisor.list() {
4212 let status = module.status_for_control("list").map_err(|err| {
4213 RouterError::backend(
4214 0,
4215 frame.header.corr,
4216 format!("failed to read supervisor status: {err}"),
4217 )
4218 })?;
4219 let (configured, _) = module.configuration().map_err(|err| {
4220 RouterError::backend(
4221 0,
4222 frame.header.corr,
4223 format!("failed to read module configuration: {err}"),
4224 )
4225 })?;
4226 let image = module.running_image_agreement().await;
4228 let resources = Some(module.child_resource_usage());
4231 let pending_reload = Some(reload_verdict(
4232 &configured.program,
4233 status.spawned_from.as_deref(),
4234 image,
4235 ));
4236 modules.push(
4239 SupervisorEntry::new(
4240 status.module_id,
4241 status.state.to_string(),
4242 status.enabled,
4243 status.live,
4244 status.health.status,
4245 )
4246 .with_launch_nonce_env(Some(!cfg!(unix)))
4247 .with_protocol(status.protocol)
4248 .with_pending_reload(pending_reload)
4249 .with_last_probe_ms(status.health.last_probe_ms)
4250 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
4251 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
4252 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
4253 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
4254 .with_restart_count(Some(status.restart_count))
4255 .with_max_restarts(Some(status.max_restarts))
4256 .with_lifetime_restarts(Some(status.lifetime_restarts))
4257 .with_spawn_generation(Some(status.spawn_generation))
4258 .with_restart_window_secs(Some(status.restart_window.as_secs()))
4259 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
4260 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
4261 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
4262 .with_resources(resources),
4263 );
4264 }
4265 let response = ClientControlResponse::SupervisorList {
4266 generation,
4267 modules,
4268 };
4269 Ok(vec![control_response_body_frame(
4270 &frame,
4271 &response,
4272 "ClientControlResponse::SupervisorList",
4273 )?])
4274 }
4275
4276 fn handle_supervisor_stderr_tail(
4277 &self,
4278 frame: Frame,
4279 module_id: String,
4280 max_lines: Option<u32>,
4281 max_bytes: Option<u32>,
4282 ) -> Result<Vec<Frame>, RouterError> {
4283 let Some(module) = self.supervisor.get(&module_id) else {
4284 return Ok(vec![control_error_frame(
4285 &frame,
4286 "unknown_module",
4287 format!("module_id '{module_id}' is not supervised"),
4288 )?]);
4289 };
4290
4291 let snapshot = module.stderr_tail(
4292 max_lines.map(|value| value as usize),
4293 max_bytes.map(|value| value as usize),
4294 );
4295
4296 let response = ClientControlResponse::SupervisorStderrTail {
4297 module_id,
4298 tail: StderrTail {
4299 capture: match snapshot.capture {
4300 CaptureState::Captured => StderrCaptureState::Captured,
4301 CaptureState::Incomplete { reason } => {
4302 StderrCaptureState::Incomplete { reason }
4303 }
4304 CaptureState::NotCaptured { reason } => {
4305 StderrCaptureState::NotCaptured { reason }
4306 }
4307 },
4308 entries: snapshot
4309 .entries
4310 .into_iter()
4311 .map(|entry| match entry {
4312 TailEntry::Line {
4313 text,
4314 truncated,
4315 at_ms,
4316 } => StderrTailEntry::Line {
4317 text,
4318 truncated,
4319 at_ms,
4320 },
4321 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
4322 })
4323 .collect(),
4324 dropped_lines: snapshot.dropped_lines,
4325 },
4326 };
4327 Ok(vec![control_response_body_frame(
4328 &frame,
4329 &response,
4330 "ClientControlResponse::SupervisorStderrTail",
4331 )?])
4332 }
4333
4334 async fn handle_supervisor_terminals(
4335 &self,
4336 frame: Frame,
4337 module_id: String,
4338 ) -> Result<Vec<Frame>, RouterError> {
4339 let Some(module) = self.supervisor.get(&module_id) else {
4340 return Ok(vec![control_error_frame(
4341 &frame,
4342 "unknown_module",
4343 format!("module_id '{module_id}' is not supervised"),
4344 )?]);
4345 };
4346
4347 let terminals = module
4350 .read_durable_terminal_history()
4351 .await
4352 .map_err(|error| {
4353 RouterError::backend(
4354 0,
4355 frame.header.corr,
4356 format!("failed to read terminal history: {error}"),
4357 )
4358 })?;
4359 let response = ClientControlResponse::SupervisorTerminals {
4360 module_id,
4361 terminals,
4362 };
4363 Ok(vec![control_response_body_frame(
4364 &frame,
4365 &response,
4366 "ClientControlResponse::SupervisorTerminals",
4367 )?])
4368 }
4369
4370 fn handle_supervisor_routes(
4371 &self,
4372 frame: Frame,
4373 module_id: Option<String>,
4374 ) -> Result<Vec<Frame>, RouterError> {
4375 let modules = self
4376 .forwarding
4377 .route_census(module_id.as_deref())
4378 .map_err(RouterError::Forwarding)?
4379 .into_iter()
4380 .map(|(module_id, routes)| SupervisorRouteModule {
4381 module_id,
4382 routes: routes
4383 .into_iter()
4384 .map(|route| SupervisorRoute {
4385 consumer: match route.principal {
4386 Principal::Reserved { module_id } => {
4387 SupervisorRouteConsumer::Reserved { module_id }
4388 }
4389 Principal::Direct | Principal::Unverified => {
4390 SupervisorRouteConsumer::Direct {
4391 connection_id: route.goodbye_target.connection_id.get(),
4392 }
4393 }
4394 },
4395 age_ms: Instant::now()
4396 .saturating_duration_since(route.bound_at)
4397 .as_millis()
4398 .try_into()
4399 .unwrap_or(u64::MAX),
4400 draining: route.draining,
4401 drain_reason: route.drain_reason,
4402 })
4403 .collect(),
4404 })
4405 .collect();
4406 let response = ClientControlResponse::SupervisorRoutes { modules };
4407 Ok(vec![control_response_body_frame(
4408 &frame,
4409 &response,
4410 "ClientControlResponse::SupervisorRoutes",
4411 )?])
4412 }
4413
4414 async fn handle_supervisor_provenance(
4415 &self,
4416 frame: Frame,
4417 module_id: Option<String>,
4418 ) -> Result<Vec<Frame>, RouterError> {
4419 let mut selected = if let Some(module_id) = module_id {
4420 let Some(module) = self.supervisor.get(&module_id) else {
4421 return Ok(vec![control_error_frame(
4422 &frame,
4423 "unknown_module",
4424 format!("module_id '{module_id}' is not supervised"),
4425 )?]);
4426 };
4427 vec![module]
4428 } else {
4429 self.supervisor.list()
4430 };
4431
4432 let mut modules = Vec::with_capacity(selected.len());
4433 for module in selected.drain(..) {
4434 let (status, observed_image) = module
4435 .status_and_running_image_agreement()
4436 .await
4437 .map_err(|err| {
4438 RouterError::backend(
4439 0,
4440 frame.header.corr,
4441 format!("failed to read supervisor status: {err}"),
4442 )
4443 })?;
4444 let module_declared = self
4445 .registry
4446 .get_module(&status.module_id)
4447 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4448 .and_then(|registration| registration.manifest.provenance)
4449 .map(|build| ModuleDeclaredProvenance::Reported { build })
4450 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4451 #[cfg(test)]
4452 let running_image = match &self.provenance_probe_override {
4453 Some(result) => result.clone(),
4454 None => observed_image,
4455 };
4456 #[cfg(not(test))]
4457 let running_image = observed_image;
4458 modules.push(SupervisorModuleProvenance {
4459 module_id: status.module_id,
4460 module_declared,
4461 daemon_observed: SupervisorObservedProcess {
4462 pid: status.pid,
4463 spawned_at_ms: status.spawned_at_ms,
4464 spawned_from: status.spawned_from,
4465 running_image,
4466 },
4467 });
4468 }
4469 let daemon = SupervisorDaemonProvenance {
4470 daemon_build: self.daemon_provenance.build.clone(),
4471 daemon_observed: DaemonObservedProcess {
4472 pid: self.daemon_provenance.pid,
4473 started_at_ms: self
4474 .daemon_provenance
4475 .start_clock
4476 .map(|clock| clock.started_at_ms())
4477 .or(self.daemon_provenance.started_at_ms),
4478 running_image: self
4479 .daemon_provenance
4480 .probe
4481 .observe(
4482 self.daemon_provenance.pid,
4483 self.daemon_provenance.executable_path.as_deref(),
4484 self.daemon_provenance.executable_identity,
4485 self.daemon_provenance.process_start_time,
4486 )
4487 .await,
4488 },
4489 };
4490 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
4491 Ok(vec![control_response_body_frame(
4492 &frame,
4493 &response,
4494 "ClientControlResponse::SupervisorProvenance",
4495 )?])
4496 }
4497
4498 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4499 self.refresh_capability_requirements();
4500 let generation = self
4501 .registry
4502 .generation()
4503 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4504 let modules = self
4505 .supervisor
4506 .list()
4507 .into_iter()
4508 .map(|module| {
4509 let status = module.status_for_control("health").map_err(|err| {
4510 RouterError::backend(
4511 0,
4512 frame.header.corr,
4513 format!("failed to read supervisor health: {err}"),
4514 )
4515 })?;
4516 let module_id = status.module_id;
4517 let capability_detail = self
4518 .capability_evaluator
4519 .required_problem_detail(&module_id);
4520 Ok(SupervisorHealthEntry {
4521 module_id,
4522 status: status.health.status,
4523 detail: append_capability_problem_detail(
4524 status.health.detail,
4525 capability_detail,
4526 ),
4527 metrics: status.health.metrics,
4528 consecutive_failures: status.health.consecutive_failures,
4529 late_answer_count: status.health.late_answer_count,
4530 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4531 last_action: status.health.last_action,
4532 last_action_ms: status.health.last_action_ms,
4533 last_probe_ms: status.health.last_probe_ms,
4534 })
4535 })
4536 .collect::<Result<Vec<_>, RouterError>>()?;
4537 let response = ClientControlResponse::SupervisorHealth {
4538 generation,
4539 modules,
4540 };
4541 Ok(vec![control_response_body_frame(
4542 &frame,
4543 &response,
4544 "ClientControlResponse::SupervisorHealth",
4545 )?])
4546 }
4547
4548 async fn handle_supervisor_restart(
4549 &self,
4550 frame: Frame,
4551 module_id: String,
4552 drain_timeout_ms: Option<u64>,
4553 ) -> Result<Vec<Frame>, RouterError> {
4554 let operation_lock = self.supervisor.operation_lock();
4555 let _operation_guard = operation_lock.lock().await;
4556 let Some(module) = self.supervisor.get(&module_id) else {
4557 return Ok(vec![control_error_frame(
4558 &frame,
4559 "unknown_module",
4560 format!("module_id '{module_id}' is not supervised"),
4561 )?]);
4562 };
4563
4564 self.route_outages.mark_operator_action(&module_id);
4565 if let Err(err) = module.restart(drain_timeout_ms).await {
4566 self.route_outages
4567 .operator_action_ended_unrefused(&module_id);
4568 let (code, message) = match err {
4569 crate::supervise::SuperviseError::Disabled { .. } => {
4570 ("module_disabled", err.to_string())
4571 }
4572 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4573 ("swap_in_progress", err.to_string())
4574 }
4575 _ => (
4576 "target_unavailable",
4577 format!("failed to restart module_id '{module_id}': {err}"),
4578 ),
4579 };
4580 return Ok(vec![control_error_frame(&frame, code, message)?]);
4581 }
4582
4583 let response = ClientControlResponse::SupervisorAck {
4584 module_id,
4585 applied: true,
4586 };
4587 Ok(vec![control_response_body_frame(
4588 &frame,
4589 &response,
4590 "ClientControlResponse::SupervisorAck",
4591 )?])
4592 }
4593
4594 async fn handle_supervisor_swap(
4598 &self,
4599 frame: Frame,
4600 module_id: String,
4601 ready_timeout_ms: Option<u64>,
4602 ) -> Result<Vec<Frame>, RouterError> {
4603 let module = {
4610 let operation_lock = self.supervisor.operation_lock();
4611 let _operation_guard = operation_lock.lock().await;
4612 self.supervisor.get(&module_id)
4613 };
4614 let Some(module) = module else {
4615 return Ok(vec![control_error_frame(
4616 &frame,
4617 "unknown_module",
4618 format!("module_id '{module_id}' is not supervised"),
4619 )?]);
4620 };
4621
4622 self.route_outages.mark_operator_action(&module_id);
4623 if let Err(err) = module
4624 .swap(ready_timeout_ms.map(Duration::from_millis))
4625 .await
4626 {
4627 self.route_outages
4628 .operator_action_ended_unrefused(&module_id);
4629 use crate::supervise::SuperviseError;
4630 let message = err.to_string();
4631 let error = match err {
4632 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4633 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4634 code: "swap_refused".to_string(),
4635 message,
4636 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4637 },
4638 SuperviseError::SwapFailed {
4639 arm,
4640 candidate_exit,
4641 ..
4642 } => ErrorBody {
4643 code: "swap_failed".to_string(),
4644 message,
4645 detail: Some(serde_json::json!({
4646 "arm": arm.as_str(),
4647 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4648 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4649 })),
4650 },
4651 _ => ErrorBody::new(
4652 "target_unavailable",
4653 format!("failed to swap module_id '{module_id}': {message}"),
4654 ),
4655 };
4656 return Ok(vec![control_error_body_frame(&frame, error)?]);
4657 }
4658 self.route_outages
4662 .operator_action_ended_unrefused(&module_id);
4663
4664 let response = ClientControlResponse::SupervisorAck {
4665 module_id,
4666 applied: true,
4667 };
4668 Ok(vec![control_response_body_frame(
4669 &frame,
4670 &response,
4671 "ClientControlResponse::SupervisorAck",
4672 )?])
4673 }
4674
4675 async fn handle_supervisor_reload(
4676 &self,
4677 frame: Frame,
4678 module_id: String,
4679 ) -> Result<Vec<Frame>, RouterError> {
4680 let operation_lock = self.supervisor.operation_lock();
4681 let _operation_guard = operation_lock.lock().await;
4682 let Some(module) = self.supervisor.get(&module_id) else {
4683 return Ok(vec![control_error_frame(
4684 &frame,
4685 "unknown_module",
4686 format!("module_id '{module_id}' is not supervised"),
4687 )?]);
4688 };
4689
4690 self.route_outages.mark_operator_action(&module_id);
4691 if let Err(err) = module.reload().await {
4692 self.route_outages
4693 .operator_action_ended_unrefused(&module_id);
4694 let (code, message) = match err {
4695 crate::supervise::SuperviseError::Disabled { .. } => {
4696 ("module_disabled", err.to_string())
4697 }
4698 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4699 ("swap_in_progress", err.to_string())
4700 }
4701 _ => (
4702 "reload_failed",
4703 format!("failed to reload module_id '{module_id}': {err}"),
4704 ),
4705 };
4706 return Ok(vec![control_error_frame(&frame, code, message)?]);
4707 }
4708
4709 let response = ClientControlResponse::SupervisorAck {
4710 module_id,
4711 applied: true,
4712 };
4713 Ok(vec![control_response_body_frame(
4714 &frame,
4715 &response,
4716 "ClientControlResponse::SupervisorAck",
4717 )?])
4718 }
4719
4720 async fn handle_supervisor_rescan(
4721 &self,
4722 frame: Frame,
4723 preview: bool,
4724 ) -> Result<Vec<Frame>, RouterError> {
4725 let Some(context) = self.rescan.clone() else {
4726 return Ok(vec![control_error_frame(
4727 &frame,
4728 "rescan_unavailable",
4729 "the daemon was not started with a reloadable config path".to_string(),
4730 )?]);
4731 };
4732
4733 let operation_lock = self.supervisor.operation_lock();
4734 let _operation_guard = operation_lock.lock().await;
4735 let loaded = match crate::daemon_config::load(&context.config_path) {
4736 Ok(config) => config,
4737 Err(err) => {
4738 return Ok(vec![control_error_frame(
4739 &frame,
4740 "invalid_daemon_config",
4741 format!("supervisor rescan rejected daemon config: {err}"),
4742 )?])
4743 }
4744 };
4745 let Some(config) = loaded else {
4753 return Ok(vec![control_error_frame(
4754 &frame,
4755 "invalid_daemon_config",
4756 format!(
4757 "daemon config not found at {}; refusing to rescan (an absent config would \
4758 retire every supervised module)",
4759 context.config_path.display()
4760 ),
4761 )?]);
4762 };
4763 let (
4764 configured_port,
4765 storage_config,
4766 admission_facts_carrier_module_id,
4767 admission_facts_targets,
4768 scope_authority_owners,
4769 modules,
4770 reserved_capabilities,
4771 ) = (
4772 config.port,
4773 config.storage,
4774 config.admission_facts_carrier_module_id,
4775 config.admission_facts_targets,
4776 config.scope_authority_owners,
4777 config.modules,
4778 config.reserved_capabilities,
4779 );
4780
4781 let mut restart_required = Vec::new();
4790 for section in RestartRequiredSection::ALL {
4791 let changed = match section {
4792 RestartRequiredSection::Port => configured_port != context.configured_port,
4793 RestartRequiredSection::Storage => storage_config != context.storage_config,
4794 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4795 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4796 }
4797 RestartRequiredSection::AdmissionFactsTargets => {
4798 admission_facts_targets != context.admission_facts_targets
4799 }
4800 RestartRequiredSection::ScopeAuthorityOwners => {
4801 scope_authority_owners != context.scope_authority_owners
4802 }
4803 };
4804 if changed {
4805 restart_required.push(section.label().to_string());
4806 }
4807 }
4808 if !restart_required.is_empty() {
4809 warn!(
4810 config_path = %context.config_path.display(),
4811 sections = %restart_required.join(", "),
4812 "daemon config changed outside the modules section; restart the daemon to apply those changes"
4813 );
4814 }
4815
4816 for configured in &modules {
4817 if let Err(err) = validate_spec(&configured.module_spec()) {
4818 return Ok(vec![control_error_frame(
4819 &frame,
4820 "invalid_daemon_config",
4821 format!("supervisor rescan rejected daemon config: {err}"),
4822 )?]);
4823 }
4824 }
4825
4826 let configured_capabilities = modules
4827 .iter()
4828 .map(|module| (module.module_id.clone(), module.enabled))
4829 .collect::<Vec<_>>();
4830 let preview_capability_warnings = if preview {
4831 let (_, registrations) = self.runtime_capability_snapshot()?;
4832 let current_modules = self
4833 .supervisor
4834 .list()
4835 .into_iter()
4836 .map(|module| module.module_id().to_string())
4837 .collect::<BTreeSet<_>>();
4838 let resulting_modules = configured_capabilities.clone();
4839 let removed = current_modules
4840 .into_iter()
4841 .filter(|module_id| {
4842 !resulting_modules
4843 .iter()
4844 .any(|(configured_id, _)| configured_id == module_id)
4845 })
4846 .collect::<Vec<_>>();
4847 self.capability_evaluator.preview_removal_warnings(
4848 resulting_modules,
4849 &removed,
4850 ®istrations,
4851 )
4852 } else {
4853 Vec::new()
4854 };
4855 let result = match self
4856 .reconcile_supervised_modules(&context.supervisor, modules, preview)
4857 .await
4858 {
4859 Ok(result) => result,
4860 Err(message) => {
4861 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4862 }
4863 };
4864 if !preview {
4865 self.capability_evaluator
4866 .configure(configured_capabilities, reserved_capabilities);
4867 self.capability_evaluator.wake_deadline_loop();
4868 self.refresh_capability_requirements();
4869 }
4870 let mut result = result;
4871 result.restart_required = restart_required;
4872 result.capability_warnings = preview_capability_warnings;
4873 let response = ClientControlResponse::SupervisorRescan { result };
4874 Ok(vec![control_response_body_frame(
4875 &frame,
4876 &response,
4877 "ClientControlResponse::SupervisorRescan",
4878 )?])
4879 }
4880
4881 async fn handle_supervisor_release_reserved(
4882 &self,
4883 frame: Frame,
4884 module_id: String,
4885 ) -> Result<Vec<Frame>, RouterError> {
4886 let Some(context) = self.rescan.clone() else {
4887 return Ok(vec![control_error_frame(
4888 &frame,
4889 "release_unavailable",
4890 "reserved-id release requires a daemon started with a reloadable config path",
4891 )?]);
4892 };
4893 let operation_lock = self.supervisor.operation_lock();
4894 let _operation_guard = operation_lock.lock().await;
4895 let loaded = match crate::daemon_config::load(&context.config_path) {
4896 Ok(Some(config)) => config,
4897 Ok(None) => {
4898 return Ok(vec![control_error_frame(
4899 &frame,
4900 "invalid_daemon_config",
4901 format!(
4902 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4903 context.config_path.display()
4904 ),
4905 )?])
4906 }
4907 Err(err) => {
4908 return Ok(vec![control_error_frame(
4909 &frame,
4910 "invalid_daemon_config",
4911 format!("unable to verify reserved-id release against daemon config: {err}"),
4912 )?])
4913 }
4914 };
4915 if loaded
4916 .modules
4917 .iter()
4918 .any(|configured| configured.module_id == module_id)
4919 {
4920 return Ok(vec![control_error_frame(
4921 &frame,
4922 "reserved_module_configured",
4923 format!(
4924 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4925 ),
4926 )?]);
4927 }
4928 if !self.supervisor.release_retained_reserved_gate(&module_id) {
4929 return Ok(vec![control_error_frame(
4930 &frame,
4931 "reserved_gate_not_retained",
4932 format!(
4933 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4934 ),
4935 )?]);
4936 }
4937
4938 let response = ClientControlResponse::SupervisorAck {
4939 module_id,
4940 applied: true,
4941 };
4942 Ok(vec![control_response_body_frame(
4943 &frame,
4944 &response,
4945 "ClientControlResponse::SupervisorAck",
4946 )?])
4947 }
4948
4949 async fn reconcile_supervised_modules(
4958 &self,
4959 supervisor: &Supervisor,
4960 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4961 preview: bool,
4962 ) -> Result<SupervisorRescanResult, String> {
4963 let mut current = BTreeMap::new();
4964 for module in self.supervisor.list() {
4965 let (spec, health) = module.configuration().map_err(|err| {
4966 format!(
4967 "failed to read configuration for module_id '{}': {err}",
4968 module.module_id()
4969 )
4970 })?;
4971 let enabled = module
4972 .status()
4973 .map_err(|err| {
4974 format!(
4975 "failed to read status for module_id '{}': {err}",
4976 module.module_id()
4977 )
4978 })?
4979 .enabled;
4980 current.insert(
4981 module.module_id().to_string(),
4982 (module, spec, health, enabled),
4983 );
4984 }
4985 let configured = configured_modules
4986 .into_iter()
4987 .map(|module| (module.module_id.clone(), module))
4988 .collect::<BTreeMap<_, _>>();
4989
4990 let added = configured
4991 .keys()
4992 .filter(|module_id| !current.contains_key(*module_id))
4993 .cloned()
4994 .collect::<Vec<_>>();
4995 let removed = current
4996 .keys()
4997 .filter(|module_id| !configured.contains_key(*module_id))
4998 .cloned()
4999 .collect::<Vec<_>>();
5000 let mut changed_pending_reload = Vec::new();
5001 let mut configuration_changes = BTreeSet::new();
5002 let mut enabled_changes = BTreeSet::new();
5003 let mut unchanged = 0_u32;
5004
5005 for (module_id, configured_module) in &configured {
5006 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5007 else {
5008 continue;
5009 };
5010 let launch_changed = *current_spec != configured_module.module_spec();
5014 let configuration_changed =
5015 launch_changed || *current_health != configured_module.health;
5016 let enabled_changed = *current_enabled != configured_module.enabled;
5017 if configuration_changed {
5018 configuration_changes.insert(module_id.clone());
5019 }
5020 if launch_changed {
5021 changed_pending_reload.push(module_id.clone());
5022 }
5023 if enabled_changed {
5024 enabled_changes.insert(module_id.clone());
5025 }
5026 if !configuration_changed && !enabled_changed {
5027 unchanged = unchanged.saturating_add(1);
5028 }
5029 }
5030
5031 if preview {
5037 return Ok(SupervisorRescanResult {
5038 added,
5039 removed,
5040 changed_pending_reload,
5041 enabled_changes: enabled_changes.iter().cloned().collect(),
5042 unchanged,
5043 preview: true,
5044 restart_required: Vec::new(),
5048 capability_warnings: Vec::new(),
5049 });
5050 }
5051
5052 for module_id in &removed {
5053 let module = ¤t
5054 .get(module_id)
5055 .expect("removed module came from current supervisor state")
5056 .0;
5057 module.retire().await.map_err(|err| {
5058 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5059 })?;
5060 self.supervisor.record_rescan_removal(module_id);
5094 self.supervisor.retire(module_id);
5095 self.route_outages.forget(module_id);
5096 }
5097
5098 for module_id in configured.keys() {
5099 let Some((module, _, _, _)) = current.get(module_id) else {
5100 continue;
5101 };
5102 let configured_module = configured
5103 .get(module_id)
5104 .expect("configured module id came from configured map");
5105 if configuration_changes.contains(module_id) {
5106 module
5107 .update_configuration(
5108 configured_module.module_spec(),
5109 configured_module.health.clone(),
5110 configured_module.drain_timeout_ms,
5111 )
5112 .await
5113 .map_err(|err| {
5114 format!(
5115 "failed to update module_id '{module_id}' configuration during rescan: {err}"
5116 )
5117 })?;
5118 }
5119 if enabled_changes.contains(module_id) {
5120 self.route_outages.mark_operator_action(module_id);
5123 module
5124 .set_enabled(configured_module.enabled)
5125 .await
5126 .map_err(|err| {
5127 self.route_outages.operator_action_ended_unrefused(module_id);
5128 format!(
5129 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5130 configured_module.enabled
5131 )
5132 })?;
5133 }
5134 }
5135
5136 for module_id in &added {
5137 let configured_module = configured
5138 .get(module_id)
5139 .expect("added module id came from configured map");
5140 supervisor
5141 .supervise_configured_with_health(
5142 configured_module.module_spec(),
5143 configured_module.enabled,
5144 configured_module.health.clone(),
5145 configured_module.drain_timeout_ms,
5146 configured_module.restart,
5147 )
5148 .map_err(|err| {
5149 format!("failed to add module_id '{module_id}' during rescan: {err}")
5150 })?;
5151 }
5152
5153 Ok(SupervisorRescanResult {
5154 added,
5155 removed,
5156 changed_pending_reload,
5157 enabled_changes: enabled_changes.iter().cloned().collect(),
5158 unchanged,
5159 preview: false,
5160 restart_required: Vec::new(),
5163 capability_warnings: Vec::new(),
5164 })
5165 }
5166
5167 async fn handle_supervisor_set_enabled(
5168 &self,
5169 frame: Frame,
5170 module_id: String,
5171 enabled: bool,
5172 ) -> Result<Vec<Frame>, RouterError> {
5173 let operation_lock = self.supervisor.operation_lock();
5174 let _operation_guard = operation_lock.lock().await;
5175 let Some(module) = self.supervisor.get(&module_id) else {
5176 return Ok(vec![control_error_frame(
5177 &frame,
5178 "unknown_module",
5179 format!("module_id '{module_id}' is not supervised"),
5180 )?]);
5181 };
5182
5183 self.route_outages.mark_operator_action(&module_id);
5186 let applied = match module.set_enabled(enabled).await {
5187 Ok(applied) => applied,
5188 Err(err) => {
5189 self.route_outages
5190 .operator_action_ended_unrefused(&module_id);
5191 return Ok(vec![control_error_frame(
5192 &frame,
5193 "target_unavailable",
5194 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
5195 )?]);
5196 }
5197 };
5198 if !applied {
5199 self.route_outages
5202 .operator_action_ended_unrefused(&module_id);
5203 }
5204
5205 self.capability_evaluator.wake_deadline_loop();
5206 self.refresh_capability_requirements();
5207 let response = ClientControlResponse::SupervisorAck { module_id, applied };
5208 Ok(vec![control_response_body_frame(
5209 &frame,
5210 &response,
5211 "ClientControlResponse::SupervisorAck",
5212 )?])
5213 }
5214
5215 async fn handle_supervisor_health_probe(
5216 &self,
5217 frame: Frame,
5218 module_id: String,
5219 ) -> Result<Vec<Frame>, RouterError> {
5220 self.refresh_capability_requirements();
5221 let Some(registration) = self
5222 .registry
5223 .get_module(&module_id)
5224 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5225 else {
5226 return Ok(vec![control_error_frame(
5227 &frame,
5228 "unknown_module",
5229 format!("module_id '{module_id}' is not registered"),
5230 )?]);
5231 };
5232
5233 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
5246 {
5247 return Ok(vec![control_error_frame(
5248 &frame,
5249 "health_not_advertised",
5250 format!("module_id '{module_id}' did not advertise health.check"),
5251 )?]);
5252 }
5253
5254 let deadline = Instant::now() + self.health_probe_timeout;
5255 let pending = match self.forwarding.begin_module_control_rpc_for(
5256 &module_id,
5257 MODULE_CONTROL_OP_HEALTH_CHECK,
5258 deadline,
5259 ) {
5260 Ok(pending) => pending,
5261 Err(err) => {
5262 return Ok(vec![control_error_frame(
5263 &frame,
5264 forwarding_error_code(&err),
5265 err.to_string(),
5266 )?])
5267 }
5268 };
5269
5270 let PendingModuleControlRpc {
5271 endpoint,
5272 module_sink,
5273 negotiated_ver,
5274 corr: probe_corr,
5275 receiver,
5276 } = pending;
5277 let mut guard =
5278 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
5279 let probe_body =
5280 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5281 RouterError::backend(
5282 0,
5283 frame.header.corr,
5284 format!("failed to encode health.check request: {err}"),
5285 )
5286 })?;
5287 let probe_frame = Frame::build_with_version(
5288 negotiated_ver,
5289 FrameType::Request,
5290 control_flags(),
5291 0,
5292 0,
5293 probe_corr,
5294 probe_body,
5295 )
5296 .map_err(RouterError::FrameBuild)?;
5297
5298 if let Err(err) = module_sink.send(probe_frame).await {
5299 return Ok(vec![control_error_frame(
5300 &frame,
5301 "target_unavailable",
5302 err.to_string(),
5303 )?]);
5304 }
5305
5306 match timeout_at(deadline, receiver).await {
5307 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5308 guard.disarm();
5309 let Some(report) = response.health_report() else {
5310 return Ok(vec![control_error_frame(
5311 &frame,
5312 "invalid_control_body",
5313 "health.check RPC returned a non-health response",
5314 )?]);
5315 };
5316 let HealthReport {
5321 status,
5322 detail,
5323 metrics,
5324 } = report;
5325 let capability_detail = self
5326 .capability_evaluator
5327 .required_problem_detail(&module_id);
5328 let response = ClientControlResponse::SupervisorHealthProbe {
5329 module_id,
5330 status,
5331 detail: append_capability_problem_detail(detail, capability_detail),
5332 metrics,
5333 };
5334 Ok(vec![control_response_body_frame(
5335 &frame,
5336 &response,
5337 "ClientControlResponse::SupervisorHealthProbe",
5338 )?])
5339 }
5340 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
5341 guard.disarm();
5342 Ok(vec![control_error_body_frame(&frame, body)?])
5343 }
5344 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5345 guard.disarm();
5346 Ok(vec![control_error_frame(
5347 &frame,
5348 "target_unavailable",
5349 message,
5350 )?])
5351 }
5352 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5353 guard.disarm();
5354 Ok(vec![control_error_frame(
5355 &frame,
5356 "invalid_control_body",
5357 message,
5358 )?])
5359 }
5360 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5361 guard.disarm();
5362 Ok(vec![control_error_frame(
5363 &frame,
5364 "invalid_control_body",
5365 format!("expected module-control op '{expected}', got '{actual}'"),
5366 )?])
5367 }
5368 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
5369 guard.disarm();
5370 Ok(vec![control_error_frame(
5371 &frame,
5372 "module_timeout",
5373 format!(
5374 "module_id '{module_id}' answered health.check after {:?}",
5375 self.health_probe_timeout
5376 ),
5377 )?])
5378 }
5379 Ok(Err(_)) => Ok(vec![control_error_frame(
5380 &frame,
5381 "target_unavailable",
5382 "health.check waiter was canceled before the module responded",
5383 )?]),
5384 Err(_) => Ok(vec![control_error_frame(
5385 &frame,
5386 "module_timeout",
5387 format!(
5388 "module_id '{module_id}' did not answer health.check within {:?}",
5389 self.health_probe_timeout
5390 ),
5391 )?]),
5392 }
5393 }
5394
5395 fn supervisor_status(
5396 &self,
5397 module_id: &str,
5398 corr: u64,
5399 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5400 self.supervisor
5401 .get(module_id)
5402 .map(|module| {
5403 let warming = module.is_warming_for_control("status").map_err(|err| {
5404 RouterError::backend(
5405 0,
5406 corr,
5407 format!(
5408 "failed to read supervisor warming state for module_id '{module_id}': {err}"
5409 ),
5410 )
5411 })?;
5412 module.status_for_control("status").map_err(|err| {
5413 RouterError::backend(
5414 0,
5415 corr,
5416 format!(
5417 "failed to read supervisor status for module_id '{module_id}': {err}"
5418 ),
5419 )
5420 }).map(|status| (status, warming))
5421 })
5422 .transpose()
5423 }
5424
5425 fn guard_module_control_op(
5426 &self,
5427 frame: &Frame,
5428 module_id: &str,
5429 op: &str,
5430 ) -> Result<Option<Frame>, RouterError> {
5431 if self.module_grants_op(module_id, op, frame.header.corr)? {
5432 return Ok(None);
5433 }
5434
5435 Ok(Some(control_error_frame(
5436 frame,
5437 "op_not_allowed",
5438 format!("module_id '{module_id}' did not grant control op '{op}'"),
5439 )?))
5440 }
5441
5442 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
5443 let Some(registration) = self
5444 .registry
5445 .get_module(module_id)
5446 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
5447 else {
5448 return Ok(false);
5449 };
5450 Ok(module_registration_grants_op(®istration.control_ops, op))
5451 }
5452
5453 fn handle_status_update(
5454 &self,
5455 endpoint: ModuleEndpointId,
5456 frame: Frame,
5457 ) -> Result<Vec<Frame>, RouterError> {
5458 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
5459 Ok(update) => update,
5460 Err(err) => {
5461 if is_known_module_push_op(&frame.body) {
5466 return Ok(vec![control_error_frame(
5467 &frame,
5468 "invalid_control_body",
5469 format!("malformed module control push body: {err}"),
5470 )?]);
5471 }
5472 return Ok(Vec::new());
5473 }
5474 };
5475
5476 match update {
5477 ModuleControlPush::RouteStatus {
5478 route_channel,
5479 route_epoch,
5480 status,
5481 } => {
5482 self.forwarding
5483 .cache_status(endpoint, route_channel, route_epoch, status)
5484 .map_err(RouterError::Forwarding)?;
5485 }
5486 }
5487 Ok(Vec::new())
5488 }
5489
5490 fn handle_route_poll(
5491 &self,
5492 ctx: &RouteCtx,
5493 frame: Frame,
5494 route_channel: u16,
5495 route_epoch: u32,
5496 kind: PollKind,
5497 ) -> Result<Vec<Frame>, RouterError> {
5498 let snapshot = self
5499 .forwarding
5500 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
5501 .map_err(RouterError::Forwarding)?;
5502 let response = match (kind, snapshot) {
5503 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
5504 ClientControlResponse::RoutePoll {
5505 route_channel,
5506 route_epoch,
5507 status,
5508 live: None,
5509 }
5510 }
5511 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5512 route_channel,
5513 route_epoch,
5514 status: None,
5515 live: None,
5516 },
5517 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5518 let live = self
5544 .process_liveness
5545 .as_ref()
5546 .and_then(|source| source.process_live(&module_id))
5547 .unwrap_or(true);
5548 ClientControlResponse::RoutePoll {
5549 route_channel,
5550 route_epoch,
5551 status: None,
5552 live: Some(live),
5553 }
5554 }
5555 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5556 route_channel,
5557 route_epoch,
5558 status: None,
5559 live: Some(false),
5560 },
5561 };
5562
5563 Ok(vec![control_response_body_frame(
5564 &frame,
5565 &response,
5566 "ClientControlResponse::RoutePoll",
5567 )?])
5568 }
5569
5570 pub(crate) fn observe_module_control_completion(
5571 &self,
5572 completion: ModuleControlRpcCompletion,
5573 ) -> bool {
5574 match completion {
5575 ModuleControlRpcCompletion::Unknown => false,
5576 ModuleControlRpcCompletion::Settled => true,
5577 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5578 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5579 info!(
5580 module_id = %module_id,
5581 latency_ms,
5582 "late health.check answer proves the module is alive"
5583 );
5584 match self
5585 .supervisor
5586 .record_late_health_answer(&module_id, latency_ms)
5587 {
5588 Ok(true) => {}
5589 Ok(false) => debug!(
5590 module_id = %module_id,
5591 latency_ms,
5592 "late health.check answer has no active supervisor snapshot"
5593 ),
5594 Err(err) => warn!(
5595 module_id = %module_id,
5596 latency_ms,
5597 error = %err,
5598 "failed to record late health.check answer"
5599 ),
5600 }
5601 true
5602 }
5603 }
5604 }
5605
5606 fn refuse_to_end_module_connection_for_a_client(
5627 &self,
5628 module_connection_id: ConnectionId,
5629 corr: u64,
5630 err: ForwardingError,
5631 ) -> Result<(), RouterError> {
5632 if let ForwardingError::ConnectionClosing { connection_id } = err {
5633 if connection_id != module_connection_id {
5634 warn!(
5635 module_connection_id = module_connection_id.get(),
5636 client_connection_id = connection_id.get(),
5637 corr,
5638 "dropping a route.bind response for a closing client; the module connection keeps serving"
5639 );
5640 return Ok(());
5641 }
5642 }
5643 Err(RouterError::Forwarding(err))
5644 }
5645
5646 fn handle_module_relay_response(
5647 &self,
5648 connection_id: ConnectionId,
5649 frame: Frame,
5650 ) -> Result<Vec<Frame>, RouterError> {
5651 let mut secondary_error = None;
5652 let outcome = match frame.header.ty {
5653 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5654 Ok(probe) if probe.op == "route.bind" => {
5655 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5656 Ok(ModuleControlResponse::RouteBindAck {}) => {
5657 RouteBindRelayOutcome::Accepted
5658 }
5659 Ok(other) => {
5660 let message =
5661 format!("route.bind response carried unexpected body: {other:?}");
5662 secondary_error = Some(control_error_frame(
5663 &frame,
5664 "invalid_control_body",
5665 message.clone(),
5666 )?);
5667 RouteBindRelayOutcome::ModuleGone(message)
5668 }
5669 Err(err) => {
5670 let message = format!("malformed route.bind response body: {err}");
5671 secondary_error = Some(control_error_frame(
5672 &frame,
5673 "invalid_control_body",
5674 message.clone(),
5675 )?);
5676 RouteBindRelayOutcome::ModuleGone(message)
5677 }
5678 }
5679 }
5680 Ok(probe) => {
5681 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5682 {
5683 Ok(response) => ModuleControlRpcOutcome::Response(response),
5684 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5685 "malformed {} response body: {err}",
5686 probe.op
5687 )),
5688 };
5689 let completion = self
5690 .forwarding
5691 .complete_module_control_rpc(
5692 connection_id,
5693 frame.header.corr,
5694 Some(&probe.op),
5695 outcome,
5696 )
5697 .map_err(RouterError::Forwarding)?;
5698 if !self.observe_module_control_completion(completion) {
5699 debug!(
5700 connection_id = connection_id.get(),
5701 corr = frame.header.corr,
5702 op = %probe.op,
5703 "dropping late or unknown module-control RPC response"
5704 );
5705 }
5706 return Ok(Vec::new());
5707 }
5708 Err(err) => {
5709 if let Some(expected_op) = self
5710 .forwarding
5711 .pending_module_control_op(connection_id, frame.header.corr)
5712 .map_err(RouterError::Forwarding)?
5713 {
5714 let completion = self
5715 .forwarding
5716 .complete_module_control_rpc(
5717 connection_id,
5718 frame.header.corr,
5719 None,
5720 ModuleControlRpcOutcome::MalformedResponse(format!(
5721 "malformed {expected_op} response body: {err}"
5722 )),
5723 )
5724 .map_err(RouterError::Forwarding)?;
5725 if !self.observe_module_control_completion(completion) {
5726 debug!(
5727 connection_id = connection_id.get(),
5728 corr = frame.header.corr,
5729 "dropping late malformed module-control RPC response"
5730 );
5731 }
5732 return Ok(Vec::new());
5733 }
5734 let message = format!("malformed route.bind response body: {err}");
5735 secondary_error = Some(control_error_frame(
5736 &frame,
5737 "invalid_control_body",
5738 message.clone(),
5739 )?);
5740 RouteBindRelayOutcome::ModuleGone(message)
5741 }
5742 },
5743 FrameType::Error => {
5744 if self
5745 .forwarding
5746 .pending_module_control_op(connection_id, frame.header.corr)
5747 .map_err(RouterError::Forwarding)?
5748 .is_some()
5749 {
5750 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5751 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5752 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5753 "malformed module-control ERROR body: {err}"
5754 )),
5755 };
5756 let completion = self
5757 .forwarding
5758 .complete_module_control_rpc(
5759 connection_id,
5760 frame.header.corr,
5761 None,
5762 outcome,
5763 )
5764 .map_err(RouterError::Forwarding)?;
5765 if !self.observe_module_control_completion(completion) {
5766 debug!(
5767 connection_id = connection_id.get(),
5768 corr = frame.header.corr,
5769 "dropping late or unknown module-control RPC error"
5770 );
5771 }
5772 return Ok(Vec::new());
5773 }
5774 match serde_json::from_slice::<ErrorBody>(&frame.body) {
5775 Ok(body) => RouteBindRelayOutcome::Rejected(body),
5776 Err(err) => {
5777 let message = format!("malformed route.bind ERROR body: {err}");
5778 secondary_error = Some(control_error_frame(
5779 &frame,
5780 "invalid_control_body",
5781 message.clone(),
5782 )?);
5783 RouteBindRelayOutcome::ModuleGone(message)
5784 }
5785 }
5786 }
5787 ty => {
5788 return Ok(vec![control_error_frame(
5789 &frame,
5790 "unsupported_control_frame",
5791 format!("unsupported module channel-0 frame {ty:?}"),
5792 )?])
5793 }
5794 };
5795
5796 let settled =
5797 self.forwarding
5798 .complete_pending_relay(connection_id, frame.header.corr, outcome);
5799 let completion = match settled {
5800 Ok(completion) => completion,
5801 Err(err) => {
5802 self.refuse_to_end_module_connection_for_a_client(
5803 connection_id,
5804 frame.header.corr,
5805 err,
5806 )?;
5807 return Ok(secondary_error.into_iter().collect());
5808 }
5809 };
5810 if let Some(target) = completion.abandoned.as_ref() {
5811 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5812 }
5813 if !completion.settled {
5814 debug!(
5815 connection_id = connection_id.get(),
5816 corr = frame.header.corr,
5817 frame_type = ?frame.header.ty,
5818 "dropping late or unknown route.bind relay response"
5819 );
5820 }
5821 Ok(secondary_error.into_iter().collect())
5822 }
5823
5824 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5825 debug!(connection_id = connection_id.get(), "handling GOODBYE");
5826 self.cleanup_connection_with_end_reason(
5830 connection_id,
5831 RegistrationEndReason::ExplicitGoodbye,
5832 )
5833 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5834 Ok(Vec::new())
5835 }
5836}
5837
5838impl Default for ControlHandler {
5839 fn default() -> Self {
5840 Self::new(Arc::new(Registry::default()))
5841 }
5842}
5843
5844impl crate::supervise::SwapPromotionObserver for ControlHandler {
5845 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5846 self.apply_registration_capabilities(registration);
5847 }
5848}
5849
5850fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5851 CapabilityRequirementStatus {
5852 consumer: status.consumer,
5853 capability: status.capability,
5854 need: match status.need {
5855 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5856 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5857 },
5858 verdict: status.verdict.as_str().to_string(),
5859 episode_seq: status.episode_seq,
5860 config_satisfiable: status.config_satisfiable,
5861 runtime_available: status.runtime_available,
5862 detail: status.detail,
5863 }
5864}
5865
5866fn append_capability_problem_detail(
5867 detail: Option<String>,
5868 capability_detail: Option<String>,
5869) -> Option<String> {
5870 match (detail, capability_detail) {
5871 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5872 (Some(detail), None) => Some(detail),
5873 (None, Some(capability_detail)) => Some(capability_detail),
5874 (None, None) => None,
5875 }
5876}
5877
5878fn subc_ops() -> Vec<String> {
5879 SUBC_CONTROL_OPS
5880 .iter()
5881 .map(|op| (*op).to_string())
5882 .collect()
5883}
5884
5885fn module_subc_ops() -> Vec<String> {
5886 SUBC_CONTROL_OPS
5887 .iter()
5888 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5889 .map(|op| (*op).to_string())
5890 .collect()
5891}
5892
5893#[cfg(test)]
5894fn module_baseline_control_ops() -> Vec<String> {
5895 MODULE_BASELINE_CONTROL_OPS
5896 .iter()
5897 .map(|op| (*op).to_string())
5898 .collect()
5899}
5900
5901fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5902 let mut seen = HashSet::new();
5903 let mut effective = Vec::new();
5904 for op in MODULE_BASELINE_CONTROL_OPS {
5905 if seen.insert((*op).to_string()) {
5906 effective.push((*op).to_string());
5907 }
5908 }
5909 for op in declared.unwrap_or_default() {
5910 if seen.insert(op.clone()) {
5911 effective.push(op);
5912 }
5913 }
5914 effective
5915}
5916
5917fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5918 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5919}
5920
5921fn target_module_id(target: &RouteTarget) -> &str {
5922 match target {
5923 RouteTarget::ToolProvider { module_id }
5924 | RouteTarget::ManagementSurface { module_id }
5925 | RouteTarget::InternalService { module_id, .. } => module_id,
5926 }
5927}
5928
5929fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5930 roles.iter().any(|role| match (target, role) {
5931 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5932 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5933 (
5934 RouteTarget::InternalService { service_id, .. },
5935 ProviderRole::InternalService {
5936 service_id: provided,
5937 ..
5938 },
5939 ) => service_id == provided,
5940 _ => false,
5941 })
5942}
5943
5944fn is_routable_role(role: &ProviderRole) -> bool {
5945 matches!(
5946 role,
5947 ProviderRole::ToolProvider { .. }
5948 | ProviderRole::ManagementSurface { .. }
5949 | ProviderRole::InternalService { .. }
5950 )
5951}
5952
5953#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5954enum ControlRequestBodyError {
5955 UnknownOp,
5956 InvalidBody,
5957}
5958
5959#[derive(Debug, Deserialize)]
5960struct ControlOpProbe {
5961 op: String,
5962}
5963
5964const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5967
5968fn is_known_module_push_op(body: &[u8]) -> bool {
5969 serde_json::from_slice::<ControlOpProbe>(body)
5970 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
5971 .unwrap_or(false)
5972}
5973
5974fn is_known_module_request_op(body: &[u8]) -> bool {
5975 serde_json::from_slice::<ControlOpProbe>(body)
5976 .map(|probe| is_module_to_subc_op(&probe.op))
5977 .unwrap_or(false)
5978}
5979
5980fn is_module_to_subc_op(op: &str) -> bool {
5981 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
5982}
5983
5984fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
5985 debug!(
5986 op = %op,
5987 connection_id = connection_id.get(),
5988 corr,
5989 "control dispatch"
5990 );
5991}
5992
5993fn log_slow_control_dispatch(
5994 dispatch_started_at: Option<StdInstant>,
5995 op: &'static str,
5996 connection_id: ConnectionId,
5997 corr: u64,
5998) {
5999 let Some(dispatch_started_at) = dispatch_started_at else {
6000 return;
6001 };
6002 let elapsed = dispatch_started_at.elapsed();
6003 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6004 warn!(
6005 op = %op,
6006 connection_id = connection_id.get(),
6007 corr,
6008 elapsed_ms = elapsed.as_millis() as u64,
6009 "slow control dispatch"
6010 );
6011 }
6012}
6013
6014fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6015 match request {
6016 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6017 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6018 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6019 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6020 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6021 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6022 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6023 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6024 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6025 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6026 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6027 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6028 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6029 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6030 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6031 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6032 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6033 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6034 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6035 }
6036}
6037
6038fn principal_label(principal: &Principal) -> String {
6039 match principal {
6040 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6041 Principal::Direct => "direct".to_string(),
6042 other => format!("{other:?}"),
6043 }
6044}
6045
6046fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6047 match request {
6048 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6049 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6050 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6051 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6052 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6053 }
6054}
6055
6056fn parse_client_control_request(
6057 body: &[u8],
6058) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6059 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6060 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6061 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6062 ControlRequestBodyError::InvalidBody
6063 }
6064 Ok(_) => ControlRequestBodyError::UnknownOp,
6065 Err(_) => ControlRequestBodyError::InvalidBody,
6066 };
6067 (err, classification)
6068 })
6069}
6070
6071fn parse_module_control_request_from_module(
6072 body: &[u8],
6073) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6074 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6075 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6076 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6077 Ok(_) => ControlRequestBodyError::UnknownOp,
6078 Err(_) => ControlRequestBodyError::InvalidBody,
6079 };
6080 (err, classification)
6081 })
6082}
6083
6084#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6085enum ProviderRoleKind {
6086 ToolProvider,
6087 PipelineStage,
6088 ManagementSurface,
6089 InternalService,
6090}
6091
6092fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6093 match role {
6094 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6095 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6096 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6097 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
6098 }
6099}
6100
6101fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
6102 roles.iter().map(provider_role_kind).collect()
6103}
6104
6105const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
6108
6109#[derive(Debug, Default, PartialEq, Eq)]
6111struct ScopeOutcomeCounts {
6112 created: usize,
6113 replaced: usize,
6114 updated: usize,
6115 unchanged: usize,
6116 refused: usize,
6117}
6118
6119impl ScopeOutcomeCounts {
6120 fn of(results: &[ScopeRecordResult]) -> Self {
6121 let mut counts = Self::default();
6122 for result in results {
6123 let slot = match result.outcome {
6124 ScopeRecordOutcome::Created => &mut counts.created,
6125 ScopeRecordOutcome::Replaced => &mut counts.replaced,
6126 ScopeRecordOutcome::Updated => &mut counts.updated,
6127 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
6128 ScopeRecordOutcome::Refused => &mut counts.refused,
6129 };
6130 *slot += 1;
6131 }
6132 counts
6133 }
6134}
6135
6136#[cfg(test)]
6137mod scope_outcome_count_tests {
6138 use super::*;
6139
6140 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
6141 ScopeRecordResult {
6142 scope_ref: "r".to_string(),
6143 scope_epoch: 1,
6144 outcome,
6145 code: None,
6146 message: None,
6147 version: None,
6148 parent_state: None,
6149 }
6150 }
6151
6152 #[test]
6155 fn every_outcome_is_counted_in_its_own_field() {
6156 let results = [
6157 result(ScopeRecordOutcome::Created),
6158 result(ScopeRecordOutcome::Created),
6159 result(ScopeRecordOutcome::Replaced),
6160 result(ScopeRecordOutcome::Updated),
6161 result(ScopeRecordOutcome::Unchanged),
6162 result(ScopeRecordOutcome::Refused),
6163 result(ScopeRecordOutcome::Refused),
6164 result(ScopeRecordOutcome::Refused),
6165 ];
6166 assert_eq!(
6167 ScopeOutcomeCounts::of(&results),
6168 ScopeOutcomeCounts {
6169 created: 2,
6170 replaced: 1,
6171 updated: 1,
6172 unchanged: 1,
6173 refused: 3,
6174 }
6175 );
6176 }
6177}
6178
6179fn capability_census_trigger(
6183 old: Option<&CapabilityDeclarations>,
6184 new: Option<&CapabilityDeclarations>,
6185) -> bool {
6186 let old_provides = old
6187 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
6188 .unwrap_or_default();
6189 let old_denies = old
6190 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
6191 .unwrap_or_default();
6192 let new = new.cloned().unwrap_or(CapabilityDeclarations {
6193 provides: Vec::new(),
6194 requires: Vec::new(),
6195 must_never_reach: Vec::new(),
6196 });
6197
6198 new.provides
6199 .iter()
6200 .any(|capability| !old_provides.contains(capability))
6201 || new
6202 .must_never_reach
6203 .iter()
6204 .any(|capability| !old_denies.contains(capability))
6205}
6206
6207fn denied_capability<'a>(
6210 opening_manifest: &'a ModuleManifest,
6211 target_manifest: &ModuleManifest,
6212) -> Option<&'a str> {
6213 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
6214 let target_capabilities = target_manifest.capabilities.as_ref()?;
6215 opening_capabilities
6216 .must_never_reach
6217 .iter()
6218 .find(|denied| {
6219 target_capabilities
6220 .provides
6221 .iter()
6222 .any(|provided| provided == *denied)
6223 })
6224 .map(String::as_str)
6225}
6226
6227fn catalog_update_frozen_field_message(
6228 registered: &ModuleManifest,
6229 provides: &[ProviderRole],
6230) -> Option<String> {
6231 let old_has_provides = !registered.provides.is_empty();
6232 let new_has_provides = !provides.is_empty();
6233 if old_has_provides != new_has_provides {
6234 return Some(format!(
6235 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
6236 registered.module_id
6237 ));
6238 }
6239
6240 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
6241 return Some(format!(
6242 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
6243 registered.module_id
6244 ));
6245 }
6246
6247 let registered_concurrency = manifest_concurrency(registered);
6248 let mut candidate = registered.clone();
6249 candidate.provides = provides.to_vec();
6250 let candidate_concurrency = manifest_concurrency(&candidate);
6251 if candidate_concurrency != registered_concurrency {
6252 return Some(format!(
6253 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
6254 registered.module_id, registered_concurrency, candidate_concurrency
6255 ));
6256 }
6257
6258 None
6261}
6262
6263fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
6264 manifest.provides.iter().any(is_routable_role)
6265}
6266
6267fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
6273 manifest
6274 .provides
6275 .iter()
6276 .find_map(|provider| match provider {
6277 ProviderRole::ToolProvider { concurrency, .. }
6278 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
6279 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
6280 })
6281 .unwrap_or(Concurrency::ModuleManaged)
6282}
6283
6284fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
6291 let has_management_surface = manifest
6292 .provides
6293 .iter()
6294 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
6295 if !has_management_surface {
6296 return false;
6297 }
6298 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
6299 return false;
6300 };
6301 let Some(provides) = raw
6302 .get("manifest")
6303 .and_then(|manifest| manifest.get("provides"))
6304 .and_then(serde_json::Value::as_array)
6305 else {
6306 return false;
6307 };
6308 provides.iter().any(|role| {
6313 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
6314 && role.get("concurrency").is_none()
6315 })
6316}
6317
6318fn negotiate_version(peer_version: u8) -> Result<u8, String> {
6319 if peer_version != PROTOCOL_VERSION {
6320 return Err(format!(
6321 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
6322 ));
6323 }
6324 Ok(PROTOCOL_VERSION)
6325}
6326
6327fn pong(frame: &Frame) -> Result<Frame, RouterError> {
6328 Frame::build_with_version(
6329 response_version(frame),
6330 FrameType::Pong,
6331 frame.header.flags,
6332 0,
6333 0,
6334 frame.header.corr,
6335 Vec::new(),
6336 )
6337 .map_err(RouterError::FrameBuild)
6338}
6339
6340fn control_error_frame(
6341 frame: &Frame,
6342 code: &'static str,
6343 message: impl Into<String>,
6344) -> Result<Frame, RouterError> {
6345 control_error_body_frame(
6346 frame,
6347 ErrorBody {
6348 code: code.to_string(),
6349 message: message.into(),
6350 detail: None,
6351 },
6352 )
6353}
6354
6355fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
6356 let body = serde_json::to_vec(&error).map_err(|err| {
6357 RouterError::backend(
6358 0,
6359 frame.header.corr,
6360 format!("failed to encode control ERROR: {err}"),
6361 )
6362 })?;
6363
6364 Frame::build_with_version(
6365 response_version(frame),
6366 FrameType::Error,
6367 control_flags(),
6368 0,
6369 0,
6370 frame.header.corr,
6371 body,
6372 )
6373 .map_err(RouterError::FrameBuild)
6374}
6375
6376fn control_response_body_frame<T: Serialize>(
6377 frame: &Frame,
6378 reply: &T,
6379 label: &'static str,
6380) -> Result<Frame, RouterError> {
6381 let body = serde_json::to_vec(reply).map_err(|err| {
6382 RouterError::backend(
6383 0,
6384 frame.header.corr,
6385 format!("failed to encode {label}: {err}"),
6386 )
6387 })?;
6388
6389 Frame::build_with_version(
6390 response_version(frame),
6391 FrameType::Response,
6392 control_flags(),
6393 0,
6394 0,
6395 frame.header.corr,
6396 body,
6397 )
6398 .map_err(RouterError::FrameBuild)
6399}
6400
6401fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6431 match err {
6432 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6433 ForwardingError::NoModuleConnection => "target_unavailable",
6434 ForwardingError::ModuleReloading { .. } => "module_reloading",
6435 ForwardingError::ClientRouteChannelExhausted { .. }
6436 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6437 ForwardingError::StaleModuleEndpoint
6438 | ForwardingError::UnknownReservation { .. }
6439 | ForwardingError::ConnectionClosing { .. }
6440 | ForwardingError::ClientEgressClosed { .. }
6441 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
6442 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
6445 ForwardingError::RelayCorrelationExhausted
6446 | ForwardingError::RouteOpenBuild(_)
6447 | ForwardingError::Poisoned => "forwarding_error",
6448 }
6449}
6450
6451fn response_version(frame: &Frame) -> u8 {
6452 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
6453 frame.header.ver
6454 } else {
6455 PROTOCOL_VERSION
6456 }
6457}
6458
6459fn control_flags() -> Flags {
6460 Flags::new(false, Priority::Passive, false)
6461}
6462
6463fn send_goodbye_target_best_effort(
6468 counters: &DaemonCounters,
6469 target: &GoodbyeTarget,
6470 context: &'static str,
6471) {
6472 let Ok(frame) = Frame::build_with_version(
6473 target.negotiated_ver,
6474 FrameType::Goodbye,
6475 control_flags(),
6476 target.channel,
6477 target.epoch,
6478 0,
6479 Vec::new(),
6480 ) else {
6481 return;
6482 };
6483 crate::forwarding::send_module_route_goodbye(
6484 counters,
6485 &target.sink,
6486 frame,
6487 target.module_id.as_deref(),
6488 context,
6489 );
6490}
6491
6492pub(crate) fn send_route_control_pushes(
6493 forwarding: &ForwardingTable,
6494 routes: Vec<EndpointRoute>,
6495 push: ClientControlPush,
6496) {
6497 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
6498 for route in routes {
6499 let target = route.goodbye_target;
6500 if let Some((existing, channels)) = targets
6501 .iter_mut()
6502 .find(|(existing, _)| existing.connection_id == target.connection_id)
6503 {
6504 debug_assert_eq!(
6505 existing.negotiated_ver, target.negotiated_ver,
6506 "one connection cannot negotiate multiple frame versions"
6507 );
6508 if !channels.contains(&target.channel) {
6509 channels.push(target.channel);
6510 }
6511 continue;
6512 }
6513 let channel = target.channel;
6514 targets.push((target, vec![channel]));
6515 }
6516 for (target, mut channels) in targets {
6517 channels.sort_unstable();
6518 let mut push = push.clone();
6519 match &mut push {
6520 ClientControlPush::RouteClosing {
6521 channels: covered, ..
6522 }
6523 | ClientControlPush::RouteClosed {
6524 channels: covered, ..
6525 } => *covered = channels,
6526 }
6527 let body = match serde_json::to_vec(&push) {
6528 Ok(body) => body,
6529 Err(err) => {
6530 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
6531 continue;
6532 }
6533 };
6534 let frame = match Frame::build_with_version(
6535 target.negotiated_ver,
6536 FrameType::Push,
6537 control_flags(),
6538 0,
6539 0,
6540 0,
6541 body.clone(),
6542 ) {
6543 Ok(frame) => frame,
6544 Err(err) => {
6545 warn!(
6546 route_channel = target.channel,
6547 error = %err,
6548 "failed to build route lifecycle control PUSH frame"
6549 );
6550 continue;
6551 }
6552 };
6553 if let Err(err) = target.sink.try_send(frame) {
6554 if target.close_on_delivery_failure() {
6555 warn!(
6556 target_connection_id = target.connection_id.get(),
6557 route_channel = target.channel,
6558 error = %err,
6559 "route lifecycle control PUSH was not delivered to client; closing target connection"
6560 );
6561 let _ = forwarding.escalate_client_delivery_failure(
6562 target.connection_id,
6563 target.channel,
6564 target.epoch,
6565 CloseReason::new(
6566 "route_lifecycle_push_delivery_failed",
6567 format!(
6568 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6569 target.channel
6570 ),
6571 ),
6572 crate::forwarding::UndeliveredFrame {
6573 module_id: target.module_id.as_deref(),
6574 sink: &target.sink,
6575 },
6576 );
6577 }
6578 }
6579 }
6580}
6581
6582#[cfg(test)]
6583mod tests {
6584 #[cfg(unix)]
6585 #[tokio::test]
6586 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
6587 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
6588 let path = dir.join("subc.jsonc");
6589 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
6590 "program":"/bin/sleep","args":["60"],"protocol":"none",
6591 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
6592 }}}).to_string()).unwrap();
6593 let mut configured = crate::daemon_config::load(&path)
6594 .unwrap()
6595 .unwrap()
6596 .modules
6597 .pop()
6598 .unwrap();
6599 let registry = std::sync::Arc::new(crate::Registry::default());
6600 let handle = crate::SupervisorHandle::new();
6601 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
6602 .with_handle(handle.clone());
6603 let module = supervisor
6604 .supervise_configured_with_health(
6605 configured.module_spec(),
6606 true,
6607 configured.health.clone(),
6608 None,
6609 configured.restart,
6610 )
6611 .unwrap();
6612 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
6613 let before = module.status().unwrap().pid;
6614 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
6615 configured.health.cadence = std::time::Duration::from_secs(3600);
6616 let health_only = handler
6617 .reconcile_supervised_modules(&supervisor, vec![configured.clone()], false)
6618 .await
6619 .unwrap();
6620 assert!(
6621 health_only.changed_pending_reload.is_empty(),
6622 "health policy is already applied live"
6623 );
6624 assert_eq!(module.status().unwrap().pid, before);
6625 assert_eq!(
6626 module.configuration().unwrap().1.http,
6627 configured.health.http
6628 );
6629 configured.args = vec!["61".into()];
6630 let launch = handler
6631 .reconcile_supervised_modules(&supervisor, vec![configured], false)
6632 .await
6633 .unwrap();
6634 assert_eq!(launch.changed_pending_reload, ["stock"]);
6635 assert_eq!(
6636 module.status().unwrap().pid,
6637 before,
6638 "a launch edit is stored until reload"
6639 );
6640 module.drain().await.unwrap();
6641 }
6642 use cortexkit_test_support::ScratchDir;
6643 use std::{
6644 collections::BTreeMap,
6645 fmt,
6646 path::PathBuf,
6647 sync::{Arc, Mutex},
6648 time::Duration,
6649 };
6650
6651 use serde_json::{json, Value};
6652 use subc_protocol::{
6653 manifest::{
6654 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6655 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6656 },
6657 session::HealthStatus,
6658 FrameType,
6659 };
6660
6661 use super::*;
6662 use crate::{
6663 forwarding::{DataRoute, DataRouteState},
6664 registry::ChannelState,
6665 router::FrameSink,
6666 stderr_tail::DEFAULT_MAX_LINE_BYTES,
6667 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6668 RouteCtx, Router,
6669 };
6670 use tokio::{
6671 sync::mpsc,
6672 time::{sleep, Instant},
6673 };
6674 use tracing::{
6675 field::{Field, Visit},
6676 Event, Subscriber,
6677 };
6678 use tracing_subscriber::{layer::Context, prelude::*, Layer};
6679
6680 fn fake_aft_stub_path() -> PathBuf {
6699 let mut path = std::env::current_exe().expect("current_exe available in tests");
6700 path.pop(); path.pop(); path.push(if cfg!(windows) {
6703 "fake-aft-stub.exe"
6704 } else {
6705 "fake-aft-stub"
6706 });
6707 assert!(
6708 path.exists(),
6709 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6710 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6711 path.display()
6712 );
6713 path
6714 }
6715
6716 fn client_retries(code: &str) -> bool {
6721 subc_protocol::error_codes::is_retryable_route_open(code)
6722 }
6723
6724 #[test]
6735 fn retryability_of_forwarding_codes_matches_the_failure() {
6736 let transient = [
6739 ForwardingError::NoModuleConnection,
6740 ForwardingError::ModuleReloading {
6741 module_id: "m".into(),
6742 },
6743 ForwardingError::StaleModuleEndpoint,
6744 ForwardingError::UnknownReservation {
6745 client_channel: 1,
6746 module_channel: 1,
6747 },
6748 ForwardingError::ConnectionClosing {
6749 connection_id: ConnectionId::new(1),
6750 },
6751 ForwardingError::ClientEgressClosed {
6752 connection_id: ConnectionId::new(1),
6753 },
6754 ForwardingError::ModuleEgressUnavailable {
6755 connection_id: ConnectionId::new(1),
6756 },
6757 ];
6758 for err in transient {
6759 let code = forwarding_error_code(&err);
6760 assert!(
6761 client_retries(code),
6762 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6763 );
6764 }
6765
6766 let permanent = [
6771 ForwardingError::ConnectionRoleConflict {
6772 connection_id: ConnectionId::new(1),
6773 },
6774 ForwardingError::ClientRouteChannelExhausted {
6775 connection_id: ConnectionId::new(1),
6776 },
6777 ForwardingError::ModuleRouteChannelExhausted {
6778 endpoint: ModuleEndpointId {
6779 connection_id: ConnectionId::new(1),
6780 generation: 1,
6781 },
6782 },
6783 ForwardingError::RelayCorrelationExhausted,
6784 ForwardingError::RouteOpenBuild("x".into()),
6785 ForwardingError::Poisoned,
6786 ];
6787 for err in permanent {
6788 let code = forwarding_error_code(&err);
6789 assert!(
6790 !client_retries(code),
6791 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6792 );
6793 }
6794 }
6795
6796 #[tokio::test]
6824 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6825 let handler = ControlHandler::default();
6826 let frame =
6827 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6828
6829 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6833 assert_eq!(
6834 stamped,
6835 Principal::Direct,
6836 "a caller that proved nothing must not be stamped as a supervised module"
6837 );
6838
6839 let forged = handler
6844 .route_open_principal(
6845 &frame,
6846 Some(ConsumerIdentity {
6847 module_id: "aft".to_string(),
6848 launch_nonce: "not-a-real-nonce".to_string(),
6849 }),
6850 )
6851 .unwrap();
6852 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6853 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6854 }
6855
6856 #[test]
6873 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6874 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6875 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6876 let ClientControlRequest::RouteOpen {
6877 consumer_identity, ..
6878 } = parsed
6879 else {
6880 panic!("route.open body must parse as RouteOpen");
6881 };
6882 assert_eq!(
6883 consumer_identity,
6884 Some(ConsumerIdentity {
6885 module_id: "aft".to_string(),
6886 launch_nonce: "n".to_string(),
6887 }),
6888 "the wire field name must reach the value route_open_principal reads"
6889 );
6890 }
6891
6892 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6893 ModuleManifest::builder(module_id, "0.1.0")
6894 .protocol_ver(protocol_ver)
6895 .provides(vec![ProviderRole::ToolProvider {
6896 tools: vec![Tool {
6897 name: "read".to_string(),
6898 description: None,
6899 execution_mode: ExecutionMode::Pure,
6900 schema: json!({"type": "object"}),
6901 }],
6902 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6903 concurrency: Concurrency::ModuleManaged,
6904 emits_push: true,
6905 sub_supervises: true,
6906 }])
6907 .build()
6908 }
6909
6910 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6911 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6912 }
6913
6914 fn hello_frame_with_control_ops(
6915 module_id: &str,
6916 protocol_ver: u8,
6917 corr: u64,
6918 control_ops: Option<Vec<String>>,
6919 ) -> Frame {
6920 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6921 }
6922
6923 fn hello_frame_with_nonce(
6924 module_id: &str,
6925 protocol_ver: u8,
6926 corr: u64,
6927 launch_nonce: Option<&str>,
6928 ) -> Frame {
6929 hello_frame_full(
6930 module_id,
6931 protocol_ver,
6932 corr,
6933 None,
6934 launch_nonce.map(ToOwned::to_owned),
6935 )
6936 }
6937
6938 fn hello_frame_full(
6939 module_id: &str,
6940 protocol_ver: u8,
6941 corr: u64,
6942 control_ops: Option<Vec<String>>,
6943 launch_nonce: Option<String>,
6944 ) -> Frame {
6945 let body = serde_json::to_vec(&ModuleHelloBody {
6946 manifest: manifest(module_id, protocol_ver),
6947 protocol_ver,
6948 control_ops,
6949 launch_nonce,
6950 })
6951 .unwrap();
6952 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6953 }
6954
6955 fn non_routable_hello_frame_with_control_ops(
6956 module_id: &str,
6957 corr: u64,
6958 control_ops: Option<Vec<String>>,
6959 ) -> Frame {
6960 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6961 manifest.provides.clear();
6962 let body = serde_json::to_vec(&ModuleHelloBody {
6963 manifest,
6964 protocol_ver: PROTOCOL_VERSION,
6965 control_ops,
6966 launch_nonce: None,
6967 })
6968 .unwrap();
6969 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6970 }
6971
6972 fn capability_grammar_hello_frame(
6973 capabilities: Value,
6974 runtime_computed: Option<Value>,
6975 corr: u64,
6976 ) -> Frame {
6977 let mut body = serde_json::to_value(ModuleHelloBody {
6978 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
6979 protocol_ver: PROTOCOL_VERSION,
6980 control_ops: None,
6981 launch_nonce: None,
6982 })
6983 .expect("HELLO body serializes");
6984 body["manifest"]["capabilities"] = capabilities;
6985 if let Some(runtime_computed) = runtime_computed {
6986 body["runtime_computed"] = runtime_computed;
6987 }
6988 Frame::build(
6989 FrameType::Hello,
6990 control_flags(),
6991 0,
6992 0,
6993 corr,
6994 serde_json::to_vec(&body).expect("HELLO body reserializes"),
6995 )
6996 .expect("HELLO frame builds")
6997 }
6998
6999 fn channel_request(channel: u16, corr: u64) -> Frame {
7000 Frame::build(
7001 FrameType::Request,
7002 Flags::new(true, Priority::Interactive, false),
7003 channel,
7004 0,
7005 corr,
7006 b"opaque".to_vec(),
7007 )
7008 .unwrap()
7009 }
7010
7011 fn route_ctx(
7012 connection_id: ConnectionId,
7013 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7014 let (tx, rx) = mpsc::channel(8);
7015 (
7016 RouteCtx {
7017 connection_id,
7018 egress: FrameSink::new(tx),
7019 },
7020 rx,
7021 )
7022 }
7023
7024 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7025 serde_json::from_slice(&frame.body).unwrap()
7026 }
7027
7028 async fn hello_via_sink(
7033 handler: &ControlHandler,
7034 ctx: &RouteCtx,
7035 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7036 hello: Frame,
7037 ) -> Frame {
7038 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7039 assert!(
7040 replies.is_empty(),
7041 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7042 );
7043 let ack = rx
7044 .try_recv()
7045 .expect("HELLO_ACK is queued on the module sink")
7046 .frame;
7047 assert_eq!(ack.header.ty, FrameType::HelloAck);
7048 ack
7049 }
7050
7051 fn parse_error(frame: &Frame) -> Value {
7052 serde_json::from_slice(&frame.body).unwrap()
7053 }
7054
7055 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7056 serde_json::from_slice(&frame.body).unwrap()
7057 }
7058
7059 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7060 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7061 route_channel,
7062 route_epoch: 0,
7063 kind,
7064 })
7065 .unwrap();
7066 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7067 }
7068
7069 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7070 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7071 module_id: module_id.to_string(),
7072 })
7073 .unwrap();
7074 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7075 }
7076
7077 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7078 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7079 }
7080
7081 fn route_open_frame_with_consumer_capabilities(
7082 corr: u64,
7083 module_id: &str,
7084 project_root: ScratchDir,
7085 consumer_capabilities: Option<Vec<String>>,
7086 ) -> Frame {
7087 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7088 target: RouteTarget::ToolProvider {
7089 module_id: module_id.to_string(),
7090 },
7091 identity: BindIdentity::new(
7092 project_root.path().to_path_buf(),
7093 "unit".to_string(),
7094 "session".to_string(),
7095 ),
7096 consumer_identity: None,
7097 consumer_capabilities,
7098 role_versions: None,
7099 admission_facts: None,
7100 scope: None,
7101 })
7102 .unwrap();
7103 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7104 }
7105
7106 fn route_open_frame_with_role_versions(
7107 corr: u64,
7108 module_id: &str,
7109 project_root: ScratchDir,
7110 role_versions: Option<BTreeMap<String, String>>,
7111 ) -> Frame {
7112 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7113 target: RouteTarget::ToolProvider {
7114 module_id: module_id.to_string(),
7115 },
7116 identity: BindIdentity::new(
7117 project_root.path().to_path_buf(),
7118 "unit".to_string(),
7119 format!("session-{corr}"),
7120 ),
7121 consumer_identity: None,
7122 consumer_capabilities: None,
7123 role_versions,
7124 admission_facts: None,
7125 scope: None,
7126 })
7127 .unwrap();
7128 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7129 }
7130
7131 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
7132 entries
7133 .iter()
7134 .map(|(role, version)| (role.to_string(), version.to_string()))
7135 .collect()
7136 }
7137
7138 fn route_open_frame_with_admission_facts(
7139 corr: u64,
7140 module_id: &str,
7141 project_root: ScratchDir,
7142 consumer_identity: Option<subc_control::ConsumerIdentity>,
7143 facts: Option<Value>,
7144 ) -> Frame {
7145 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7146 target: RouteTarget::ToolProvider {
7147 module_id: module_id.to_string(),
7148 },
7149 identity: BindIdentity::new(
7150 project_root.path().to_path_buf(),
7151 "unit".to_string(),
7152 format!("session-{corr}"),
7153 ),
7154 consumer_identity,
7155 consumer_capabilities: None,
7156 role_versions: None,
7157 admission_facts: facts,
7158 scope: None,
7159 })
7160 .unwrap();
7161 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7162 }
7163
7164 #[derive(Clone, Default)]
7165 struct EventCapture {
7166 events: Arc<Mutex<Vec<CapturedEvent>>>,
7167 }
7168
7169 #[derive(Clone, Debug)]
7170 struct CapturedEvent {
7171 target: String,
7172 level: tracing::Level,
7173 fields: BTreeMap<String, String>,
7174 }
7175
7176 impl EventCapture {
7177 fn events(&self) -> Vec<CapturedEvent> {
7178 self.events.lock().unwrap().clone()
7179 }
7180 }
7181
7182 impl<S> Layer<S> for EventCapture
7183 where
7184 S: Subscriber,
7185 {
7186 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
7187 let mut visitor = EventFieldVisitor::default();
7188 event.record(&mut visitor);
7189 self.events.lock().unwrap().push(CapturedEvent {
7190 target: event.metadata().target().to_string(),
7191 level: *event.metadata().level(),
7192 fields: visitor.fields,
7193 });
7194 }
7195 }
7196
7197 #[derive(Default)]
7198 struct EventFieldVisitor {
7199 fields: BTreeMap<String, String>,
7200 }
7201
7202 impl Visit for EventFieldVisitor {
7203 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
7204 self.fields
7205 .insert(field.name().to_string(), format!("{value:?}"));
7206 }
7207 }
7208
7209 fn health_response(corr: u64, status: HealthStatus) -> Frame {
7210 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
7211 status,
7212 detail: Some("warming".to_string()),
7213 metrics: Some(json!({"queue_depth": 3})),
7214 })
7215 .unwrap();
7216 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7217 }
7218
7219 fn route_bind_ack(corr: u64) -> Frame {
7220 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
7221 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7222 }
7223
7224 fn unique_project_root(label: &str) -> ScratchDir {
7225 ScratchDir::new(label)
7226 }
7227
7228 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
7229 match parse_route_poll(frame) {
7230 ClientControlResponse::RoutePoll {
7231 status: None,
7232 live: Some(live),
7233 ..
7234 } => assert_eq!(live, expected_live),
7235 other => panic!("unexpected route.poll response: {other:?}"),
7236 }
7237 }
7238
7239 fn bind_liveness_route(
7240 registry: &Registry,
7241 forwarding: &ForwardingTable,
7242 module_id: &str,
7243 ) -> (RouteCtx, u16, u32) {
7244 let module_connection = ConnectionId::new(101);
7245 let client_connection = ConnectionId::new(202);
7246 let registration = registry
7247 .register_with_control_ops(
7248 manifest(module_id, PROTOCOL_VERSION),
7249 PROTOCOL_VERSION,
7250 module_connection,
7251 module_baseline_control_ops(),
7252 )
7253 .unwrap();
7254 let (module_tx, _module_rx) = mpsc::channel(8);
7255 let endpoint = forwarding
7256 .register_module_connection(
7257 module_connection,
7258 module_id.to_string(),
7259 PROTOCOL_VERSION,
7260 manifest_concurrency(®istration.manifest),
7261 FrameSink::new(module_tx),
7262 )
7263 .unwrap();
7264 let (client_ctx, _client_rx) = route_ctx(client_connection);
7265 let pending = forwarding
7266 .begin_route_bind_relay_for_test(
7267 client_connection,
7268 client_ctx.egress.clone(),
7269 1,
7270 module_id,
7271 )
7272 .unwrap();
7273 assert_eq!(pending.endpoint, endpoint);
7274 let route_channel = pending.client_channel;
7275 let route_epoch = pending.client_epoch;
7276 forwarding
7277 .complete_pending_relay(
7278 module_connection,
7279 pending.corr,
7280 RouteBindRelayOutcome::Accepted,
7281 )
7282 .unwrap();
7283 (client_ctx, route_channel, route_epoch)
7284 }
7285
7286 struct FakeProcessLiveness {
7287 live: Option<bool>,
7288 }
7289
7290 impl ModuleProcessLiveness for FakeProcessLiveness {
7291 fn process_live(&self, _module_id: &str) -> Option<bool> {
7292 self.live
7293 }
7294 }
7295
7296 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7297 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
7298 {
7299 let registry = Arc::new(Registry::default());
7300 let supervisor_handle = SupervisorHandle::new();
7301 let supervisor = Supervisor::new_for_test(
7302 Arc::clone(®istry),
7303 RestartPolicy::new(1, Duration::from_millis(10)),
7304 )
7305 .with_handle(supervisor_handle.clone());
7306 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
7307 let module = supervisor
7308 .spawn(ModuleSpec {
7309 module_id: "stderr-tail-wire".to_string(),
7310 program: fake_aft_stub_path(),
7311 args: Vec::new(),
7312 env: vec![
7313 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
7314 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
7315 ],
7316 reserved: false,
7317 reserved_prefixes: Vec::new(),
7318 protocol: ModuleProtocol::Subc,
7319 overlap: Default::default(),
7320 })
7321 .unwrap();
7322
7323 let deadline = Instant::now() + Duration::from_secs(5);
7324 loop {
7325 let tail = module.stderr_tail(None, None);
7326 if tail
7327 .entries
7328 .iter()
7329 .any(|entry| matches!(entry, TailEntry::ProcessStart))
7330 && tail.entries.iter().any(|entry| {
7331 matches!(
7332 entry,
7333 TailEntry::Line {
7334 truncated: true,
7335 ..
7336 }
7337 )
7338 })
7339 {
7340 break;
7341 }
7342 assert!(
7343 Instant::now() < deadline,
7344 "module did not produce a truncated line and restart boundary: {tail:?}"
7345 );
7346 sleep(Duration::from_millis(10)).await;
7347 }
7348
7349 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7350 let request = ClientControlRequest::SupervisorStderrTail {
7351 module_id: "stderr-tail-wire".to_string(),
7352 max_lines: None,
7353 max_bytes: None,
7354 };
7355 let frame = Frame::build(
7356 FrameType::Request,
7357 control_flags(),
7358 0,
7359 0,
7360 1,
7361 serde_json::to_vec(&request).unwrap(),
7362 )
7363 .unwrap();
7364 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7365 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7366 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
7367 serde_json::from_slice(&responses[0].body).unwrap()
7368 else {
7369 panic!("expected supervisor.stderr_tail response");
7370 };
7371
7372 assert!(
7373 tail.entries
7374 .iter()
7375 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
7376 "the control response lost the restart boundary"
7377 );
7378 let Some(StderrTailEntry::Line {
7379 text,
7380 truncated,
7381 at_ms,
7382 }) = tail.entries.iter().find(|entry| {
7383 matches!(
7384 entry,
7385 StderrTailEntry::Line {
7386 truncated: true,
7387 ..
7388 }
7389 )
7390 })
7391 else {
7392 panic!("the control response lost the truncated line");
7393 };
7394 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
7395 assert!(*truncated);
7396 assert!(
7397 at_ms.is_some(),
7398 "the control response lost the line's capture time"
7399 );
7400 }
7401
7402 #[tokio::test(flavor = "current_thread")]
7407 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
7408 let dir = ScratchDir::new("terminals-off-worker");
7409 let journal_path = dir.join("terminals.jsonl");
7410 let registry = Arc::new(Registry::default());
7411 let supervisor_handle = SupervisorHandle::new();
7412 let supervisor =
7413 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7414 .with_handle(supervisor_handle.clone())
7415 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7416 let module = supervisor
7417 .spawn(ModuleSpec {
7418 module_id: "terminal-off-worker".to_string(),
7419 program: fake_aft_stub_path(),
7420 args: Vec::new(),
7421 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7422 reserved: false,
7423 reserved_prefixes: Vec::new(),
7424 protocol: ModuleProtocol::Subc,
7425 overlap: Default::default(),
7426 })
7427 .unwrap();
7428 let deadline = Instant::now() + Duration::from_secs(5);
7429 while module.terminal_history().entries.len() != 2 {
7430 assert!(Instant::now() < deadline, "module did not record two exits");
7431 sleep(Duration::from_millis(10)).await;
7432 }
7433
7434 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
7435 let handler =
7436 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
7437 let frame = Frame::build(
7438 FrameType::Request,
7439 control_flags(),
7440 0,
7441 0,
7442 1,
7443 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
7444 module_id: "terminal-off-worker".to_string(),
7445 })
7446 .unwrap(),
7447 )
7448 .unwrap();
7449 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7450 let spawned_at = std::time::Instant::now();
7451 let read = tokio::spawn({
7452 let handler = Arc::clone(&handler);
7453 async move { handler.handle_control_frame(&ctx, frame).await }
7454 });
7455 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
7458 .await
7459 .unwrap()
7460 .expect("the history read reached its pause");
7461 let elapsed = spawned_at.elapsed();
7462 assert!(
7463 elapsed < Duration::from_secs(2) && !read.is_finished(),
7464 "this task could not run while the history read was paused \
7465 (resumed after {elapsed:?}, read finished: {})",
7466 read.is_finished()
7467 );
7468
7469 drop(release);
7470 let responses = read.await.unwrap().unwrap();
7471 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7472 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
7473 panic!("expected supervisor.terminals response");
7474 };
7475 assert_eq!(terminals.entries.len(), 2);
7476 assert_eq!(terminals.journal_skipped_lines, 0);
7477 assert_eq!(terminals.journal_read_errors, 0);
7478 }
7479
7480 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7481 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
7482 let registry = Arc::new(Registry::default());
7483 let supervisor_handle = SupervisorHandle::new();
7484 let supervisor =
7485 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7486 .with_handle(supervisor_handle.clone());
7487 let module = supervisor
7488 .spawn(ModuleSpec {
7489 module_id: "terminal-golden".to_string(),
7490 program: fake_aft_stub_path(),
7491 args: Vec::new(),
7492 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7493 reserved: false,
7494 reserved_prefixes: Vec::new(),
7495 protocol: ModuleProtocol::Subc,
7496 overlap: Default::default(),
7497 })
7498 .unwrap();
7499
7500 let deadline = Instant::now() + Duration::from_secs(5);
7501 while module.terminal_history().entries.len() != 2 {
7502 assert!(
7503 Instant::now() < deadline,
7504 "module did not retain two terminal exits: {:?}",
7505 module.terminal_history()
7506 );
7507 sleep(Duration::from_millis(10)).await;
7508 }
7509
7510 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7511 let request = ClientControlRequest::SupervisorTerminals {
7512 module_id: "terminal-golden".to_string(),
7513 };
7514 let frame = Frame::build(
7515 FrameType::Request,
7516 control_flags(),
7517 0,
7518 0,
7519 1,
7520 serde_json::to_vec(&request).unwrap(),
7521 )
7522 .unwrap();
7523 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7524 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7525 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7526 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
7527 panic!("expected supervisor.terminals response");
7528 };
7529 assert_eq!(terminals.entries.len(), 2);
7530 assert_eq!(terminals.dropped, 0);
7531
7532 let mut rendered = serde_json::to_value(response).unwrap();
7533 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
7536 for (index, entry) in rendered["entries"]
7537 .as_array_mut()
7538 .expect("terminal response entries array")
7539 .iter_mut()
7540 .enumerate()
7541 {
7542 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
7543 }
7544
7545 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7546 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
7547 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
7548 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7549 std::fs::write(&golden_path, &serialized).unwrap();
7550 }
7551 let expected: Value =
7552 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
7553 assert_eq!(rendered, expected);
7554 }
7555
7556 #[test]
7557 fn hello_registers_manifest_and_returns_ack() {
7558 let registry = Arc::new(Registry::default());
7559 let handler = ControlHandler::new(Arc::clone(®istry));
7560 let conn = ConnectionId::new(1);
7561
7562 let responses = handler
7563 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
7564 .unwrap();
7565
7566 assert_eq!(responses.len(), 1);
7567 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7568 assert_eq!(responses[0].header.channel, 0);
7569 assert_eq!(responses[0].header.corr, 7);
7570 let ack = parse_ack(&responses[0]);
7571 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
7572 assert!(ack
7573 .subc_capabilities
7574 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
7575 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
7576 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
7577 assert!(ack
7578 .subc_ops
7579 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
7580 assert!(ack
7581 .subc_ops
7582 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
7583
7584 let registration = registry.get_module("aft").unwrap().unwrap();
7585 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
7586 assert_eq!(registration.state, ChannelState::Active);
7587 assert_eq!(registration.connection_id, conn);
7588 assert_eq!(registration.control_ops, module_baseline_control_ops());
7589 }
7590
7591 #[test]
7592 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7593 let invalid_identifiers = [
7594 ("case_change", "credentials-Provider/v1"),
7595 ("leading_zero", "credentials-provider/v01"),
7596 ("trailing_hyphen", "credentials-provider-/v1"),
7597 ("consecutive_hyphens", "credentials--provider/v1"),
7598 ("uppercase", "Credentials-provider/v1"),
7599 ("missing_v", "credentials-provider/1"),
7600 ("whitespace", "credentials provider/v1"),
7601 ("zero_version", "credentials-provider/v0"),
7602 ("out_of_range_version", "credentials-provider/v4294967296"),
7603 (
7604 "overlength_name",
7605 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7606 ),
7607 ];
7608 let mut cases = invalid_identifiers
7609 .into_iter()
7610 .map(|(name, identifier)| {
7611 (
7612 format!("identifier_{name}"),
7613 "capabilities.provides[0]".to_string(),
7614 identifier.to_string(),
7615 json!({ "provides": [identifier] }),
7616 None,
7617 )
7618 })
7619 .collect::<Vec<_>>();
7620 cases.extend([
7621 (
7622 "unknown_need".to_string(),
7623 "capabilities.requires[0].need".to_string(),
7624 "deferred".to_string(),
7625 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7626 None,
7627 ),
7628 (
7629 "duplicate_provides".to_string(),
7630 "capabilities.provides[1]".to_string(),
7631 "credentials-provider/v1".to_string(),
7632 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7633 None,
7634 ),
7635 (
7636 "duplicate_must_never_reach".to_string(),
7637 "capabilities.must_never_reach[1]".to_string(),
7638 "credentials-provider/v1".to_string(),
7639 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7640 None,
7641 ),
7642 (
7643 "duplicate_requires_same_need".to_string(),
7644 "capabilities.requires[1]".to_string(),
7645 "credentials-provider/v1".to_string(),
7646 json!({ "requires": [
7647 { "capability": "credentials-provider/v1", "need": "required" },
7648 { "capability": "credentials-provider/v1", "need": "required" }
7649 ] }),
7650 None,
7651 ),
7652 (
7653 "duplicate_requires_conflicting_need".to_string(),
7654 "capabilities.requires[1]".to_string(),
7655 "credentials-provider/v1".to_string(),
7656 json!({ "requires": [
7657 { "capability": "credentials-provider/v1", "need": "required" },
7658 { "capability": "credentials-provider/v1", "need": "optional" }
7659 ] }),
7660 None,
7661 ),
7662 (
7663 "capabilities_root_pointer".to_string(),
7664 "runtime_computed[0]".to_string(),
7665 "/capabilities".to_string(),
7666 json!({}),
7667 Some(json!(["/capabilities"])),
7668 ),
7669 (
7670 "capabilities_descendant_pointer".to_string(),
7671 "runtime_computed[0]".to_string(),
7672 "/capabilities/provides".to_string(),
7673 json!({}),
7674 Some(json!(["/capabilities/provides"])),
7675 ),
7676 (
7677 "malformed_pointer_without_leading_slash".to_string(),
7678 "runtime_computed[0]".to_string(),
7679 "capabilities".to_string(),
7680 json!({}),
7681 Some(json!(["capabilities"])),
7682 ),
7683 (
7684 "malformed_pointer_escape".to_string(),
7685 "runtime_computed[0]".to_string(),
7686 "/roles/~2/tools".to_string(),
7687 json!({}),
7688 Some(json!(["/roles/~2/tools"])),
7689 ),
7690 (
7691 "unknown_capabilities_field".to_string(),
7692 "capabilities.future".to_string(),
7693 "<array>".to_string(),
7694 json!({ "future": [] }),
7695 None,
7696 ),
7697 ]);
7698
7699 for (index, (name, field, value, capabilities, runtime_computed)) in
7700 cases.into_iter().enumerate()
7701 {
7702 let registry = Arc::new(Registry::default());
7703 let handler = ControlHandler::new(Arc::clone(®istry));
7704 let response = handler
7705 .handle_control(
7706 ConnectionId::new((index + 1) as u64),
7707 capability_grammar_hello_frame(
7708 capabilities,
7709 runtime_computed,
7710 index as u64 + 1,
7711 ),
7712 )
7713 .expect("invalid HELLO returns a refusal");
7714
7715 assert_eq!(response.len(), 1, "{name} must emit one refusal");
7716 let error = parse_error(&response[0]);
7717 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7718 let message = error["message"]
7719 .as_str()
7720 .expect("error message is a string");
7721 assert!(
7722 message.contains(&field),
7723 "{name}: field missing from {message}"
7724 );
7725 assert!(
7726 message.contains(&value),
7727 "{name}: value missing from {message}"
7728 );
7729 assert_eq!(
7730 registry
7731 .active_registration_count()
7732 .expect("registry reads"),
7733 0,
7734 "{name}: refused HELLO must not create a catalog entry"
7735 );
7736 }
7737 }
7738
7739 #[test]
7740 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7741 let registry = Arc::new(Registry::default());
7742 let handler = ControlHandler::new(Arc::clone(®istry));
7743 let capabilities = json!({
7744 "provides": ["credentials-provider/v1"],
7745 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7746 "must_never_reach": ["federation-transport/v1"]
7747 });
7748 let response = handler
7749 .handle_control(
7750 ConnectionId::new(99),
7751 capability_grammar_hello_frame(
7752 capabilities.clone(),
7753 Some(json!(["/roles/0/tools"])),
7754 99,
7755 ),
7756 )
7757 .expect("valid HELLO registers");
7758 assert_eq!(response[0].header.ty, FrameType::HelloAck);
7759
7760 let request = Frame::build(
7761 FrameType::Request,
7762 control_flags(),
7763 0,
7764 0,
7765 100,
7766 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7767 .expect("catalog request serializes"),
7768 )
7769 .expect("catalog request frame builds");
7770 let response = handler
7771 .handle_catalog_list(request, None)
7772 .expect("catalog list succeeds");
7773 let ClientControlResponse::CatalogList { modules, .. } =
7774 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7775 else {
7776 panic!("catalog request must return catalog.list");
7777 };
7778 assert_eq!(modules.len(), 1);
7779 assert_eq!(
7780 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7781 capabilities
7782 );
7783 }
7784
7785 #[test]
7786 fn catalog_list_mirrors_management_operation_description() {
7787 let registry = Arc::new(Registry::default());
7788 let handler = ControlHandler::new(Arc::clone(®istry));
7789 let description = "List managed records and return their identifiers and metadata.";
7790 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7791 manifest.provides = vec![ProviderRole::ManagementSurface {
7792 operations: vec![ManagementOperation {
7793 name: "records.list".to_string(),
7794 kind: ManagementOperationKind::Query,
7795 description: Some(description.to_string()),
7796 }],
7797 config_schema: json!({"type": "object"}),
7798 observability: vec![ObservabilitySurface {
7799 name: "records.stats".to_string(),
7800 kind: ObservabilityKind::Snapshot,
7801 }],
7802 identity_scope: vec![IdentityScope::Project],
7803 concurrency: Concurrency::ModuleManaged,
7804 }];
7805 registry
7806 .register_with_control_ops(
7807 manifest,
7808 PROTOCOL_VERSION,
7809 ConnectionId::new(99),
7810 Vec::new(),
7811 )
7812 .expect("described management manifest registers");
7813
7814 let request = Frame::build(
7815 FrameType::Request,
7816 control_flags(),
7817 0,
7818 0,
7819 100,
7820 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7821 .expect("catalog request serializes"),
7822 )
7823 .expect("catalog request frame builds");
7824 let response = handler
7825 .handle_catalog_list(request, None)
7826 .expect("catalog list succeeds");
7827 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7828 assert_eq!(
7829 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7830 "catalog.list must preserve the declared operation description verbatim"
7831 );
7832 }
7833
7834 #[test]
7835 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7836 let registry = Arc::new(Registry::default());
7837 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7838 [("vault".to_string(), true), ("squatter".to_string(), true)],
7839 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7840 );
7841 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7842 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7843 provides: vec!["credentials-provider/v1".to_string()],
7844 requires: Vec::new(),
7845 must_never_reach: Vec::new(),
7846 });
7847 let frame = Frame::build(
7848 FrameType::Hello,
7849 control_flags(),
7850 0,
7851 0,
7852 77,
7853 serde_json::to_vec(&ModuleHelloBody {
7854 manifest: squatter,
7855 protocol_ver: PROTOCOL_VERSION,
7856 control_ops: None,
7857 launch_nonce: None,
7858 })
7859 .expect("HELLO serializes"),
7860 )
7861 .expect("HELLO frame builds");
7862 let response = handler
7863 .handle_control(ConnectionId::new(77), frame)
7864 .expect("reserved claim receives a typed refusal");
7865 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7866 assert_eq!(
7867 registry
7868 .active_registration_count()
7869 .expect("registry reads"),
7870 0,
7871 "a reserved capability refusal must not leave a catalog entry"
7872 );
7873 }
7874
7875 #[test]
7876 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
7877 for settlement in ["timeout", "inconclusive", "drop"] {
7878 let breakers = RouteBindBreakers::default();
7879 let RouteBindAdmission::Admitted {
7880 guard: mut old,
7881 probe: false,
7882 } = breakers.admit("prov")
7883 else {
7884 panic!("ordinary relay admitted")
7885 };
7886 let RouteBindAdmission::Admitted {
7887 guard: mut opener, ..
7888 } = breakers.admit("prov")
7889 else {
7890 panic!("second relay admitted")
7891 };
7892 assert!(
7893 !opener
7894 .record_timeout(1, Duration::ZERO)
7895 .unwrap()
7896 .reopened_after_probe
7897 );
7898 let RouteBindAdmission::Admitted {
7899 guard: mut probe,
7900 probe: true,
7901 } = breakers.admit("prov")
7902 else {
7903 panic!("one cooldown probe admitted")
7904 };
7905 match settlement {
7906 "timeout" => assert!(
7907 !old.record_timeout(1, Duration::ZERO)
7908 .unwrap()
7909 .reopened_after_probe
7910 ),
7911 "inconclusive" => old.record_inconclusive(),
7912 "drop" => drop(old),
7913 _ => unreachable!(),
7914 }
7915 assert!(
7916 matches!(
7917 breakers.admit("prov"),
7918 RouteBindAdmission::Refused {
7919 probe_in_flight: true,
7920 ..
7921 }
7922 ),
7923 "{settlement} of a pre-open relay cannot release the real probe"
7924 );
7925 assert!(
7926 probe
7927 .record_timeout(1, Duration::ZERO)
7928 .unwrap()
7929 .reopened_after_probe
7930 );
7931 assert!(matches!(
7932 breakers.admit("prov"),
7933 RouteBindAdmission::Admitted { probe: true, .. }
7934 ));
7935 }
7936 let breakers = RouteBindBreakers::default();
7937 let admit = || match breakers.admit("prov") {
7938 RouteBindAdmission::Admitted { guard, .. } => guard,
7939 _ => panic!("relay admitted"),
7940 };
7941 admit().record_timeout(1, Duration::ZERO);
7942 let mut old_probe = admit();
7943 breakers.reset_for_new_module_connection("prov");
7944 admit().record_timeout(1, Duration::ZERO);
7945 let _new_probe = admit();
7946 old_probe.record_inconclusive();
7947 assert!(matches!(
7948 breakers.admit("prov"),
7949 RouteBindAdmission::Refused {
7950 probe_in_flight: true,
7951 ..
7952 }
7953 ));
7954 }
7955
7956 #[tokio::test]
7957 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
7958 for candidate in [false, true] {
7959 let registry = Arc::new(Registry::default());
7960 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7961 [("vault".to_string(), true), ("squatter".to_string(), true)],
7962 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7963 );
7964 let conn = ConnectionId::new(77);
7965 let (ctx, mut rx) = route_ctx(conn);
7966 let initial = capability_manifest("squatter", &[], &[]);
7967 if candidate {
7968 registry
7969 .register_candidate_with_control_ops(
7970 initial.clone(),
7971 PROTOCOL_VERSION,
7972 conn,
7973 module_baseline_control_ops(),
7974 )
7975 .unwrap();
7976 handler
7977 .forwarding
7978 .register_candidate_module_connection(
7979 conn,
7980 "squatter".to_string(),
7981 PROTOCOL_VERSION,
7982 manifest_concurrency(&initial),
7983 ctx.egress.clone(),
7984 )
7985 .unwrap();
7986 } else {
7987 hello_via_sink(
7988 &handler,
7989 &ctx,
7990 &mut rx,
7991 hello_frame_with_manifest(initial.clone(), 1),
7992 )
7993 .await;
7994 }
7995 let response = handler
7996 .handle_control_frame(
7997 &ctx,
7998 catalog_update_with_capabilities_frame(
7999 2,
8000 capability_manifest("squatter", &["credentials-provider/v1"], &[])
8001 .capabilities
8002 .unwrap(),
8003 ),
8004 )
8005 .await
8006 .unwrap();
8007 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8008 assert_eq!(
8009 registry
8010 .get_module_by_connection(conn)
8011 .unwrap()
8012 .unwrap()
8013 .manifest,
8014 initial
8015 );
8016 }
8017 }
8018
8019 #[test]
8020 fn server_describe_surfaces_required_capability_verdict_fields() {
8021 let registry = Arc::new(Registry::default());
8022 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8023 [
8024 ("consumer".to_string(), true),
8025 ("provider".to_string(), false),
8026 ],
8027 BTreeMap::new(),
8028 );
8029 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
8030 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8031 provides: Vec::new(),
8032 requires: vec![subc_protocol::manifest::CapabilityRequirement {
8033 capability: "credentials-provider/v1".to_string(),
8034 need: subc_protocol::manifest::CapabilityNeed::Required,
8035 }],
8036 must_never_reach: Vec::new(),
8037 });
8038 let hello = Frame::build(
8039 FrameType::Hello,
8040 control_flags(),
8041 0,
8042 0,
8043 78,
8044 serde_json::to_vec(&ModuleHelloBody {
8045 manifest: consumer,
8046 protocol_ver: PROTOCOL_VERSION,
8047 control_ops: None,
8048 launch_nonce: None,
8049 })
8050 .expect("HELLO serializes"),
8051 )
8052 .expect("HELLO frame builds");
8053 handler
8054 .handle_control(ConnectionId::new(78), hello)
8055 .expect("consumer registers");
8056 let describe = Frame::build(
8057 FrameType::Request,
8058 control_flags(),
8059 0,
8060 0,
8061 79,
8062 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8063 .expect("request serializes"),
8064 )
8065 .expect("describe frame builds");
8066 let response = handler
8067 .handle_server_describe(describe)
8068 .expect("server.describe succeeds");
8069 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8070 let requirement = &rendered["capability_requirements"][0];
8071 assert_eq!(requirement["consumer"], "consumer");
8072 assert_eq!(requirement["verdict"], "never_provided");
8073 assert_eq!(requirement["episode_seq"], 1);
8074 assert_eq!(requirement["config_satisfiable"], false);
8075 assert_eq!(requirement["runtime_available"], false);
8076 assert!(requirement["detail"]
8077 .as_str()
8078 .expect("detail string")
8079 .contains("credentials-provider/v1"));
8080 }
8081
8082 #[test]
8083 fn catalog_list_omits_capabilities_for_legacy_manifest() {
8084 let registry = Arc::new(Registry::default());
8085 let handler = ControlHandler::new(Arc::clone(®istry));
8086 let hello = handler
8087 .handle_control(
8088 ConnectionId::new(101),
8089 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
8090 )
8091 .expect("legacy HELLO registers");
8092 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
8093
8094 let request = Frame::build(
8095 FrameType::Request,
8096 control_flags(),
8097 0,
8098 0,
8099 102,
8100 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8101 .expect("catalog request serializes"),
8102 )
8103 .expect("catalog request frame builds");
8104 let response = handler
8105 .handle_catalog_list(request, None)
8106 .expect("catalog list succeeds");
8107 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8108 assert!(
8109 body["modules"][0].get("capabilities").is_none(),
8110 "legacy manifest must retain an absent capabilities field on catalog.list"
8111 );
8112 }
8113
8114 #[test]
8115 fn hello_ack_omits_storage_when_no_storage_config() {
8116 let registry = Arc::new(Registry::default());
8117 let handler = ControlHandler::new(Arc::clone(®istry));
8118 let responses = handler
8119 .handle_control(
8120 ConnectionId::new(1),
8121 hello_frame("aft", PROTOCOL_VERSION, 7),
8122 )
8123 .unwrap();
8124 let ack = parse_ack(&responses[0]);
8125 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
8126 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
8127 }
8128
8129 #[tokio::test]
8130 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
8131 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
8132 let registry = Arc::new(Registry::default());
8133 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
8134 let responses = handler
8135 .handle_control(
8136 ConnectionId::new(1),
8137 hello_frame("aft", PROTOCOL_VERSION, 7),
8138 )
8139 .unwrap();
8140 let ack = parse_ack(&responses[0]);
8141 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
8142
8143 let described = handler
8144 .handle_control_frame(
8145 &route_ctx(ConnectionId::new(2)).0,
8146 Frame::build(
8147 FrameType::Request,
8148 control_flags(),
8149 0,
8150 0,
8151 9,
8152 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8153 )
8154 .unwrap(),
8155 )
8156 .await
8157 .unwrap();
8158 let ClientControlResponse::ServerDescribe { machine_id, .. } =
8159 serde_json::from_slice(&described[0].body).unwrap()
8160 else {
8161 panic!("server.describe answered with another shape");
8162 };
8163 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
8164 }
8165
8166 #[test]
8167 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
8168 let registry = Arc::new(Registry::default());
8171 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
8172 crate::daemon_config::StorageConfig::Sqlite {
8173 data_home: std::path::PathBuf::from("/data"),
8174 },
8175 ));
8176
8177 let responses = handler
8178 .handle_control(
8179 ConnectionId::new(1),
8180 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
8181 )
8182 .unwrap();
8183 let ack = parse_ack(&responses[0]);
8184 assert_eq!(
8185 ack.storage,
8186 Some(serde_json::json!({
8187 "module_id": "alfonso-routing",
8188 "storage_namespace": "default",
8189 "isolation": { "kind": "module" },
8190 "backend": {
8191 "backend": "sqlite",
8192 "path": "/data/cortexkit/alfonso-routing/store.db"
8193 }
8194 })),
8195 "the delivered descriptor is the module's own sqlite store path"
8196 );
8197 }
8198
8199 #[test]
8200 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
8201 let registry = Arc::new(Registry::default());
8202 let handler = ControlHandler::new(Arc::clone(®istry));
8203 let conn = ConnectionId::new(1);
8204 let responses = handler
8205 .handle_control(
8206 conn,
8207 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8208 )
8209 .unwrap();
8210 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8211 let registration = registry.get_module("aft").unwrap().unwrap();
8212 assert_eq!(registration.control_ops, module_baseline_control_ops());
8213
8214 let frame =
8215 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
8216 assert!(handler
8217 .guard_module_control_op(&frame, "aft", "route.bind")
8218 .unwrap()
8219 .is_none());
8220 let error = handler
8221 .guard_module_control_op(&frame, "aft", "test.synthetic")
8222 .unwrap()
8223 .expect("synthetic ungranted op should be rejected");
8224 assert_eq!(error.header.ty, FrameType::Error);
8225 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
8226 }
8227
8228 #[test]
8229 fn hello_control_ops_some_adds_optional_grants() {
8230 let registry = Arc::new(Registry::default());
8231 let handler = ControlHandler::new(Arc::clone(®istry));
8232 handler
8233 .handle_control(
8234 ConnectionId::new(1),
8235 hello_frame_with_control_ops(
8236 "aft",
8237 PROTOCOL_VERSION,
8238 7,
8239 Some(vec![
8240 "future.synthetic".to_string(),
8241 "route.bind".to_string(),
8242 ]),
8243 ),
8244 )
8245 .unwrap();
8246 let registration = registry.get_module("aft").unwrap().unwrap();
8247 assert_eq!(
8248 registration.control_ops,
8249 vec![
8250 "route.bind".to_string(),
8251 "route.status".to_string(),
8252 "future.synthetic".to_string(),
8253 ]
8254 );
8255 let frame =
8256 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
8257 assert!(handler
8258 .guard_module_control_op(&frame, "aft", "future.synthetic")
8259 .unwrap()
8260 .is_none());
8261 }
8262
8263 #[tokio::test]
8264 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
8265 let registry = Arc::new(Registry::default());
8266 let forwarding = Arc::new(ForwardingTable::default());
8267 let handler =
8268 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8269 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
8270 hello_via_sink(
8271 &handler,
8272 &module_ctx,
8273 &mut module_rx,
8274 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8275 )
8276 .await;
8277
8278 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
8279 let responses = handler
8280 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
8281 .await
8282 .unwrap();
8283 assert_eq!(responses.len(), 1);
8284 assert_eq!(responses[0].header.ty, FrameType::Error);
8285 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
8286 assert!(module_rx.try_recv().is_err());
8287 }
8288
8289 #[tokio::test]
8290 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
8291 let registry = Arc::new(Registry::default());
8292 let forwarding = Arc::new(ForwardingTable::default());
8293 let handler =
8294 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8295 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
8296 hello_via_sink(
8297 &handler,
8298 &module_ctx,
8299 &mut module_rx,
8300 hello_frame_with_control_ops(
8301 "aft",
8302 PROTOCOL_VERSION,
8303 7,
8304 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8305 ),
8306 )
8307 .await;
8308
8309 let project_root = unique_project_root("demux");
8310 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
8311 let route_handler = handler.clone();
8312 let route_task = tokio::spawn(async move {
8313 route_handler
8314 .handle_control_frame(
8315 &route_client_ctx,
8316 route_open_frame(100, "aft", project_root),
8317 )
8318 .await
8319 .unwrap()
8320 });
8321 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8322 .await
8323 .unwrap()
8324 .unwrap();
8325 assert!(matches!(
8326 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
8327 ModuleControlRequest::RouteBind { .. }
8328 ));
8329
8330 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
8331 let health_handler = handler.clone();
8332 let health_task = tokio::spawn(async move {
8333 health_handler
8334 .handle_control_frame(
8335 &health_client_ctx,
8336 supervisor_health_probe_frame(101, "aft"),
8337 )
8338 .await
8339 .unwrap()
8340 });
8341 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8342 .await
8343 .unwrap()
8344 .unwrap();
8345 assert_eq!(
8346 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8347 ModuleControlRequest::HealthCheck {}
8348 );
8349
8350 handler
8351 .handle_control_frame(
8352 &module_ctx,
8353 health_response(health_frame.header.corr, HealthStatus::Degraded),
8354 )
8355 .await
8356 .unwrap();
8357 let health_response = health_task.await.unwrap();
8358 assert_eq!(health_response.len(), 1);
8359 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8360 ClientControlResponse::SupervisorHealthProbe {
8361 module_id,
8362 status,
8363 detail,
8364 metrics,
8365 } => {
8366 assert_eq!(module_id, "aft");
8367 assert_eq!(status, HealthStatus::Degraded);
8368 assert_eq!(detail.as_deref(), Some("warming"));
8369 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
8370 }
8371 other => panic!("unexpected health response: {other:?}"),
8372 }
8373
8374 handler
8375 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8376 .await
8377 .unwrap();
8378 let route_response = route_task.await.unwrap();
8379 assert!(route_response.is_empty());
8380 let published = route_client_rx.recv().await.unwrap();
8381 assert!(matches!(
8382 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8383 ClientControlResponse::RouteOpen { .. }
8384 ));
8385 }
8386
8387 async fn relay_route_open(
8392 handler: &ControlHandler,
8393 client_connection: ConnectionId,
8394 client_egress: &FrameSink,
8395 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8396 corr: u64,
8397 module_id: &str,
8398 project_root_label: &str,
8399 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
8400 let ctx = RouteCtx {
8401 connection_id: client_connection,
8402 egress: client_egress.clone(),
8403 };
8404 let handler = handler.clone();
8405 let project_root = unique_project_root(project_root_label);
8406 let module_id = module_id.to_string();
8407 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
8408 let task = tokio::spawn(async move {
8409 let _guard = tracing::dispatcher::set_default(&dispatch);
8410 handler
8411 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8412 .await
8413 .unwrap()
8414 });
8415 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8416 .await
8417 .expect("module receives the relayed route.bind")
8418 .expect("module egress is open");
8419 (task, bind.frame)
8420 }
8421
8422 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8423 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8424 ModuleControlRequest::RouteBind {
8425 route_channel,
8426 epoch,
8427 ..
8428 } => (route_channel, epoch),
8429 other => panic!("expected a route.bind request, got {other:?}"),
8430 }
8431 }
8432
8433 fn published_route(frame: &Frame) -> (u16, u32) {
8434 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
8435 ClientControlResponse::RouteOpen {
8436 route_channel,
8437 route_epoch,
8438 } => (route_channel, route_epoch),
8439 other => panic!("expected a route.open response, got {other:?}"),
8440 }
8441 }
8442
8443 #[tokio::test]
8463 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
8464 let registry = Arc::new(Registry::default());
8465 let forwarding = Arc::new(ForwardingTable::default());
8466 let handler =
8467 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8468
8469 let module_connection = ConnectionId::new(30);
8470 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8471 hello_via_sink(
8472 &handler,
8473 &module_ctx,
8474 &mut module_rx,
8475 hello_frame("aft", PROTOCOL_VERSION, 7),
8476 )
8477 .await;
8478
8479 let dying_client = ConnectionId::new(31);
8480 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
8481
8482 let (first_task, first_bind) = relay_route_open(
8485 &handler,
8486 dying_client,
8487 &dying_ctx.egress,
8488 &mut module_rx,
8489 100,
8490 "aft",
8491 "closing-first",
8492 )
8493 .await;
8494 handler
8495 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
8496 .await
8497 .unwrap();
8498 assert!(first_task.await.unwrap().is_empty());
8499 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
8500
8501 let (second_task, second_bind) = relay_route_open(
8503 &handler,
8504 dying_client,
8505 &dying_ctx.egress,
8506 &mut module_rx,
8507 101,
8508 "aft",
8509 "closing-second",
8510 )
8511 .await;
8512 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
8513
8514 assert!(forwarding
8517 .escalate_client_delivery_failure(
8518 dying_client,
8519 first_channel,
8520 first_epoch,
8521 CloseReason::new(
8522 "module_to_client_delivery_failed",
8523 "client egress refused a module frame",
8524 ),
8525 crate::forwarding::UndeliveredFrame {
8526 module_id: None,
8527 sink: &dying_ctx.egress,
8528 },
8529 )
8530 .unwrap());
8531 assert!(!dying_ctx.egress.is_closed());
8532
8533 let ack = handler
8535 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
8536 .await;
8537 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
8538 if module_loop_error.is_some() {
8539 handler.cleanup_connection(module_connection).unwrap();
8543 }
8544 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8548 .await
8549 .ok()
8550 .flatten();
8551
8552 assert!(
8554 registry
8555 .get_module_by_connection(module_connection)
8556 .unwrap()
8557 .is_some(),
8558 "one client's closing connection ended the shared module connection: \
8559 {module_loop_error:?}"
8560 );
8561 let cotenant = ConnectionId::new(32);
8563 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
8564 let (cotenant_task, cotenant_bind) = relay_route_open(
8565 &handler,
8566 cotenant,
8567 &cotenant_ctx.egress,
8568 &mut module_rx,
8569 102,
8570 "aft",
8571 "closing-cotenant",
8572 )
8573 .await;
8574 handler
8575 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
8576 .await
8577 .unwrap();
8578 assert!(cotenant_task.await.unwrap().is_empty());
8579 let (cotenant_channel, cotenant_epoch) =
8580 published_route(&cotenant_rx.recv().await.unwrap());
8581 assert!(matches!(
8582 forwarding
8583 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
8584 .unwrap(),
8585 DataRoute::Client(DataRouteState::Bound(_))
8586 ));
8587
8588 let goodbye = post_ack_module_frame
8591 .expect("module receives a GOODBYE for the abandoned route channel");
8592 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8593 assert_eq!(goodbye.header.channel, abandoned_channel);
8594 assert_eq!(goodbye.header.epoch, abandoned_epoch);
8595
8596 assert!(dying_rx.try_recv().is_err());
8600 let second_response = second_task.await.unwrap();
8601 assert_eq!(second_response.len(), 1);
8602 assert_eq!(
8603 parse_error(&second_response[0])["code"],
8604 "target_unavailable"
8605 );
8606 }
8607
8608 async fn route_open_held_after_bind_send() -> (
8612 ControlHandler,
8613 Arc<tokio::sync::Barrier>,
8614 RouteCtx,
8615 mpsc::Receiver<crate::router::OutboundFrame>,
8616 ) {
8617 let hold = Arc::new(tokio::sync::Barrier::new(2));
8618 let handler = ControlHandler::with_forwarding(
8619 Arc::new(Registry::default()),
8620 Arc::new(ForwardingTable::default()),
8621 )
8622 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
8623 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
8624 hello_via_sink(
8625 &handler,
8626 &module_ctx,
8627 &mut module_rx,
8628 hello_frame("aft", PROTOCOL_VERSION, 7),
8629 )
8630 .await;
8631 (handler, hold, module_ctx, module_rx)
8632 }
8633
8634 #[tokio::test]
8641 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
8642 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8643 let client = ConnectionId::new(41);
8644 let (client_ctx, mut client_rx) = route_ctx(client);
8645
8646 let (open_task, bind) = relay_route_open(
8647 &handler,
8648 client,
8649 &client_ctx.egress,
8650 &mut module_rx,
8651 200,
8652 "aft",
8653 "acked-before-mark",
8654 )
8655 .await;
8656 let (module_channel, module_epoch) = route_bind_channel(&bind);
8657 hold.wait().await;
8659
8660 assert!(handler
8662 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
8663 .await
8664 .unwrap()
8665 .is_empty());
8666 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
8667
8668 hold.wait().await;
8669 assert!(
8670 open_task.await.unwrap().is_empty(),
8671 "route.open answers accepted"
8672 );
8673
8674 match module_rx.try_recv() {
8677 Err(mpsc::error::TryRecvError::Empty) => {}
8678 Ok(outbound) => panic!(
8679 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
8680 outbound.frame.header.ty,
8681 outbound.frame.header.channel,
8682 outbound.frame.header.epoch
8683 ),
8684 Err(err) => panic!("module egress closed: {err}"),
8685 }
8686
8687 let router = Router::with_control_handler(Arc::new(handler.clone()));
8690 let request = Frame::build(
8691 FrameType::Request,
8692 Flags::new(true, Priority::Interactive, false),
8693 client_channel,
8694 client_epoch,
8695 201,
8696 b"first request".to_vec(),
8697 )
8698 .unwrap();
8699 router
8700 .route_for_connection(&client_ctx, request)
8701 .await
8702 .unwrap();
8703 let delivered = module_rx.try_recv().unwrap().frame;
8704 assert_eq!(delivered.header.ty, FrameType::Request);
8705 assert_eq!(
8706 (delivered.header.channel, delivered.header.epoch),
8707 (module_channel, module_epoch)
8708 );
8709 assert_eq!(delivered.body, b"first request");
8710 }
8711
8712 #[tokio::test]
8718 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
8719 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8720 let client = ConnectionId::new(42);
8721 let (client_ctx, _client_rx) = route_ctx(client);
8722
8723 let (open_task, bind) = relay_route_open(
8724 &handler,
8725 client,
8726 &client_ctx.egress,
8727 &mut module_rx,
8728 300,
8729 "aft",
8730 "released-before-mark",
8731 )
8732 .await;
8733 let (module_channel, module_epoch) = route_bind_channel(&bind);
8734 hold.wait().await;
8735
8736 handler.cleanup_connection(client).unwrap();
8737 assert!(
8738 module_rx.try_recv().is_err(),
8739 "client teardown must not GOODBYE a bind route.open is still queueing"
8740 );
8741
8742 hold.wait().await;
8743 let response = open_task.await.unwrap();
8744 assert_eq!(response.len(), 1);
8745 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8746
8747 let goodbye = module_rx
8748 .try_recv()
8749 .expect("module is told to drop the binding the daemon abandoned")
8750 .frame;
8751 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8752 assert_eq!(
8753 (goodbye.header.channel, goodbye.header.epoch),
8754 (module_channel, module_epoch)
8755 );
8756 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
8757 }
8758
8759 #[tokio::test]
8764 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
8765 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8766 let client = ConnectionId::new(43);
8767 let (client_ctx, client_rx) = route_ctx(client);
8768
8769 let (open_task, bind) = relay_route_open(
8770 &handler,
8771 client,
8772 &client_ctx.egress,
8773 &mut module_rx,
8774 400,
8775 "aft",
8776 "gone-client-before-mark",
8777 )
8778 .await;
8779 let (module_channel, module_epoch) = route_bind_channel(&bind);
8780 hold.wait().await;
8781
8782 drop(client_rx);
8783 assert!(client_ctx.egress.is_closed());
8784 handler
8785 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
8786 .await
8787 .unwrap();
8788
8789 hold.wait().await;
8790 let response = open_task.await.unwrap();
8791 assert_eq!(response.len(), 1);
8792 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8793
8794 let goodbye = module_rx
8795 .try_recv()
8796 .expect("module is told to drop the binding it acked")
8797 .frame;
8798 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8799 assert_eq!(
8800 (goodbye.header.channel, goodbye.header.epoch),
8801 (module_channel, module_epoch)
8802 );
8803 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
8804 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
8805 }
8806
8807 #[test]
8814 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
8815 let handler = ControlHandler::default();
8816 let module_connection = ConnectionId::new(30);
8817 let client_connection = ConnectionId::new(31);
8818
8819 handler
8820 .refuse_to_end_module_connection_for_a_client(
8821 module_connection,
8822 77,
8823 ForwardingError::ConnectionClosing {
8824 connection_id: client_connection,
8825 },
8826 )
8827 .expect("a closing client must never end the module connection");
8828
8829 assert!(matches!(
8830 handler.refuse_to_end_module_connection_for_a_client(
8831 module_connection,
8832 78,
8833 ForwardingError::ConnectionClosing {
8834 connection_id: module_connection,
8835 },
8836 ),
8837 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
8838 connection_id
8839 })) if connection_id == module_connection
8840 ));
8841 assert!(matches!(
8842 handler.refuse_to_end_module_connection_for_a_client(
8843 module_connection,
8844 79,
8845 ForwardingError::Poisoned,
8846 ),
8847 Err(RouterError::Forwarding(ForwardingError::Poisoned))
8848 ));
8849 assert!(matches!(
8850 handler.refuse_to_end_module_connection_for_a_client(
8851 module_connection,
8852 80,
8853 ForwardingError::StaleModuleEndpoint,
8854 ),
8855 Err(RouterError::Forwarding(
8856 ForwardingError::StaleModuleEndpoint
8857 ))
8858 ));
8859 }
8860
8861 #[tokio::test]
8874 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
8875 let registry = Arc::new(Registry::default());
8876 let forwarding = Arc::new(ForwardingTable::default());
8877 let supervisor = SupervisorHandle::new();
8878 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8879 let handler =
8880 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8881 .with_supervisor(supervisor);
8882
8883 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8884 hello_via_sink(
8885 &handler,
8886 &target_ctx,
8887 &mut target_rx,
8888 hello_frame("target", PROTOCOL_VERSION, 1),
8889 )
8890 .await;
8891
8892 let wrong_nonce = handler
8896 .handle_control_frame(
8897 &route_ctx(ConnectionId::new(91)).0,
8898 route_open_frame_with_admission_facts(
8899 20,
8900 "target",
8901 unique_project_root("admission-facts"),
8902 Some(subc_control::ConsumerIdentity {
8903 module_id: "fed".to_string(),
8904 launch_nonce: "not-the-real-nonce".to_string(),
8905 }),
8906 None,
8907 ),
8908 )
8909 .await
8910 .unwrap();
8911 assert_eq!(
8912 parse_error(&wrong_nonce[0])["code"],
8913 "bad_consumer_identity",
8914 "a mismatched launch nonce must be refused, not stamped Reserved"
8915 );
8916
8917 let never_spawned = handler
8921 .handle_control_frame(
8922 &route_ctx(ConnectionId::new(92)).0,
8923 route_open_frame_with_admission_facts(
8924 21,
8925 "target",
8926 unique_project_root("admission-facts"),
8927 Some(subc_control::ConsumerIdentity {
8928 module_id: "never-spawned".to_string(),
8929 launch_nonce: "any-nonce".to_string(),
8930 }),
8931 None,
8932 ),
8933 )
8934 .await
8935 .unwrap();
8936 assert_eq!(
8937 parse_error(&never_spawned[0])["code"],
8938 "bad_consumer_identity",
8939 "an unspawned module_id must be refused rather than accepted for lack of a record"
8940 );
8941 }
8942
8943 #[tokio::test]
8966 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
8967 let registry = Arc::new(Registry::default());
8968 let forwarding = Arc::new(ForwardingTable::default());
8969 let supervisor = SupervisorHandle::new();
8970 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8971 let handler =
8972 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8973 .with_supervisor(supervisor);
8974
8975 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
8976 hello_via_sink(
8977 &handler,
8978 &target_ctx,
8979 &mut target_rx,
8980 hello_frame("target", PROTOCOL_VERSION, 1),
8981 )
8982 .await;
8983
8984 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
8985 let route_handler = handler.clone();
8986 let route_task = tokio::spawn(async move {
8987 route_handler
8988 .handle_control_frame(
8989 &client_ctx,
8990 route_open_frame_with_admission_facts(
8991 30,
8992 "target",
8993 unique_project_root("admission-facts"),
8994 Some(subc_control::ConsumerIdentity {
8995 module_id: "fed".to_string(),
8996 launch_nonce: "fed-nonce".to_string(),
8997 }),
8998 None,
8999 ),
9000 )
9001 .await
9002 .unwrap()
9003 });
9004
9005 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9012 .await
9013 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
9014 .expect("module control channel closed before route.bind");
9015 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9016 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9017 panic!("expected route.bind")
9018 };
9019 assert_eq!(
9020 principal,
9021 Some(Principal::Reserved {
9022 module_id: "fed".to_string()
9023 }),
9024 "a correctly attested consumer must be stamped Reserved for its own id"
9025 );
9026
9027 handler
9028 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9029 .await
9030 .unwrap();
9031 assert!(route_task.await.unwrap().is_empty());
9032 assert!(
9033 matches!(
9034 serde_json::from_slice::<ClientControlResponse>(
9035 &client_rx.recv().await.unwrap().body
9036 )
9037 .unwrap(),
9038 ClientControlResponse::RouteOpen { .. }
9039 ),
9040 "the route must actually open, not merely avoid an error"
9041 );
9042 }
9043
9044 #[tokio::test(start_paused = true)]
9045 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
9046 let registry = Arc::new(Registry::default());
9047 let forwarding = Arc::new(ForwardingTable::default());
9048 let supervisor = SupervisorHandle::new();
9049 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9050 let handler =
9051 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9052 .with_supervisor(supervisor);
9053
9054 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
9055 hello_via_sink(
9056 &handler,
9057 &target_ctx,
9058 &mut target_rx,
9059 hello_frame("target", PROTOCOL_VERSION, 1),
9060 )
9061 .await;
9062
9063 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
9064 let direct_handler = handler.clone();
9065 let direct_open = tokio::spawn(async move {
9066 direct_handler
9067 .handle_control_frame(
9068 &direct_ctx,
9069 route_open_frame(2, "target", unique_project_root("route-census-direct")),
9070 )
9071 .await
9072 .unwrap()
9073 });
9074 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9075 .await
9076 .expect("no direct route.bind within 5s")
9077 .expect("target control channel closed before direct route.bind");
9078 handler
9079 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
9080 .await
9081 .unwrap();
9082 assert!(direct_open.await.unwrap().is_empty());
9083 let _ = direct_rx.recv().await.unwrap();
9084
9085 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
9086 let reserved_handler = handler.clone();
9087 let reserved_open = tokio::spawn(async move {
9088 reserved_handler
9089 .handle_control_frame(
9090 &reserved_ctx,
9091 route_open_frame_with_admission_facts(
9092 3,
9093 "target",
9094 unique_project_root("admission-facts"),
9095 Some(ConsumerIdentity {
9096 module_id: "fed".to_string(),
9097 launch_nonce: "fed-nonce".to_string(),
9098 }),
9099 None,
9100 ),
9101 )
9102 .await
9103 .unwrap()
9104 });
9105 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9106 .await
9107 .expect("no reserved route.bind within 5s")
9108 .expect("target control channel closed before reserved route.bind");
9109 handler
9110 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
9111 .await
9112 .unwrap();
9113 assert!(reserved_open.await.unwrap().is_empty());
9114 let _ = reserved_rx.recv().await.unwrap();
9115
9116 forwarding
9117 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
9118 .unwrap();
9119 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
9120 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
9121 module_id: Some("target".to_string()),
9122 })
9123 .unwrap();
9124 let census_frame =
9125 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
9126 let response = handler
9127 .handle_control_frame(&census_ctx, census_frame)
9128 .await
9129 .unwrap()
9130 .pop()
9131 .unwrap();
9132 let actual: Value = serde_json::from_slice(&response.body).unwrap();
9133 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
9134 assert!(matches!(
9135 decoded,
9136 ClientControlResponse::SupervisorRoutes { .. }
9137 ));
9138 let routes = actual["modules"][0]["routes"].as_array().unwrap();
9139 assert_eq!(routes.len(), 2);
9140 assert!(routes.iter().all(|route| route["draining"] == true));
9141 assert!(
9144 routes.iter().all(|route| route["drain_reason"] == "reload"),
9145 "draining routes must name the drain's reason: {routes:?}"
9146 );
9147 assert!(routes.iter().any(|route| {
9148 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
9149 }));
9150 assert!(routes.iter().any(|route| {
9151 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
9152 }));
9153
9154 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9155 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
9156 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9157 std::fs::write(
9158 &golden_path,
9159 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
9160 )
9161 .unwrap();
9162 }
9163 let expected: Value =
9164 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
9165 assert_eq!(actual, expected);
9166 }
9167
9168 async fn query_live_roots(
9169 handler: &ControlHandler,
9170 module_ctx: &RouteCtx,
9171 ) -> ModuleControlResponseToModule {
9172 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
9173 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
9174 let response = handler
9175 .handle_control_frame(module_ctx, frame)
9176 .await
9177 .unwrap()
9178 .pop()
9179 .unwrap();
9180 serde_json::from_slice(&response.body).unwrap()
9181 }
9182
9183 #[tokio::test(start_paused = true)]
9184 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
9185 let registry = Arc::new(Registry::default());
9186 let forwarding = Arc::new(ForwardingTable::default());
9187 let handler = ControlHandler::with_forwarding(registry, forwarding);
9188 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
9189 hello_via_sink(
9190 &handler,
9191 &target_ctx,
9192 &mut target_rx,
9193 hello_frame("target", PROTOCOL_VERSION, 1),
9194 )
9195 .await;
9196 let root = unique_project_root("live-roots-known");
9197 let path = ProjectRootId::from_path_allowing_missing(root.path())
9198 .unwrap()
9199 .as_path()
9200 .to_path_buf();
9201 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
9202 let open_handler = handler.clone();
9203 let opened = tokio::spawn(async move {
9204 open_handler
9205 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
9206 .await
9207 .unwrap()
9208 });
9209 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9210 .await
9211 .unwrap()
9212 .unwrap();
9213 handler
9214 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
9215 .await
9216 .unwrap();
9217 assert!(opened.await.unwrap().is_empty());
9218 let _ = client_rx.recv().await.unwrap();
9219
9220 let root = unique_project_root("live-roots-pending");
9221 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
9222 .unwrap()
9223 .as_path()
9224 .to_path_buf();
9225 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
9226 let open_handler = handler.clone();
9227 let pending = tokio::spawn(async move {
9228 open_handler
9229 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
9230 .await
9231 .unwrap()
9232 });
9233 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9234 .await
9235 .unwrap()
9236 .unwrap();
9237 let actual = query_live_roots(&handler, &target_ctx).await;
9238 let ModuleControlResponseToModule::LiveRoots {
9239 roots,
9240 unknown_root_bindings,
9241 total_bindings,
9242 } = actual
9243 else {
9244 panic!("expected live roots")
9245 };
9246 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
9247 assert_eq!(unknown_root_bindings, 0);
9248 assert_eq!(
9249 roots.len(),
9250 2,
9251 "root-known arm must retain each canonical root"
9252 );
9253 assert_eq!(
9254 total_bindings,
9255 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9256 );
9257 let counts = roots
9258 .iter()
9259 .map(|root| (root.project_root.clone(), root.bound, root.pending))
9260 .collect::<Vec<_>>();
9261 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
9262 expected.sort_by(|a, b| a.0.cmp(&b.0));
9263 assert_eq!(
9264 counts, expected,
9265 "roots must sort by path and count pending separately"
9266 );
9267 handler
9268 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
9269 .await
9270 .unwrap();
9271 assert!(pending.await.unwrap().is_empty());
9272 }
9273
9274 #[tokio::test(start_paused = true)]
9275 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
9276 let forwarding = Arc::new(ForwardingTable::default());
9277 let handler =
9278 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9279 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
9280 hello_via_sink(
9281 &handler,
9282 &target_ctx,
9283 &mut target_rx,
9284 hello_frame("target", PROTOCOL_VERSION, 1),
9285 )
9286 .await;
9287 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
9288 let pending = forwarding
9289 .begin_route_bind_relay_for_test(
9290 client_ctx.connection_id,
9291 client_ctx.egress.clone(),
9292 2,
9293 "target",
9294 )
9295 .unwrap();
9296 forwarding
9297 .complete_pending_relay(
9298 target_ctx.connection_id,
9299 pending.corr,
9300 RouteBindRelayOutcome::Accepted,
9301 )
9302 .unwrap();
9303 let actual = query_live_roots(&handler, &target_ctx).await;
9304 let ModuleControlResponseToModule::LiveRoots {
9305 roots,
9306 unknown_root_bindings,
9307 total_bindings,
9308 } = actual
9309 else {
9310 panic!("expected live roots")
9311 };
9312 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
9313 assert_eq!(
9314 unknown_root_bindings, 1,
9315 "unknown-root arm must not read as no bindings"
9316 );
9317 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
9318 assert_eq!(
9319 total_bindings,
9320 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9321 );
9322 }
9323
9324 #[tokio::test(start_paused = true)]
9332 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
9333 let forwarding = Arc::new(ForwardingTable::default());
9334 let handler =
9335 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9336 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
9337 let replies = handler
9338 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
9339 .await
9340 .unwrap();
9341 let queued_by_hello = module_rx.len();
9342
9343 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
9344 let open_handler = handler.clone();
9345 let open = tokio::spawn(async move {
9346 open_handler
9347 .handle_control_frame(
9348 &client_ctx,
9349 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
9350 )
9351 .await
9352 .unwrap()
9353 });
9354 let mut spins = 0;
9356 while module_rx.len() == queued_by_hello {
9357 spins += 1;
9358 assert!(spins < 10_000, "route.open never queued a route.bind");
9359 tokio::task::yield_now().await;
9360 }
9361
9362 for reply in replies {
9364 module_ctx.egress.send(reply).await.unwrap();
9365 }
9366
9367 let first = module_rx.recv().await.unwrap().frame;
9368 assert_eq!(
9369 first.header.ty,
9370 FrameType::HelloAck,
9371 "the first frame a registering module reads must be its HELLO_ACK"
9372 );
9373 assert_eq!(first.header.corr, 7);
9374 let second = module_rx.recv().await.unwrap().frame;
9375 assert_eq!(second.header.ty, FrameType::Request);
9376 assert!(
9377 matches!(
9378 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
9379 ModuleControlRequest::RouteBind { .. }
9380 ),
9381 "the route.bind follows the ack"
9382 );
9383 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
9384
9385 handler
9386 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
9387 .await
9388 .unwrap();
9389 assert!(open.await.unwrap().is_empty());
9390 let _ = client_rx.recv().await.unwrap();
9391 }
9392
9393 #[tokio::test(start_paused = true)]
9394 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
9395 let handler = ControlHandler::with_forwarding(
9396 Arc::new(Registry::default()),
9397 Arc::new(ForwardingTable::default()),
9398 );
9399 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
9400 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
9401 hello_via_sink(
9402 &handler,
9403 &first_ctx,
9404 &mut first_rx,
9405 hello_frame("first", PROTOCOL_VERSION, 1),
9406 )
9407 .await;
9408 hello_via_sink(
9409 &handler,
9410 &second_ctx,
9411 &mut second_rx,
9412 hello_frame("second", PROTOCOL_VERSION, 2),
9413 )
9414 .await;
9415 let root = unique_project_root("second-only");
9416 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
9417 let cloned = handler.clone();
9418 let open = tokio::spawn(async move {
9419 cloned
9420 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
9421 .await
9422 .unwrap()
9423 });
9424 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
9425 .await
9426 .unwrap()
9427 .unwrap();
9428 let first = query_live_roots(&handler, &first_ctx).await;
9429 let second = query_live_roots(&handler, &second_ctx).await;
9430 assert!(
9431 matches!(
9432 first,
9433 ModuleControlResponseToModule::LiveRoots {
9434 total_bindings: 0,
9435 ..
9436 }
9437 ),
9438 "cross-module scope must not expose another module's roots"
9439 );
9440 assert!(
9441 matches!(
9442 second,
9443 ModuleControlResponseToModule::LiveRoots {
9444 total_bindings: 1,
9445 ..
9446 }
9447 ),
9448 "second module must see its pending route"
9449 );
9450 handler
9451 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
9452 .await
9453 .unwrap();
9454 assert!(open.await.unwrap().is_empty());
9455 }
9456
9457 #[tokio::test(start_paused = true)]
9458 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
9459 let handler = ControlHandler::with_forwarding(
9460 Arc::new(Registry::default()),
9461 Arc::new(ForwardingTable::default()),
9462 );
9463 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
9464 hello_via_sink(
9465 &handler,
9466 &target_ctx,
9467 &mut target_rx,
9468 hello_frame("target", PROTOCOL_VERSION, 1),
9469 )
9470 .await;
9471 let actual = query_live_roots(&handler, &target_ctx).await;
9472 let ModuleControlResponseToModule::LiveRoots {
9473 roots,
9474 unknown_root_bindings,
9475 total_bindings,
9476 } = actual
9477 else {
9478 panic!("expected live roots")
9479 };
9480 assert!(roots.is_empty());
9481 assert_eq!(unknown_root_bindings, 0);
9482 assert_eq!(total_bindings, 0);
9483 assert_eq!(
9484 total_bindings,
9485 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9486 );
9487 }
9488
9489 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
9496 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9497 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
9498 let text = std::fs::read_to_string(&path)
9499 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
9500 let vectors: Vec<(String, Value)> = text
9501 .lines()
9502 .filter(|line| !line.trim().is_empty())
9503 .map(|line| {
9504 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
9505 let id = entry["corpus_id"]
9506 .as_str()
9507 .expect("every vector carries a corpus_id")
9508 .to_string();
9509 (id, entry["package"].clone())
9510 })
9511 .collect();
9512 assert_eq!(
9516 vectors.len(),
9517 3,
9518 "vendored fed corpus changed size; re-sync from subc-federation"
9519 );
9520
9521 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
9536 let richest = vectors
9537 .iter()
9538 .filter_map(|(_, package)| package.as_object())
9539 .map(|object| {
9540 object
9541 .keys()
9542 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
9543 .count()
9544 })
9545 .max()
9546 .unwrap_or(0);
9547 assert!(
9548 richest >= 2,
9549 "vendored corpus no longer carries a package with unmodelled fields, \
9550 so the relay test can no longer distinguish a verbatim relay from a lossy one"
9551 );
9552
9553 vectors
9554 }
9555
9556 #[tokio::test]
9564 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
9565 for (corpus_id, package) in fed_admission_facts_vectors() {
9566 let registry = Arc::new(Registry::default());
9567 let forwarding = Arc::new(ForwardingTable::default());
9568 let supervisor = SupervisorHandle::new();
9569 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9570 let handler =
9571 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9572 .with_supervisor(supervisor)
9573 .with_admission_facts_config(
9574 Some("fed".to_string()),
9575 Some(vec!["target".to_string()]),
9576 );
9577
9578 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9579 hello_via_sink(
9580 &handler,
9581 &target_ctx,
9582 &mut target_rx,
9583 hello_frame("target", PROTOCOL_VERSION, 1),
9584 )
9585 .await;
9586
9587 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
9588 let route_handler = handler.clone();
9589 let expected = package.clone();
9590 let route_task = tokio::spawn(async move {
9591 route_handler
9592 .handle_control_frame(
9593 &client_ctx,
9594 route_open_frame_with_admission_facts(
9595 20,
9596 "target",
9597 unique_project_root("admission-facts"),
9598 Some(subc_control::ConsumerIdentity {
9599 module_id: "fed".to_string(),
9600 launch_nonce: "fed-nonce".to_string(),
9601 }),
9602 Some(package),
9603 ),
9604 )
9605 .await
9606 .unwrap()
9607 });
9608
9609 let bind_frame = target_rx.recv().await.unwrap();
9610 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9611 let ModuleControlRequest::RouteBind {
9612 admission_facts, ..
9613 } = bind
9614 else {
9615 panic!("{corpus_id}: expected route.bind")
9616 };
9617 assert_eq!(
9618 admission_facts,
9619 Some(expected),
9620 "{corpus_id}: relay must not add, drop or reshape any field"
9621 );
9622
9623 handler
9624 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9625 .await
9626 .unwrap();
9627 route_task.await.unwrap();
9628 }
9629 }
9630
9631 #[tokio::test]
9632 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
9633 let registry = Arc::new(Registry::default());
9634 let forwarding = Arc::new(ForwardingTable::default());
9635 let supervisor = SupervisorHandle::new();
9636 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9637 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
9638 let handler =
9639 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9640 .with_supervisor(supervisor)
9641 .with_admission_facts_config(
9642 Some("fed".to_string()),
9643 Some(vec!["target".to_string()]),
9644 );
9645
9646 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
9647 hello_via_sink(
9648 &handler,
9649 &target_ctx,
9650 &mut target_rx,
9651 hello_frame("target", PROTOCOL_VERSION, 1),
9652 )
9653 .await;
9654 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
9655 hello_via_sink(
9656 &handler,
9657 &other_ctx,
9658 &mut other_rx,
9659 hello_frame("other", PROTOCOL_VERSION, 2),
9660 )
9661 .await;
9662
9663 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
9664 let expected_facts = facts.clone();
9665 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
9666 let route_handler = handler.clone();
9667 let route_task = tokio::spawn(async move {
9668 route_handler
9669 .handle_control_frame(
9670 &client_ctx,
9671 route_open_frame_with_admission_facts(
9672 10,
9673 "target",
9674 unique_project_root("admission-facts"),
9675 Some(subc_control::ConsumerIdentity {
9676 module_id: "fed".to_string(),
9677 launch_nonce: "fed-nonce".to_string(),
9678 }),
9679 Some(facts.clone()),
9680 ),
9681 )
9682 .await
9683 .unwrap()
9684 });
9685 let bind_frame = target_rx.recv().await.unwrap();
9686 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9687 let ModuleControlRequest::RouteBind {
9688 admission_facts, ..
9689 } = bind
9690 else {
9691 panic!("expected route.bind")
9692 };
9693 assert_eq!(admission_facts, Some(expected_facts));
9694 handler
9695 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9696 .await
9697 .unwrap();
9698 assert!(route_task.await.unwrap().is_empty());
9699 assert!(matches!(
9700 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
9701 .unwrap(),
9702 ClientControlResponse::RouteOpen { .. }
9703 ));
9704
9705 let direct = handler
9706 .handle_control_frame(
9707 &route_ctx(ConnectionId::new(73)).0,
9708 route_open_frame_with_admission_facts(
9709 11,
9710 "target",
9711 unique_project_root("admission-facts"),
9712 None,
9713 Some(json!({"x": 1})),
9714 ),
9715 )
9716 .await
9717 .unwrap();
9718 assert_eq!(
9719 parse_error(&direct[0])["code"],
9720 "admission_facts_not_permitted"
9721 );
9722
9723 let different_reserved = handler
9724 .handle_control_frame(
9725 &route_ctx(ConnectionId::new(77)).0,
9726 route_open_frame_with_admission_facts(
9727 15,
9728 "target",
9729 unique_project_root("admission-facts"),
9730 Some(subc_control::ConsumerIdentity {
9731 module_id: "other".to_string(),
9732 launch_nonce: "other-nonce".to_string(),
9733 }),
9734 Some(json!({"x": 1})),
9735 ),
9736 )
9737 .await
9738 .unwrap();
9739 assert_eq!(
9740 parse_error(&different_reserved[0])["code"],
9741 "admission_facts_not_permitted"
9742 );
9743
9744 let other_target = handler
9745 .handle_control_frame(
9746 &route_ctx(ConnectionId::new(74)).0,
9747 route_open_frame_with_admission_facts(
9748 12,
9749 "other",
9750 unique_project_root("admission-facts"),
9751 Some(subc_control::ConsumerIdentity {
9752 module_id: "fed".to_string(),
9753 launch_nonce: "fed-nonce".to_string(),
9754 }),
9755 Some(json!({"x": 1})),
9756 ),
9757 )
9758 .await
9759 .unwrap();
9760 assert_eq!(
9761 parse_error(&other_target[0])["code"],
9762 "admission_facts_target_not_allowed"
9763 );
9764
9765 let nonexistent = handler
9766 .handle_control_frame(
9767 &route_ctx(ConnectionId::new(75)).0,
9768 route_open_frame_with_admission_facts(
9769 13,
9770 "missing",
9771 unique_project_root("admission-facts"),
9772 None,
9773 Some(json!({"x": 1})),
9774 ),
9775 )
9776 .await
9777 .unwrap();
9778 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
9779
9780 let described = handler
9781 .handle_control_frame(
9782 &route_ctx(ConnectionId::new(76)).0,
9783 Frame::build(
9784 FrameType::Request,
9785 control_flags(),
9786 0,
9787 0,
9788 14,
9789 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9790 )
9791 .unwrap(),
9792 )
9793 .await
9794 .unwrap();
9795 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9796 serde_json::from_slice(&described[0].body).unwrap()
9797 else {
9798 panic!("expected server.describe response")
9799 };
9800 assert!(capabilities
9801 .iter()
9802 .any(|cap| cap == "admission_facts_relay_v1"));
9803 }
9804
9805 #[tokio::test]
9806 async fn admission_facts_without_configured_carrier_are_rejected() {
9807 let registry = Arc::new(Registry::default());
9808 let forwarding = Arc::new(ForwardingTable::default());
9809 let handler = ControlHandler::with_forwarding(registry, forwarding);
9810 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
9811 hello_via_sink(
9812 &handler,
9813 &target_ctx,
9814 &mut target_rx,
9815 hello_frame("target", PROTOCOL_VERSION, 1),
9816 )
9817 .await;
9818
9819 let responses = handler
9820 .handle_control_frame(
9821 &route_ctx(ConnectionId::new(79)).0,
9822 route_open_frame_with_admission_facts(
9823 16,
9824 "target",
9825 unique_project_root("admission-facts"),
9826 None,
9827 Some(json!({"x": 1})),
9828 ),
9829 )
9830 .await
9831 .unwrap();
9832 assert_eq!(
9833 parse_error(&responses[0])["code"],
9834 "admission_facts_not_permitted"
9835 );
9836 }
9837
9838 #[tokio::test]
9839 async fn route_open_relays_consumer_capabilities_verbatim() {
9840 let registry = Arc::new(Registry::default());
9841 let forwarding = Arc::new(ForwardingTable::default());
9842 let handler =
9843 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9844 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
9845 hello_via_sink(
9846 &handler,
9847 &module_ctx,
9848 &mut module_rx,
9849 hello_frame("aft", PROTOCOL_VERSION, 7),
9850 )
9851 .await;
9852
9853 let expected = vec!["elicitation".to_string(), "roots".to_string()];
9854 let expected_for_request = expected.clone();
9855 let project_root = unique_project_root("consumer-capabilities-present");
9856 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
9857 let route_handler = handler.clone();
9858 let route_task = tokio::spawn(async move {
9859 route_handler
9860 .handle_control_frame(
9861 &client_ctx,
9862 route_open_frame_with_consumer_capabilities(
9863 401,
9864 "aft",
9865 project_root,
9866 Some(expected_for_request),
9867 ),
9868 )
9869 .await
9870 .unwrap()
9871 });
9872 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9873 .await
9874 .unwrap()
9875 .unwrap();
9876 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9877 let ModuleControlRequest::RouteBind {
9878 consumer_capabilities,
9879 ..
9880 } = bind
9881 else {
9882 panic!("expected route.bind request, got {bind:?}");
9883 };
9884 assert_eq!(consumer_capabilities, Some(expected.clone()));
9885
9886 handler
9887 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9888 .await
9889 .unwrap();
9890 let route_response = route_task.await.unwrap();
9891 assert!(route_response.is_empty());
9892 let published = client_rx.recv().await.unwrap();
9893 assert!(matches!(
9894 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9895 ClientControlResponse::RouteOpen { .. }
9896 ));
9897 }
9898
9899 #[tokio::test]
9900 async fn route_open_without_consumer_capabilities_relays_none() {
9901 let registry = Arc::new(Registry::default());
9902 let forwarding = Arc::new(ForwardingTable::default());
9903 let handler =
9904 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9905 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
9906 hello_via_sink(
9907 &handler,
9908 &module_ctx,
9909 &mut module_rx,
9910 hello_frame("aft", PROTOCOL_VERSION, 7),
9911 )
9912 .await;
9913
9914 let project_root = unique_project_root("consumer-capabilities-absent");
9915 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
9916 let route_handler = handler.clone();
9917 let route_task = tokio::spawn(async move {
9918 route_handler
9919 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
9920 .await
9921 .unwrap()
9922 });
9923 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9924 .await
9925 .unwrap()
9926 .unwrap();
9927 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9928 let ModuleControlRequest::RouteBind {
9929 consumer_capabilities,
9930 ..
9931 } = bind
9932 else {
9933 panic!("expected route.bind request, got {bind:?}");
9934 };
9935 assert_eq!(consumer_capabilities, None);
9936
9937 handler
9938 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9939 .await
9940 .unwrap();
9941 let route_response = route_task.await.unwrap();
9942 assert!(route_response.is_empty());
9943 let published = client_rx.recv().await.unwrap();
9944 assert!(matches!(
9945 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9946 ClientControlResponse::RouteOpen { .. }
9947 ));
9948 }
9949
9950 async fn bind_role_versions_for(
9954 sent: Option<BTreeMap<String, String>>,
9955 connection: u64,
9956 ) -> Option<BTreeMap<String, String>> {
9957 let registry = Arc::new(Registry::default());
9958 let forwarding = Arc::new(ForwardingTable::default());
9959 let handler =
9960 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9961 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
9962 hello_via_sink(
9963 &handler,
9964 &module_ctx,
9965 &mut module_rx,
9966 hello_frame("aft", PROTOCOL_VERSION, 7),
9967 )
9968 .await;
9969 let project_root = unique_project_root("role-versions");
9970 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
9971 let route_handler = handler.clone();
9972 let route_task = tokio::spawn(async move {
9973 route_handler
9974 .handle_control_frame(
9975 &client_ctx,
9976 route_open_frame_with_role_versions(403, "aft", project_root, sent),
9977 )
9978 .await
9979 .unwrap()
9980 });
9981 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9982 .await
9983 .expect("a well-formed route.open reaches the module as a bind")
9984 .unwrap();
9985 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9986 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
9987 panic!("expected route.bind request, got {bind:?}");
9988 };
9989 handler
9990 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9991 .await
9992 .unwrap();
9993 assert!(route_task.await.unwrap().is_empty());
9994 let published = client_rx.recv().await.unwrap();
9995 assert!(matches!(
9996 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9997 ClientControlResponse::RouteOpen { .. }
9998 ));
9999 role_versions
10000 }
10001
10002 #[tokio::test]
10003 async fn route_open_relays_role_versions_verbatim() {
10004 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
10005 assert_eq!(
10006 bind_role_versions_for(Some(sent.clone()), 141).await,
10007 Some(sent)
10008 );
10009 }
10010
10011 #[tokio::test]
10014 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
10015 assert_eq!(bind_role_versions_for(None, 143).await, None);
10016 assert_eq!(
10017 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
10018 None
10019 );
10020 }
10021
10022 #[tokio::test]
10023 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
10024 let registry = Arc::new(Registry::default());
10025 let forwarding = Arc::new(ForwardingTable::default());
10026 let handler =
10027 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10028 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
10029 hello_via_sink(
10030 &handler,
10031 &module_ctx,
10032 &mut module_rx,
10033 hello_frame("aft", PROTOCOL_VERSION, 7),
10034 )
10035 .await;
10036
10037 let nine: BTreeMap<String, String> = (0..9)
10038 .map(|index| (format!("role-{index}"), "v1".to_string()))
10039 .collect();
10040 for (label, malformed) in [
10041 (
10042 "invalid role name",
10043 role_versions(&[("Tool_Provider", "v1")]),
10044 ),
10045 ("invalid version", role_versions(&[("tool-provider", "v0")])),
10046 ("nine entries", nine),
10047 ] {
10048 let responses = tokio::time::timeout(
10051 Duration::from_secs(1),
10052 handler.handle_control_frame(
10053 &route_ctx(ConnectionId::new(148)).0,
10054 route_open_frame_with_role_versions(
10055 404,
10056 "aft",
10057 unique_project_root("role-versions-malformed"),
10058 Some(malformed),
10059 ),
10060 ),
10061 )
10062 .await
10063 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
10064 .unwrap();
10065 assert_eq!(responses.len(), 1, "{label}");
10066 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
10067 let error = parse_error(&responses[0]);
10068 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
10069 assert_eq!(
10070 error["detail"]["field"], "role_versions",
10071 "{label}: {error}"
10072 );
10073 assert!(
10074 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
10075 "{label}: a malformed declaration is terminal"
10076 );
10077 assert!(
10078 module_rx.try_recv().is_err(),
10079 "{label}: the module must never see a bind"
10080 );
10081 }
10082 }
10083
10084 #[tokio::test]
10088 async fn route_role_versions_capability_is_advertised() {
10089 let handler = ControlHandler::new(Arc::new(Registry::default()));
10090 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
10091 let ack = hello_via_sink(
10092 &handler,
10093 &ctx,
10094 &mut rx,
10095 hello_frame("m", PROTOCOL_VERSION, 1),
10096 )
10097 .await;
10098 let ack = parse_ack(&ack);
10099 assert!(
10100 ack.subc_capabilities
10101 .iter()
10102 .any(|c| c == "route-role-versions/v1"),
10103 "{:?}",
10104 ack.subc_capabilities
10105 );
10106
10107 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
10108 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
10109 let reply = handler
10110 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
10111 .await
10112 .unwrap()
10113 .pop()
10114 .unwrap();
10115 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10116 serde_json::from_slice(&reply.body).unwrap()
10117 else {
10118 panic!("not a server.describe reply");
10119 };
10120 assert!(
10121 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
10122 "{capabilities:?}"
10123 );
10124 }
10125
10126 #[tokio::test]
10127 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
10128 let registry = Arc::new(Registry::default());
10129 let forwarding = Arc::new(ForwardingTable::default());
10130 let handler =
10131 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10132 .with_health_probe_timeout(Duration::from_secs(5));
10133 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
10134 hello_via_sink(
10135 &handler,
10136 &module_ctx,
10137 &mut module_rx,
10138 non_routable_hello_frame_with_control_ops(
10139 "mcp",
10140 300,
10141 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10142 ),
10143 )
10144 .await;
10145 assert!(registry
10146 .get_module("mcp")
10147 .unwrap()
10148 .unwrap()
10149 .manifest
10150 .provides
10151 .is_empty());
10152
10153 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
10154 let route_response = handler
10155 .handle_control_frame(
10156 &route_client_ctx,
10157 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
10158 )
10159 .await
10160 .unwrap();
10161 assert_eq!(route_response[0].header.ty, FrameType::Error);
10162 assert_eq!(
10163 parse_error(&route_response[0])["code"],
10164 "target_unavailable"
10165 );
10166 assert!(parse_error(&route_response[0])["message"]
10167 .as_str()
10168 .unwrap()
10169 .contains("does not provide the requested target"));
10170 assert!(module_rx.try_recv().is_err());
10171
10172 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
10173 let health_handler = handler.clone();
10174 let health_task = tokio::spawn(async move {
10175 health_handler
10176 .handle_control_frame(
10177 &health_client_ctx,
10178 supervisor_health_probe_frame(302, "mcp"),
10179 )
10180 .await
10181 .unwrap()
10182 });
10183 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10184 .await
10185 .unwrap()
10186 .unwrap();
10187 assert_eq!(
10188 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10189 ModuleControlRequest::HealthCheck {}
10190 );
10191 handler
10192 .handle_control_frame(
10193 &module_ctx,
10194 health_response(health_frame.header.corr, HealthStatus::Ok),
10195 )
10196 .await
10197 .unwrap();
10198 let health_response = health_task.await.unwrap();
10199 assert_eq!(health_response[0].header.ty, FrameType::Response);
10200 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10201 ClientControlResponse::SupervisorHealthProbe {
10202 module_id, status, ..
10203 } => {
10204 assert_eq!(module_id, "mcp");
10205 assert_eq!(status, HealthStatus::Ok);
10206 }
10207 other => panic!("unexpected health response: {other:?}"),
10208 }
10209
10210 forwarding
10215 .cleanup_connection(module_ctx.connection_id)
10216 .unwrap();
10217 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
10218 let cleanup_response = tokio::time::timeout(
10219 Duration::from_millis(200),
10220 handler.handle_control_frame(
10221 &cleanup_probe_ctx,
10222 supervisor_health_probe_frame(303, "mcp"),
10223 ),
10224 )
10225 .await
10226 .expect("probe should fail immediately when the control lane is gone")
10227 .unwrap();
10228 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
10229 assert_eq!(
10230 parse_error(&cleanup_response[0])["code"],
10231 "target_unavailable"
10232 );
10233 assert!(parse_error(&cleanup_response[0])["message"]
10234 .as_str()
10235 .unwrap()
10236 .contains("no module connection"));
10237
10238 handler
10239 .cleanup_connection(module_ctx.connection_id)
10240 .unwrap();
10241 }
10242
10243 #[tokio::test]
10244 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
10245 let registry = Arc::new(Registry::default());
10246 let supervisor_handle = SupervisorHandle::new();
10247 let supervisor =
10248 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10249 .with_handle(supervisor_handle.clone())
10250 .with_connection_file_path(
10251 std::env::temp_dir()
10252 .join(format!("subc-route-open-warming-{}", std::process::id())),
10253 );
10254 let module = supervisor
10255 .supervise_configured(
10256 ModuleSpec {
10257 module_id: "warming".to_string(),
10258 program: fake_aft_stub_path(),
10259 args: Vec::new(),
10260 env: Vec::new(),
10261 reserved: false,
10262 reserved_prefixes: Vec::new(),
10263 protocol: ModuleProtocol::Subc,
10264 overlap: Default::default(),
10265 },
10266 true,
10267 )
10268 .unwrap();
10269 assert_eq!(module.state().unwrap(), ModuleState::Running);
10270
10271 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10272 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
10273 let response = handler
10274 .handle_control_frame(
10275 &ctx,
10276 route_open_frame(304, "warming", unique_project_root("warming")),
10277 )
10278 .await
10279 .unwrap();
10280 module.stop().await.unwrap();
10281
10282 assert_eq!(response[0].header.ty, FrameType::Error);
10283 let error = parse_error(&response[0]);
10284 assert_eq!(error["code"], "module_warming");
10285 assert!(error["message"]
10286 .as_str()
10287 .unwrap()
10288 .contains("state=running, enabled=true, live=false"));
10289 }
10290
10291 #[test]
10292 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
10293 let handler = ControlHandler::new(Arc::new(Registry::default()));
10294 let capture = EventCapture::default();
10295 let _subscriber =
10296 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10297 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
10298 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
10299 let pending = (0..limit).collect::<Vec<_>>();
10300 let response = handler
10301 .route_open_capacity_refusal(
10302 &ctx,
10303 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
10304 "busy",
10305 pending.len(),
10306 limit,
10307 )
10308 .unwrap();
10309 assert_eq!(parse_error(&response)["code"], "target_unavailable");
10310 let event = capture
10311 .events()
10312 .into_iter()
10313 .find(|event| {
10314 event.target == "control"
10315 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
10316 })
10317 .expect("connection admission refusal event");
10318 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10319 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10320 }
10321
10322 #[test]
10323 fn route_open_target_cap_logs_admission_reason_and_capacity() {
10324 let handler = ControlHandler::new(Arc::new(Registry::default()));
10325 let capture = EventCapture::default();
10326 let _subscriber =
10327 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10328 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
10329 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
10330 let guards = (0..limit)
10331 .map(|_| {
10332 handler
10333 .route_bind_concurrency
10334 .try_admit("busy", limit)
10335 .unwrap()
10336 })
10337 .collect::<Vec<_>>();
10338 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
10339 Err(in_flight) => in_flight,
10340 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
10341 };
10342 let response = handler
10343 .route_open_target_capacity_refusal(
10344 &ctx,
10345 &route_open_frame(397, "busy", unique_project_root("target-cap")),
10346 "busy",
10347 in_flight,
10348 )
10349 .unwrap();
10350 assert_eq!(parse_error(&response)["code"], "target_unavailable");
10351 let event = capture
10352 .events()
10353 .into_iter()
10354 .find(|event| {
10355 event.target == "control"
10356 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
10357 })
10358 .expect("target admission refusal event");
10359 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10360 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10361 drop(guards);
10362 }
10363
10364 #[tokio::test(flavor = "current_thread")]
10365 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
10366 let handler = ControlHandler::new(Arc::new(Registry::default()));
10367 let capture = EventCapture::default();
10368 let _subscriber =
10369 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10370 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
10371 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10372
10373 let _ = handler
10374 .handle_client_control_request(
10375 &ctx,
10376 frame,
10377 ClientControlRequest::SupervisorSetEnabled {
10378 module_id: "broca".to_string(),
10379 enabled: false,
10380 },
10381 )
10382 .await
10383 .unwrap();
10384
10385 let events = capture
10386 .events()
10387 .into_iter()
10388 .filter(|event| {
10389 event.target == "control"
10390 && event.fields.get("message").map(String::as_str)
10391 == Some("supervisor request received")
10392 })
10393 .collect::<Vec<_>>();
10394 assert_eq!(events.len(), 1, "one supervisor request log line");
10395 let event = &events[0];
10396 assert_eq!(event.level, tracing::Level::INFO);
10397 assert_eq!(
10398 event.fields.get("op"),
10399 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
10400 );
10401 assert_eq!(
10402 event.fields.get("module_id"),
10403 Some(&"\"broca\"".to_string())
10404 );
10405 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
10406 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
10407 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
10408 }
10409
10410 #[tokio::test(flavor = "current_thread")]
10411 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
10412 let registry = Arc::new(Registry::default());
10413 let module_connection = ConnectionId::new(303);
10414 registry
10415 .register_with_control_ops(
10416 manifest("aft", PROTOCOL_VERSION),
10417 PROTOCOL_VERSION,
10418 module_connection,
10419 module_baseline_control_ops(),
10420 )
10421 .unwrap();
10422 let handler = ControlHandler::new(registry);
10423 let capture = EventCapture::default();
10424 let _subscriber =
10425 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10426 let (ctx, _rx) = route_ctx(module_connection);
10427 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10428
10429 let _ = handler
10430 .handle_client_control_request(
10431 &ctx,
10432 frame,
10433 ClientControlRequest::SupervisorSetEnabled {
10434 module_id: "broca".to_string(),
10435 enabled: false,
10436 },
10437 )
10438 .await
10439 .unwrap();
10440
10441 let callers = capture
10442 .events()
10443 .into_iter()
10444 .filter(|event| {
10445 event.fields.get("message").map(String::as_str)
10446 == Some("supervisor request received")
10447 })
10448 .map(|event| event.fields.get("caller").cloned())
10449 .collect::<Vec<_>>();
10450 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
10451 }
10452
10453 #[tokio::test(flavor = "current_thread")]
10454 async fn supervisor_list_does_not_log_request_received() {
10455 let handler = ControlHandler::new(Arc::new(Registry::default()));
10456 let capture = EventCapture::default();
10457 let _subscriber =
10458 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10459 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
10460 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
10461
10462 handler
10463 .handle_client_control_request(&ctx, frame, ClientControlRequest::SupervisorList {})
10464 .await
10465 .unwrap();
10466
10467 assert!(capture.events().into_iter().all(|event| {
10468 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10469 }));
10470 }
10471
10472 #[tokio::test(flavor = "current_thread")]
10473 async fn supervisor_rescan_preview_does_not_log_request_received() {
10474 let handler = ControlHandler::new(Arc::new(Registry::default()));
10475 let capture = EventCapture::default();
10476 let _subscriber =
10477 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10478 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
10479 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
10480
10481 handler
10482 .handle_client_control_request(
10483 &ctx,
10484 frame,
10485 ClientControlRequest::SupervisorRescan { preview: true },
10486 )
10487 .await
10488 .unwrap();
10489
10490 assert!(capture.events().into_iter().all(|event| {
10491 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10492 }));
10493 }
10494
10495 #[tokio::test]
10500 async fn route_open_refusal_names_the_check_that_refused() {
10501 let handler = ControlHandler::new(Arc::new(Registry::default()));
10502 let capture = EventCapture::default();
10503 let _subscriber =
10504 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10505 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10506 let response = handler
10507 .handle_control_frame(
10508 &ctx,
10509 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
10510 )
10511 .await
10512 .unwrap();
10513
10514 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10515 let event = capture
10516 .events()
10517 .into_iter()
10518 .find(|event| {
10519 event.target == "control"
10520 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10521 })
10522 .expect("route.open refusal event");
10523 assert_eq!(
10524 event.fields.get("reason"),
10525 Some(&"\"not_registered\"".to_string())
10526 );
10527 }
10528
10529 #[tokio::test]
10530 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
10531 let registry = Arc::new(Registry::default());
10532 let supervisor_handle = SupervisorHandle::new();
10533 let supervisor =
10534 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10535 .with_handle(supervisor_handle.clone())
10536 .with_connection_file_path(std::env::temp_dir().join(format!(
10537 "subc-route-open-refusal-info-{}",
10538 std::process::id()
10539 )));
10540 let module = supervisor
10541 .supervise_configured(
10542 ModuleSpec {
10543 module_id: "warming".to_string(),
10544 program: fake_aft_stub_path(),
10545 args: Vec::new(),
10546 env: Vec::new(),
10547 reserved: false,
10548 reserved_prefixes: Vec::new(),
10549 protocol: ModuleProtocol::Subc,
10550 overlap: Default::default(),
10551 },
10552 true,
10553 )
10554 .unwrap();
10555 assert_eq!(module.state().unwrap(), ModuleState::Running);
10556
10557 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10558 assert!(handler
10559 .counters()
10560 .snapshot()
10561 .get("route_open_refused_by_code")
10562 .is_none());
10563 let capture = EventCapture::default();
10564 let _subscriber =
10565 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10566 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
10567 let response = handler
10568 .handle_control_frame(
10569 &ctx,
10570 route_open_frame(394, "warming", unique_project_root("refusal-info")),
10571 )
10572 .await
10573 .unwrap();
10574 module.stop().await.unwrap();
10575
10576 assert_eq!(parse_error(&response[0])["code"], "module_warming");
10577 let event = capture
10578 .events()
10579 .into_iter()
10580 .find(|event| {
10581 event.target == "control"
10582 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
10583 })
10584 .expect("route.open refusal event");
10585 assert_eq!(
10586 event.fields.get("module_id"),
10587 Some(&"\"warming\"".to_string())
10588 );
10589 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
10590 assert_eq!(
10591 event.fields.get("reason"),
10592 Some(&"\"supervised_not_registered\"".to_string())
10593 );
10594 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
10595 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
10596 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
10597 assert_eq!(
10598 handler.counters().snapshot()["route_open_refused_by_code"],
10599 json!({ "module_warming": 1 })
10600 );
10601 }
10602
10603 const OUTAGE_START: &str = "route.open refusing module: not serving";
10604 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
10605
10606 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
10607 capture
10608 .events()
10609 .into_iter()
10610 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
10611 .collect()
10612 }
10613
10614 fn supervise_stub(
10615 registry: &Arc<Registry>,
10616 module_id: &str,
10617 enabled: bool,
10618 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
10619 let supervisor_handle = SupervisorHandle::new();
10620 let supervisor =
10621 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
10622 .with_handle(supervisor_handle.clone())
10623 .with_connection_file_path(std::env::temp_dir().join(format!(
10624 "subc-route-outage-{module_id}-{}",
10625 std::process::id()
10626 )));
10627 let module = supervisor
10628 .supervise_configured(
10629 ModuleSpec {
10630 module_id: module_id.to_string(),
10631 program: fake_aft_stub_path(),
10632 args: Vec::new(),
10633 env: Vec::new(),
10634 reserved: false,
10635 reserved_prefixes: Vec::new(),
10636 protocol: ModuleProtocol::Subc,
10637 overlap: Default::default(),
10638 },
10639 enabled,
10640 )
10641 .unwrap();
10642 (supervisor_handle, module)
10643 }
10644
10645 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
10646 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
10647 module_id: module_id.to_string(),
10648 drain_timeout_ms: Some(50),
10649 })
10650 .unwrap();
10651 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
10652 }
10653
10654 #[test]
10658 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
10659 let registry = Arc::new(Registry::default());
10660 let forwarding = Arc::new(ForwardingTable::default());
10661 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10662 let second = ControlHandler::with_forwarding(registry, forwarding);
10663 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
10664 }
10665
10666 #[tokio::test(flavor = "current_thread")]
10670 async fn route_open_unknown_module_refusals_add_no_outage_state() {
10671 let handler = ControlHandler::new(Arc::new(Registry::default()));
10672 let capture = EventCapture::default();
10673 let _subscriber =
10674 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10675 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
10676 for corr in 0..8 {
10677 let response = handler
10678 .handle_control_frame(
10679 &ctx,
10680 route_open_frame(
10681 380 + corr,
10682 &format!("nobody-{corr}"),
10683 unique_project_root("outage-unknown"),
10684 ),
10685 )
10686 .await
10687 .unwrap();
10688 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10689 }
10690
10691 assert_eq!(handler.route_outages.tracked_module_count(), 0);
10692 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
10693 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
10694 }
10695
10696 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10701 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
10702 let registry = Arc::new(Registry::default());
10703 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
10704 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10705 let capture = EventCapture::default();
10706 let _subscriber =
10707 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10708 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
10709 handler.route_outages.record_accepted("outage-restart");
10712
10713 let response = handler
10714 .handle_control_frame(
10715 &ctx,
10716 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
10717 )
10718 .await
10719 .unwrap();
10720 assert_eq!(response[0].header.ty, FrameType::Error);
10721 let starts = outage_lines(&capture, OUTAGE_START);
10722 assert_eq!(starts.len(), 1, "{starts:?}");
10723 assert_eq!(starts[0].level, tracing::Level::WARN);
10724 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
10725 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
10726 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
10727 handler.route_outages.record_accepted("outage-restart");
10728 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
10729
10730 let restart = handler
10731 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
10732 .await
10733 .unwrap();
10734 assert_eq!(
10735 restart[0].header.ty,
10736 FrameType::Response,
10737 "{:?}",
10738 parse_error(&restart[0])
10739 );
10740 handler
10741 .handle_control_frame(
10742 &ctx,
10743 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
10744 )
10745 .await
10746 .unwrap();
10747 module.stop().await.unwrap();
10748
10749 let starts = outage_lines(&capture, OUTAGE_START);
10750 assert_eq!(starts.len(), 2, "{starts:?}");
10751 assert_eq!(starts[1].level, tracing::Level::INFO);
10752 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
10753 }
10754
10755 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10759 async fn failed_operator_restart_leaves_no_operator_mark() {
10760 let registry = Arc::new(Registry::default());
10761 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
10762 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10763 let capture = EventCapture::default();
10764 let _subscriber =
10765 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10766 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
10767 handler.route_outages.record_accepted("outage-disabled");
10768
10769 let restart = handler
10770 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
10771 .await
10772 .unwrap();
10773 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
10774 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
10775
10776 handler
10777 .handle_control_frame(
10778 &ctx,
10779 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
10780 )
10781 .await
10782 .unwrap();
10783 let starts = outage_lines(&capture, OUTAGE_START);
10784 assert_eq!(starts.len(), 1, "{starts:?}");
10785 assert_eq!(starts[0].level, tracing::Level::WARN);
10786 }
10787
10788 #[tokio::test(flavor = "current_thread")]
10789 async fn route_open_unknown_module_escapes_target_module_id() {
10790 let handler = ControlHandler::new(Arc::new(Registry::default()));
10791 let capture = EventCapture::default();
10792 let _subscriber =
10793 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10794 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
10795 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10796 let response = handler
10797 .handle_control_frame(
10798 &ctx,
10799 route_open_frame(
10800 395,
10801 hostile_module_id,
10802 unique_project_root("hostile-target-module-id"),
10803 ),
10804 )
10805 .await
10806 .unwrap();
10807
10808 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10809 let event = capture
10810 .events()
10811 .into_iter()
10812 .find(|event| {
10813 event.target == "control"
10814 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10815 })
10816 .expect("route.open unknown-module refusal event");
10817 let logged = event.fields.get("module_id").expect("module_id field");
10818 assert!(!logged.bytes().any(|byte| byte < 0x20));
10819 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10820 }
10821
10822 #[tokio::test(flavor = "current_thread")]
10823 async fn route_open_module_rejection_uses_daemon_counter_key() {
10824 let registry = Arc::new(Registry::default());
10825 let forwarding = Arc::new(ForwardingTable::default());
10826 let handler =
10827 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10828 let module_connection = ConnectionId::new(95);
10829 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10830 hello_via_sink(
10831 &handler,
10832 &module_ctx,
10833 &mut module_rx,
10834 hello_frame("aft", PROTOCOL_VERSION, 395),
10835 )
10836 .await;
10837
10838 let client_connection = ConnectionId::new(96);
10839 let (client_ctx, _client_rx) = route_ctx(client_connection);
10840 let capture = EventCapture::default();
10841 let _subscriber =
10842 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10843 let (route_task, bind) = relay_route_open(
10844 &handler,
10845 client_connection,
10846 &client_ctx.egress,
10847 &mut module_rx,
10848 396,
10849 "aft",
10850 "hostile-module-code",
10851 )
10852 .await;
10853 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
10854 let rejection = Frame::build(
10855 FrameType::Error,
10856 control_flags(),
10857 0,
10858 0,
10859 bind.header.corr,
10860 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
10861 )
10862 .unwrap();
10863 handler
10864 .handle_control_frame(&module_ctx, rejection)
10865 .await
10866 .unwrap();
10867
10868 let response = route_task.await.unwrap();
10869 assert_eq!(parse_error(&response[0])["code"], hostile_code);
10870 let counters = handler.counters().snapshot();
10871 assert_eq!(
10872 counters["route_open_refused_by_code"],
10873 json!({ "module_rejected": 1 })
10874 );
10875 assert!(counters["route_open_refused_by_code"]
10876 .get(hostile_code)
10877 .is_none());
10878
10879 let event = capture
10880 .events()
10881 .into_iter()
10882 .find(|event| {
10883 event.target == "control"
10884 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
10885 })
10886 .expect("route.open module-rejection refusal event");
10887 let logged = event.fields.get("module_code").expect("module_code field");
10888 assert!(!logged.bytes().any(|byte| byte < 0x20));
10889 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10890 }
10891
10892 #[tokio::test]
10893 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
10894 let registry = Arc::new(Registry::default());
10895 let supervisor_handle = SupervisorHandle::new();
10896 let missing_program = std::env::temp_dir().join(format!(
10897 "subc-route-open-missing-program-{}",
10898 std::process::id()
10899 ));
10900 let supervisor =
10901 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10902 .with_handle(supervisor_handle.clone());
10903 let module = supervisor
10904 .supervise_configured(
10905 ModuleSpec {
10906 module_id: "failed".to_string(),
10907 program: missing_program,
10908 args: Vec::new(),
10909 env: Vec::new(),
10910 reserved: false,
10911 reserved_prefixes: Vec::new(),
10912 protocol: ModuleProtocol::Subc,
10913 overlap: Default::default(),
10914 },
10915 true,
10916 )
10917 .unwrap();
10918 assert_eq!(module.state().unwrap(), ModuleState::Failed);
10919
10920 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10921 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
10922 let response = handler
10923 .handle_control_frame(
10924 &ctx,
10925 route_open_frame(305, "failed", unique_project_root("failed")),
10926 )
10927 .await
10928 .unwrap();
10929
10930 assert_eq!(response[0].header.ty, FrameType::Error);
10931 let error = parse_error(&response[0]);
10932 assert_eq!(error["code"], "target_unavailable");
10933 assert!(error["message"]
10934 .as_str()
10935 .unwrap()
10936 .contains("state=failed, enabled=true, live=false"));
10937 }
10938
10939 #[tokio::test]
10940 async fn route_open_role_mismatch_remains_target_unavailable() {
10941 let registry = Arc::new(Registry::default());
10942 let handler = ControlHandler::new(Arc::clone(®istry));
10943 handler
10944 .handle_control(
10945 ConnectionId::new(41),
10946 non_routable_hello_frame_with_control_ops("health-only", 306, None),
10947 )
10948 .unwrap();
10949
10950 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
10951 let response = handler
10952 .handle_control_frame(
10953 &ctx,
10954 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
10955 )
10956 .await
10957 .unwrap();
10958
10959 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10960 assert!(parse_error(&response[0])["message"]
10961 .as_str()
10962 .unwrap()
10963 .contains("does not provide the requested target"));
10964 }
10965
10966 #[tokio::test]
10967 async fn route_open_inactive_registration_remains_target_unavailable() {
10968 let registry = Arc::new(Registry::default());
10969 let handler = ControlHandler::new(Arc::clone(®istry));
10970 handler
10971 .handle_control(
10972 ConnectionId::new(43),
10973 hello_frame("inactive", PROTOCOL_VERSION, 308),
10974 )
10975 .unwrap();
10976 assert!(registry
10977 .set_module_state_for_test("inactive", ChannelState::Closed)
10978 .unwrap());
10979
10980 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
10981 let response = handler
10982 .handle_control_frame(
10983 &ctx,
10984 route_open_frame(309, "inactive", unique_project_root("inactive")),
10985 )
10986 .await
10987 .unwrap();
10988
10989 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10990 assert!(parse_error(&response[0])["message"]
10991 .as_str()
10992 .unwrap()
10993 .contains("is not active"));
10994 }
10995
10996 #[tokio::test]
10997 async fn late_health_reply_is_recorded_through_the_module_response_path() {
10998 let registry = Arc::new(Registry::default());
10999 let forwarding = Arc::new(ForwardingTable::default());
11000 let supervisor_handle = SupervisorHandle::new();
11001 let supervisor =
11002 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
11003 .with_forwarding(Arc::clone(&forwarding))
11004 .with_handle(supervisor_handle.clone());
11005 let module = supervisor
11006 .supervise_configured(
11007 crate::ModuleSpec {
11008 module_id: "late-health-response".to_string(),
11009 program: PathBuf::from("disabled-module"),
11010 args: Vec::new(),
11011 env: Vec::new(),
11012 reserved: false,
11013 reserved_prefixes: Vec::new(),
11014 protocol: ModuleProtocol::Subc,
11015 overlap: Default::default(),
11016 },
11017 false,
11018 )
11019 .unwrap();
11020 let handler =
11021 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11022 .with_supervisor(supervisor_handle);
11023 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
11024 handler
11025 .handle_control_frame(
11026 &module_ctx,
11027 hello_frame_with_control_ops(
11028 "late-health-response",
11029 PROTOCOL_VERSION,
11030 7,
11031 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11032 ),
11033 )
11034 .await
11035 .unwrap();
11036 let probe_started_at = Instant::now() - Duration::from_millis(80);
11037 let pending = forwarding
11038 .begin_health_probe_rpc_for(
11039 "late-health-response",
11040 MODULE_CONTROL_OP_HEALTH_CHECK,
11041 probe_started_at,
11042 Instant::now() - Duration::from_millis(1),
11043 )
11044 .unwrap();
11045 assert!(forwarding
11046 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
11047 .unwrap());
11048
11049 let responses = handler
11050 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
11051 .await
11052 .unwrap();
11053
11054 assert!(responses.is_empty());
11055 let health = module.status().unwrap().health;
11056 assert_eq!(health.late_answer_count, 1);
11057 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
11058 }
11059
11060 #[tokio::test]
11061 async fn health_probe_timeout_and_module_death_are_typed() {
11062 let registry = Arc::new(Registry::default());
11063 let forwarding = Arc::new(ForwardingTable::default());
11064 let handler =
11065 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11066 .with_health_probe_timeout(Duration::from_millis(50));
11067 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11068 hello_via_sink(
11069 &handler,
11070 &module_ctx,
11071 &mut module_rx,
11072 hello_frame_with_control_ops(
11073 "aft",
11074 PROTOCOL_VERSION,
11075 7,
11076 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11077 ),
11078 )
11079 .await;
11080
11081 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11082 let responses = handler
11083 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
11084 .await
11085 .unwrap();
11086 assert_eq!(responses[0].header.ty, FrameType::Error);
11087 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
11088 let _ = module_rx.try_recv();
11089
11090 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
11091 let health_handler = handler.clone();
11092 let death_task = tokio::spawn(async move {
11093 health_handler
11094 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
11095 .await
11096 .unwrap()
11097 });
11098 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11099 .await
11100 .unwrap()
11101 .unwrap();
11102 handler
11103 .cleanup_connection(module_ctx.connection_id)
11104 .unwrap();
11105 let responses = death_task.await.unwrap();
11106 assert_eq!(responses[0].header.ty, FrameType::Error);
11107 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
11108 }
11109
11110 #[test]
11111 fn hello_requires_exact_protocol_version() {
11112 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
11113 let registry = Arc::new(Registry::default());
11114 let handler = ControlHandler::new(Arc::clone(®istry));
11115 let responses = handler
11116 .handle_control(
11117 ConnectionId::new(connection),
11118 hello_frame("aft", offered, 9),
11119 )
11120 .unwrap();
11121
11122 assert_eq!(responses.len(), 1);
11123 assert_eq!(responses[0].header.ty, FrameType::Error);
11124 let error = parse_error(&responses[0]);
11125 assert_eq!(error["code"], "version_unsupported");
11126 assert!(registry.get_module("aft").unwrap().is_none());
11127 assert_eq!(registry.active_registration_count().unwrap(), 0);
11128 }
11129 }
11130
11131 #[test]
11132 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
11133 let registry = Arc::new(Registry::default());
11134 let forwarding = Arc::new(ForwardingTable::default());
11135 let handler =
11136 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11137 let module_connection = ConnectionId::new(301);
11138 let registration = registry
11139 .register_with_control_ops(
11140 manifest("aft-push", PROTOCOL_VERSION),
11141 PROTOCOL_VERSION,
11142 module_connection,
11143 module_baseline_control_ops(),
11144 )
11145 .unwrap();
11146 let (module_tx, _module_rx) = mpsc::channel(8);
11147 let endpoint = forwarding
11148 .register_module_connection(
11149 module_connection,
11150 "aft-push".to_string(),
11151 PROTOCOL_VERSION,
11152 manifest_concurrency(®istration.manifest),
11153 FrameSink::new(module_tx),
11154 )
11155 .unwrap();
11156
11157 let unknown = Frame::build(
11159 FrameType::Push,
11160 control_flags(),
11161 0,
11162 0,
11163 5,
11164 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
11165 )
11166 .unwrap();
11167 let out = handler.handle_status_update(endpoint, unknown).unwrap();
11168 assert!(
11169 out.is_empty(),
11170 "unknown push op must be ignored, got {out:?}"
11171 );
11172
11173 let malformed = Frame::build(
11175 FrameType::Push,
11176 control_flags(),
11177 0,
11178 0,
11179 6,
11180 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
11181 )
11182 .unwrap();
11183 let out = handler.handle_status_update(endpoint, malformed).unwrap();
11184 assert_eq!(out.len(), 1);
11185 assert_eq!(out[0].header.ty, FrameType::Error);
11186 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
11187 }
11188
11189 #[test]
11190 fn hello_rejected_when_connection_already_owns_client_routes() {
11191 let registry = Arc::new(Registry::default());
11192 let forwarding = Arc::new(ForwardingTable::default());
11193 let handler =
11194 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11195 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
11197 let client_connection = ConnectionId::new(202);
11198
11199 let responses = handler
11202 .handle_control(
11203 client_connection,
11204 hello_frame("aft-second", PROTOCOL_VERSION, 9),
11205 )
11206 .unwrap();
11207 assert_eq!(responses[0].header.ty, FrameType::Error);
11208 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
11209 assert!(registry.get_module("aft-second").unwrap().is_none());
11210 }
11211
11212 #[tokio::test]
11213 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
11214 let registry = Arc::new(Registry::default());
11215 let forwarding = Arc::new(ForwardingTable::default());
11216 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
11217 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
11218 hello_via_sink(
11219 &handler,
11220 &module_ctx,
11221 &mut module_rx,
11222 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
11223 )
11224 .await;
11225 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
11226 let pending = forwarding
11227 .begin_route_bind_relay_for_test(
11228 client_ctx.connection_id,
11229 client_ctx.egress.clone(),
11230 2,
11231 "alpha",
11232 )
11233 .unwrap();
11234 forwarding
11235 .complete_pending_relay(
11236 module_ctx.connection_id,
11237 pending.corr,
11238 RouteBindRelayOutcome::Accepted,
11239 )
11240 .unwrap();
11241 client_rx.try_recv().unwrap();
11242 for module_id in ["beta", "alpha"] {
11243 let replies = handler
11244 .handle_control_frame(
11245 &module_ctx,
11246 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
11247 )
11248 .await
11249 .unwrap();
11250 assert_eq!(replies.len(), 1, "second HELLO must be refused");
11251 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
11252 }
11253 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
11254 assert!(registry.get_module("beta").unwrap().is_none());
11255 assert!(matches!(
11256 forwarding
11257 .lookup_data_route(
11258 client_ctx.connection_id,
11259 pending.client_channel,
11260 pending.client_epoch,
11261 )
11262 .unwrap(),
11263 DataRoute::Client(DataRouteState::Bound(_))
11264 ));
11265 assert!(handler
11266 .hello_launch_nonces
11267 .lock()
11268 .unwrap()
11269 .presented(module_ctx.connection_id, Some("alpha-nonce")));
11270 assert!(module_rx.try_recv().is_err());
11271 }
11272
11273 #[test]
11274 fn reserved_module_hello_requires_matching_launch_nonce() {
11275 let registry = Arc::new(Registry::default());
11276 let supervisor = SupervisorHandle::new();
11277 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
11280 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11281
11282 let no_nonce = handler
11284 .handle_control(
11285 ConnectionId::new(1),
11286 hello_frame("vault", PROTOCOL_VERSION, 1),
11287 )
11288 .unwrap();
11289 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
11290 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
11291 assert!(registry.get_module("vault").unwrap().is_none());
11292
11293 let wrong = handler
11295 .handle_control(
11296 ConnectionId::new(2),
11297 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
11298 )
11299 .unwrap();
11300 assert_eq!(wrong[0].header.ty, FrameType::Error);
11301 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
11302 assert!(registry.get_module("vault").unwrap().is_none());
11303
11304 let ok = handler
11306 .handle_control(
11307 ConnectionId::new(3),
11308 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
11309 )
11310 .unwrap();
11311 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
11312 assert!(registry.get_module("vault").unwrap().is_some());
11313 }
11314
11315 #[test]
11316 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
11317 let registry = Arc::new(Registry::default());
11318 let supervisor = SupervisorHandle::new();
11319 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11320 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11321 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11322
11323 let squat = handler
11324 .handle_control(
11325 ConnectionId::new(1),
11326 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
11327 )
11328 .unwrap();
11329 assert_eq!(squat[0].header.ty, FrameType::Error);
11330 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
11331 assert!(parse_error(&squat[0])["message"]
11332 .as_str()
11333 .unwrap()
11334 .contains("fed:"));
11335
11336 let accepted_peer = handler
11337 .handle_control(
11338 ConnectionId::new(2),
11339 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
11340 )
11341 .unwrap();
11342 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
11343
11344 let accepted_short = handler
11345 .handle_control(
11346 ConnectionId::new(3),
11347 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
11348 )
11349 .unwrap();
11350 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
11351
11352 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
11353 let response = handler
11354 .handle_control(
11355 ConnectionId::new(conn),
11356 hello_frame(module_id, PROTOCOL_VERSION, conn),
11357 )
11358 .unwrap();
11359 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
11360 }
11361 }
11362
11363 #[test]
11364 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
11365 let registry = Arc::new(Registry::default());
11366 let supervisor = SupervisorHandle::new();
11367 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11368 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11369 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
11370 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11371
11372 let owner_nonce = handler
11373 .handle_control(
11374 ConnectionId::new(1),
11375 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
11376 )
11377 .unwrap();
11378 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
11379 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
11380 assert!(registry.get_module("fed:special").unwrap().is_none());
11381
11382 let exact_nonce = handler
11383 .handle_control(
11384 ConnectionId::new(2),
11385 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
11386 )
11387 .unwrap();
11388 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
11389 assert!(registry.get_module("fed:special").unwrap().is_some());
11390 }
11391
11392 #[test]
11393 fn non_reserved_module_ignores_launch_nonce() {
11394 let registry = Arc::new(Registry::default());
11395 let handler = ControlHandler::new(Arc::clone(®istry));
11398 let no_nonce = handler
11399 .handle_control(
11400 ConnectionId::new(1),
11401 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
11402 )
11403 .unwrap();
11404 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
11405 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
11406
11407 let echoed_nonce = handler
11408 .handle_control(
11409 ConnectionId::new(2),
11410 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
11411 )
11412 .unwrap();
11413 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
11414 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
11415 }
11416
11417 #[test]
11418 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
11419 let handler = ControlHandler::default();
11420 let conn = ConnectionId::new(1);
11421 let malformed = Frame::build(
11422 FrameType::Hello,
11423 control_flags(),
11424 0,
11425 0,
11426 3,
11427 b"{not json".to_vec(),
11428 )
11429 .unwrap();
11430
11431 let error = handler.handle_control(conn, malformed).unwrap();
11432 assert_eq!(error[0].header.ty, FrameType::Error);
11433 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
11434
11435 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11436 let pong = handler.handle_control(conn, ping).unwrap();
11437 assert_eq!(pong[0].header.ty, FrameType::Pong);
11438 assert_eq!(pong[0].header.corr, 4);
11439 }
11440
11441 #[test]
11442 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
11443 let registry = Arc::new(Registry::default());
11444 let handler = ControlHandler::new(Arc::clone(®istry));
11445
11446 handler
11447 .handle_control(
11448 ConnectionId::new(1),
11449 hello_frame("aft", PROTOCOL_VERSION, 1),
11450 )
11451 .unwrap();
11452 let duplicate = handler
11453 .handle_control(
11454 ConnectionId::new(2),
11455 hello_frame("aft", PROTOCOL_VERSION, 2),
11456 )
11457 .unwrap();
11458
11459 assert_eq!(duplicate[0].header.ty, FrameType::Error);
11460 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11461 let registration = registry.get_module("aft").unwrap().unwrap();
11462 assert_eq!(registration.connection_id, ConnectionId::new(1));
11463 }
11464
11465 #[test]
11466 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
11467 let registry = Arc::new(Registry::default());
11468 let forwarding = Arc::new(ForwardingTable::default());
11469 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
11470 let handler =
11471 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11472 .with_process_liveness(process_liveness);
11473 let (ctx, route_channel, route_epoch) =
11474 bind_liveness_route(®istry, &forwarding, "aft-dead");
11475 let responses = handler
11476 .handle_route_poll(
11477 &ctx,
11478 route_poll_frame(41, PollKind::Liveness, route_channel),
11479 route_channel,
11480 route_epoch,
11481 PollKind::Liveness,
11482 )
11483 .unwrap();
11484
11485 assert_eq!(responses.len(), 1);
11486 assert_eq!(responses[0].header.ty, FrameType::Response);
11487 assert_route_poll_liveness(&responses[0], false);
11488 }
11489
11490 #[test]
11491 fn liveness_poll_without_process_source_uses_bound_route() {
11492 let registry = Arc::new(Registry::default());
11493 let forwarding = Arc::new(ForwardingTable::default());
11494 let handler =
11495 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11496 let (ctx, route_channel, route_epoch) =
11497 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
11498 let responses = handler
11499 .handle_route_poll(
11500 &ctx,
11501 route_poll_frame(42, PollKind::Liveness, route_channel),
11502 route_channel,
11503 route_epoch,
11504 PollKind::Liveness,
11505 )
11506 .unwrap();
11507
11508 assert_route_poll_liveness(&responses[0], true);
11509 }
11510
11511 #[test]
11512 fn liveness_poll_untracked_process_source_uses_bound_route() {
11513 let registry = Arc::new(Registry::default());
11514 let forwarding = Arc::new(ForwardingTable::default());
11515 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
11516 let handler =
11517 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11518 .with_process_liveness(process_liveness);
11519 let (ctx, route_channel, route_epoch) =
11520 bind_liveness_route(®istry, &forwarding, "aft-untracked");
11521 let responses = handler
11522 .handle_route_poll(
11523 &ctx,
11524 route_poll_frame(43, PollKind::Liveness, route_channel),
11525 route_channel,
11526 route_epoch,
11527 PollKind::Liveness,
11528 )
11529 .unwrap();
11530
11531 assert_route_poll_liveness(&responses[0], true);
11532 }
11533
11534 #[tokio::test]
11535 async fn unknown_op_returns_unknown_control_op() {
11536 let handler = ControlHandler::default();
11537 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
11538 let request = Frame::build(
11539 FrameType::Request,
11540 control_flags(),
11541 0,
11542 0,
11543 55,
11544 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
11545 )
11546 .unwrap();
11547
11548 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11549
11550 assert_eq!(response.len(), 1);
11551 assert_eq!(response[0].header.ty, FrameType::Error);
11552 assert_eq!(response[0].header.corr, 55);
11553 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
11554 }
11555
11556 #[tokio::test]
11557 async fn supervisor_provenance_rejects_unknown_exact_module() {
11558 let handler = ControlHandler::default();
11559 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
11560 let request = Frame::build(
11561 FrameType::Request,
11562 control_flags(),
11563 0,
11564 0,
11565 57,
11566 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
11567 )
11568 .unwrap();
11569
11570 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11571
11572 assert_eq!(response.len(), 1);
11573 assert_eq!(response[0].header.ty, FrameType::Error);
11574 assert_eq!(response[0].header.corr, 57);
11575 let error = parse_error(&response[0]);
11576 assert_eq!(error["code"], "unknown_module");
11577 assert_eq!(error["message"], "module_id 'missing' is not supervised");
11578 }
11579
11580 #[test]
11581 fn provenance_probe_override_keeps_handler_tests_deterministic() {
11582 let expected = subc_control::RunningImageAgreement::Unavailable {
11583 reason: subc_control::RunningImageUnavailableReason::HashFailed,
11584 };
11585 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
11586 assert_eq!(handler.provenance_probe_override, Some(expected));
11587 }
11588
11589 #[test]
11590 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
11591 let verdict = reload_verdict(
11592 std::path::Path::new("/bin/new"),
11593 Some(std::path::Path::new("/bin/old")),
11594 subc_control::RunningImageAgreement::Unavailable {
11595 reason: subc_control::RunningImageUnavailableReason::HashFailed,
11596 },
11597 );
11598 assert!(matches!(
11599 verdict.path,
11600 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
11601 if configured == std::path::Path::new("/bin/new")
11602 && spawned_from == std::path::Path::new("/bin/old")
11603 ));
11604 }
11605
11606 #[test]
11607 fn reload_verdict_detects_replaced_image_at_same_path() {
11608 let image = subc_control::RunningImageAgreement::Mismatch {
11609 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
11610 digest: "old".into(),
11611 },
11612 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
11613 digest: "new".into(),
11614 },
11615 };
11616 let verdict = reload_verdict(
11617 std::path::Path::new("/bin/same"),
11618 Some(std::path::Path::new("/bin/same")),
11619 image.clone(),
11620 );
11621 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11622 assert_eq!(verdict.image, image);
11623 }
11624
11625 #[test]
11626 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
11627 let image = subc_control::RunningImageAgreement::Unavailable {
11628 reason: subc_control::RunningImageUnavailableReason::NotRunning,
11629 };
11630 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
11631 assert_eq!(
11632 verdict.path,
11633 subc_control::ReloadPathAgreement::Unavailable {
11634 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
11635 }
11636 );
11637 assert_eq!(verdict.image, image);
11638
11639 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
11640 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11641 };
11642 let verdict = reload_verdict(
11643 std::path::Path::new("/bin/same"),
11644 Some(std::path::Path::new("/bin/same")),
11645 unconfirmed.clone(),
11646 );
11647 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11648 assert_eq!(verdict.image, unconfirmed);
11649 }
11650
11651 #[test]
11652 fn reload_verdict_preserves_each_image_unavailability_reason() {
11653 use subc_control::RunningImageUnavailableReason as Reason;
11654
11655 for reason in [
11656 Reason::NotRunning,
11657 Reason::UnsupportedPlatform,
11658 Reason::RunningExecutableUnreadable,
11659 Reason::SpawnedPathUnreadable,
11660 Reason::HashFailed,
11661 Reason::ProcessIdentityUnconfirmed,
11662 Reason::Unknown("future_probe_reason".to_string()),
11663 ] {
11664 let image = subc_control::RunningImageAgreement::Unavailable {
11665 reason: reason.clone(),
11666 };
11667 let verdict = reload_verdict(
11668 std::path::Path::new("/bin/same"),
11669 Some(std::path::Path::new("/bin/same")),
11670 image.clone(),
11671 );
11672 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11673 assert_eq!(verdict.image, image, "{reason:?}");
11674 }
11675 }
11676
11677 #[tokio::test]
11678 async fn malformed_control_bodies_return_invalid_control_body() {
11679 let handler = ControlHandler::default();
11680 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
11681
11682 for (corr, body) in [
11683 (56, br#"{"route_channel":1}"#.as_slice()),
11684 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
11685 (
11686 58,
11687 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
11688 ),
11689 ] {
11690 let request = Frame::build(
11691 FrameType::Request,
11692 control_flags(),
11693 0,
11694 0,
11695 corr,
11696 body.to_vec(),
11697 )
11698 .unwrap();
11699 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11700
11701 assert_eq!(response.len(), 1);
11702 assert_eq!(response[0].header.ty, FrameType::Error);
11703 assert_eq!(response[0].header.corr, corr);
11704 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
11705 }
11706 }
11707
11708 #[tokio::test]
11709 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
11710 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11711 let registry = Arc::new(Registry::default());
11712 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11713 let router = Router::with_control_handler(Arc::clone(&control));
11714 let connection = router.begin_connection();
11715 let (ctx, mut rx) = route_ctx(connection.id());
11716
11717 router
11718 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
11719 .await
11720 .unwrap();
11721 let response = rx.recv().await.unwrap();
11722 let ack = parse_ack(&response);
11723 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11724 let channel = 1;
11725
11726 let goodbye =
11727 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
11728 router.route_for_connection(&ctx, goodbye).await.unwrap();
11729 assert!(rx.try_recv().is_err());
11730 assert!(registry.get_module("aft").unwrap().is_none());
11731
11732 router
11733 .route_for_connection(&ctx, channel_request(channel, 13))
11734 .await
11735 .unwrap();
11736 let error_frame = rx.recv().await.unwrap();
11737 assert_eq!(error_frame.header.ty, FrameType::Error);
11738 assert_eq!(error_frame.header.channel, channel);
11739 let captured = crate::router::test_log::captured_logs(&logs);
11740 assert_eq!(
11741 captured
11742 .lines()
11743 .filter(|line| {
11744 line.contains("module_id=aft")
11745 && line.contains("reason=explicit_goodbye")
11746 && line.contains("module registration ended")
11747 })
11748 .count(),
11749 1,
11750 "unexpected GOODBYE registry log: {captured}"
11751 );
11752 }
11753
11754 #[tokio::test]
11755 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
11756 let registry = Arc::new(Registry::default());
11757 let handler = ControlHandler::new(registry).with_capability_config(
11758 [("prov".to_string(), true), ("cons".to_string(), true)],
11759 BTreeMap::new(),
11760 );
11761 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
11762 register_capability_manifest(
11763 &handler,
11764 &provider_ctx,
11765 &mut provider_rx,
11766 capability_manifest("prov", &["thing/v1"], &[]),
11767 1,
11768 )
11769 .await;
11770 let mut consumer = capability_manifest("cons", &[], &[]);
11771 consumer.capabilities.as_mut().unwrap().requires.push(
11772 subc_protocol::manifest::CapabilityRequirement {
11773 capability: "thing/v1".to_string(),
11774 need: subc_protocol::manifest::CapabilityNeed::Required,
11775 },
11776 );
11777 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
11778 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
11779 assert_eq!(
11780 handler.capability_evaluator.verdict("cons", "thing/v1"),
11781 Some(CapabilityVerdict::Provided)
11782 );
11783 let (mut client_rx, _) = open_route_for_capability_test(
11784 &handler,
11785 &provider_ctx,
11786 &mut provider_rx,
11787 703,
11788 3,
11789 "prov",
11790 None,
11791 )
11792 .await;
11793 let goodbye =
11794 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11795 handler
11796 .handle_control_frame(&provider_ctx, goodbye)
11797 .await
11798 .unwrap();
11799 assert_eq!(
11800 handler.capability_evaluator.verdict("cons", "thing/v1"),
11801 Some(CapabilityVerdict::NeverProvided)
11802 );
11803 let closed = client_rx
11804 .try_recv()
11805 .expect("GOODBYE pushes route.closed before route GOODBYE");
11806 assert!(
11807 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
11808 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
11809 );
11810 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
11811 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11812 }
11813
11814 #[tokio::test]
11815 async fn dropping_router_connection_releases_registration() {
11816 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11817 let registry = Arc::new(Registry::default());
11818 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11819 let router = Router::with_control_handler(Arc::clone(&control));
11820 let connection = router.begin_connection();
11821 let connection_id = connection.id();
11822 let (ctx, mut rx) = route_ctx(connection_id);
11823
11824 router
11825 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
11826 .await
11827 .unwrap();
11828 let response = rx.recv().await.unwrap();
11829 let ack = parse_ack(&response);
11830 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11831 assert!(registry.get_module("aft").unwrap().is_some());
11832
11833 drop(connection);
11834
11835 assert!(registry.get_module("aft").unwrap().is_none());
11836 assert_eq!(registry.active_registration_count().unwrap(), 0);
11837
11838 control
11839 .cleanup_connection(ConnectionId::new(u64::MAX))
11840 .unwrap();
11841 let captured = crate::router::test_log::captured_logs(&logs);
11842 println!("captured registry lifecycle logs:\n{captured}");
11843 let events: Vec<_> = captured
11844 .lines()
11845 .filter(|line| {
11846 line.contains("module registered module_id=aft ")
11847 || line.contains("module registration ended")
11848 })
11849 .collect();
11850 assert_eq!(
11851 events.len(),
11852 2,
11853 "unexpected registry lifecycle logs: {captured}"
11854 );
11855 assert!(events[0].contains("module registered module_id=aft "));
11856 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
11857 assert!(events[1].contains(&format!(
11858 "module_id=aft connection_id={}",
11859 connection_id.get()
11860 )));
11861 assert!(events[1].contains("reason=connection_closed"));
11862 assert!(events[1].contains("module registration ended"));
11863 assert_eq!(
11864 captured
11865 .lines()
11866 .filter(|line| line.contains("module registered module_id=aft "))
11867 .count(),
11868 1,
11869 "legacy registration admission line must appear once: {captured}"
11870 );
11871 }
11872
11873 #[tokio::test]
11874 async fn hello_registration_keeps_legacy_module_registered_line_once() {
11875 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11876 let registry = Arc::new(Registry::default());
11877 let control = ControlHandler::new(Arc::clone(®istry));
11878 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
11879 hello_via_sink(
11880 &control,
11881 &ctx,
11882 &mut rx,
11883 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
11884 )
11885 .await;
11886
11887 let captured = crate::router::test_log::captured_logs(&logs);
11888 let admissions: Vec<_> = captured
11889 .lines()
11890 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
11891 .collect();
11892 assert_eq!(
11893 admissions.len(),
11894 1,
11895 "legacy admission line must remain exactly once: {captured}"
11896 );
11897 assert!(admissions[0].contains("routable_provider=true"));
11898 assert!(admissions[0].contains("connection_id=777"));
11899 }
11900
11901 fn capability_manifest(
11902 module_id: &str,
11903 provides: &[&str],
11904 must_never_reach: &[&str],
11905 ) -> ModuleManifest {
11906 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
11907 manifest.capabilities = Some(CapabilityDeclarations {
11908 provides: provides
11909 .iter()
11910 .map(|capability| (*capability).to_string())
11911 .collect(),
11912 requires: Vec::new(),
11913 must_never_reach: must_never_reach
11914 .iter()
11915 .map(|capability| (*capability).to_string())
11916 .collect(),
11917 });
11918 manifest
11919 }
11920
11921 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
11922 Frame::build(
11923 FrameType::Hello,
11924 control_flags(),
11925 0,
11926 0,
11927 corr,
11928 serde_json::to_vec(&ModuleHelloBody {
11929 protocol_ver: manifest.protocol_ver,
11930 manifest,
11931 control_ops: None,
11932 launch_nonce: None,
11933 })
11934 .expect("capability test HELLO serializes"),
11935 )
11936 .expect("capability test HELLO frame builds")
11937 }
11938
11939 fn catalog_update_with_capabilities_frame(
11940 corr: u64,
11941 capabilities: CapabilityDeclarations,
11942 ) -> Frame {
11943 Frame::build(
11944 FrameType::Request,
11945 control_flags(),
11946 0,
11947 0,
11948 corr,
11949 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
11950 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
11951 capabilities: Some(capabilities),
11952 ready: None,
11953 })
11954 .expect("capability catalog.update serializes"),
11955 )
11956 .expect("capability catalog.update frame builds")
11957 }
11958
11959 async fn register_capability_manifest(
11960 handler: &ControlHandler,
11961 ctx: &RouteCtx,
11962 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11963 manifest: ModuleManifest,
11964 corr: u64,
11965 ) {
11966 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
11967 }
11968
11969 async fn open_route_for_capability_test(
11970 handler: &ControlHandler,
11971 target_ctx: &RouteCtx,
11972 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11973 client_connection_id: u64,
11974 corr: u64,
11975 target_module_id: &str,
11976 consumer_identity: Option<ConsumerIdentity>,
11977 ) -> (
11978 mpsc::Receiver<crate::router::OutboundFrame>,
11979 ModuleControlRequest,
11980 ) {
11981 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
11982 let route_handler = handler.clone();
11983 let target_module_id = target_module_id.to_string();
11984 let route_task = tokio::spawn(async move {
11985 route_handler
11986 .handle_control_frame(
11987 &client_ctx,
11988 route_open_frame_with_admission_facts(
11989 corr,
11990 &target_module_id,
11991 unique_project_root("admission-facts"),
11992 consumer_identity,
11993 None,
11994 ),
11995 )
11996 .await
11997 .expect("capability test route.open succeeds")
11998 });
11999 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
12000 .await
12001 .expect("capability test route.open must reach route.bind")
12002 .expect("target control receiver stays open");
12003 let bind_request: ModuleControlRequest =
12004 serde_json::from_slice(&bind.body).expect("route.bind decodes");
12005 handler
12006 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
12007 .await
12008 .expect("capability test route.bind ACK succeeds");
12009 assert!(route_task.await.expect("route.open task joins").is_empty());
12010 let opened = client_rx
12011 .recv()
12012 .await
12013 .expect("successful route.open publishes a response");
12014 assert!(matches!(
12015 serde_json::from_slice::<ClientControlResponse>(&opened.body),
12016 Ok(ClientControlResponse::RouteOpen { .. })
12017 ));
12018 (client_rx, bind_request)
12019 }
12020
12021 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
12022 assert_eq!(frame.header.ty, FrameType::Push);
12023 assert_eq!(frame.header.channel, 0);
12024 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
12025 .expect("route.closed control push decodes");
12026 let ClientControlPush::RouteClosed { channels, .. } = &push else {
12027 panic!("expected route.closed");
12028 };
12029 assert_eq!(channels.len(), 1, "exactly one violating route closed");
12030 let channels = channels.clone();
12031 assert_eq!(
12032 push,
12033 ClientControlPush::RouteClosed {
12034 module_id: target_module_id.to_string(),
12035 channels,
12036 reason: RouteCloseReason::CapabilityDenied,
12037 drained: false,
12038 abandoned: 0,
12039 excluded_subscriptions: 0,
12040 terminal: Some(false),
12041 }
12042 );
12043 }
12044
12045 #[tokio::test]
12046 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
12047 let registry = Arc::new(Registry::default());
12048 let forwarding = Arc::new(ForwardingTable::default());
12049 let supervisor = SupervisorHandle::new();
12050 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12051 let handler =
12052 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12053 .with_supervisor(supervisor);
12054 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
12055 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
12056 register_capability_manifest(
12057 &handler,
12058 &target_ctx,
12059 &mut target_rx,
12060 capability_manifest("target", &["credentials-provider/v1"], &[]),
12061 1,
12062 )
12063 .await;
12064 register_capability_manifest(
12065 &handler,
12066 &opener_ctx,
12067 &mut opener_rx,
12068 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12069 2,
12070 )
12071 .await;
12072
12073 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
12074 let replies = handler
12075 .handle_control_frame(
12076 &client_ctx,
12077 route_open_frame_with_admission_facts(
12078 3,
12079 "target",
12080 unique_project_root("admission-facts"),
12081 Some(ConsumerIdentity {
12082 module_id: "opener".to_string(),
12083 launch_nonce: "opener-nonce".to_string(),
12084 }),
12085 None,
12086 ),
12087 )
12088 .await
12089 .expect("denied route.open returns a typed frame");
12090 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
12091 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12092 assert!(
12093 target_rx.try_recv().is_err(),
12094 "forbidden route.open must not relay route.bind"
12095 );
12096 }
12097
12098 #[tokio::test]
12099 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
12100 let registry = Arc::new(Registry::default());
12101 let forwarding = Arc::new(ForwardingTable::default());
12102 let supervisor = SupervisorHandle::new();
12103 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12104 let handler =
12105 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12106 .with_supervisor(supervisor);
12107 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
12108 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
12109 register_capability_manifest(
12110 &handler,
12111 &target_ctx,
12112 &mut target_rx,
12113 capability_manifest("target", &["credentials-provider/v1"], &[]),
12114 1,
12115 )
12116 .await;
12117 register_capability_manifest(
12118 &handler,
12119 &old_opener_ctx,
12120 &mut old_opener_rx,
12121 capability_manifest("opener", &[], &[]),
12122 2,
12123 )
12124 .await;
12125 let (mut client_rx, _) = open_route_for_capability_test(
12126 &handler,
12127 &target_ctx,
12128 &mut target_rx,
12129 712,
12130 3,
12131 "target",
12132 Some(ConsumerIdentity {
12133 module_id: "opener".to_string(),
12134 launch_nonce: "opener-nonce".to_string(),
12135 }),
12136 )
12137 .await;
12138 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12139
12140 handler
12141 .cleanup_connection(old_opener_ctx.connection_id)
12142 .expect("old opener registration cleans up");
12143 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
12144 register_capability_manifest(
12145 &handler,
12146 &new_opener_ctx,
12147 &mut new_opener_rx,
12148 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12149 4,
12150 )
12151 .await;
12152
12153 assert_capability_denied_push(
12154 client_rx
12155 .try_recv()
12156 .expect("HELLO deny addition must emit route.closed")
12157 .frame,
12158 "target",
12159 );
12160 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12161 assert!(matches!(
12162 target_rx.try_recv(),
12163 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
12164 ));
12165 }
12166
12167 #[tokio::test]
12168 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
12169 let registry = Arc::new(Registry::default());
12170 let forwarding = Arc::new(ForwardingTable::default());
12171 let supervisor = SupervisorHandle::new();
12172 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12173 let handler =
12174 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12175 .with_supervisor(supervisor);
12176 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
12177 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
12178 register_capability_manifest(
12179 &handler,
12180 &target_ctx,
12181 &mut target_rx,
12182 capability_manifest("target", &[], &[]),
12183 1,
12184 )
12185 .await;
12186 register_capability_manifest(
12187 &handler,
12188 &opener_ctx,
12189 &mut opener_rx,
12190 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12191 2,
12192 )
12193 .await;
12194 let (mut client_rx, _) = open_route_for_capability_test(
12195 &handler,
12196 &target_ctx,
12197 &mut target_rx,
12198 722,
12199 3,
12200 "target",
12201 Some(ConsumerIdentity {
12202 module_id: "opener".to_string(),
12203 launch_nonce: "opener-nonce".to_string(),
12204 }),
12205 )
12206 .await;
12207 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12208
12209 let replies = handler
12210 .handle_control_frame(
12211 &target_ctx,
12212 catalog_update_with_capabilities_frame(
12213 4,
12214 CapabilityDeclarations {
12215 provides: vec!["credentials-provider/v1".to_string()],
12216 requires: Vec::new(),
12217 must_never_reach: Vec::new(),
12218 },
12219 ),
12220 )
12221 .await
12222 .expect("claim catalog.update succeeds");
12223 assert!(matches!(
12224 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
12225 Ok(ModuleControlResponseToModule::CatalogUpdate {})
12226 ));
12227 assert_capability_denied_push(
12228 client_rx
12229 .try_recv()
12230 .expect("claim addition must emit route.closed")
12231 .frame,
12232 "target",
12233 );
12234 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12235 assert!(matches!(
12236 target_rx.try_recv(),
12237 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
12238 ));
12239 }
12240
12241 #[tokio::test]
12242 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
12243 let registry = Arc::new(Registry::default());
12244 let forwarding = Arc::new(ForwardingTable::default());
12245 let supervisor = SupervisorHandle::new();
12246 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12247 let handler =
12248 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12249 .with_supervisor(supervisor);
12250 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
12251 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
12252 register_capability_manifest(
12253 &handler,
12254 &target_ctx,
12255 &mut target_rx,
12256 capability_manifest("target", &["credentials-provider/v1"], &[]),
12257 1,
12258 )
12259 .await;
12260 register_capability_manifest(
12261 &handler,
12262 &opener_ctx,
12263 &mut opener_rx,
12264 capability_manifest("opener", &[], &[]),
12265 2,
12266 )
12267 .await;
12268 let (mut client_rx, _) = open_route_for_capability_test(
12269 &handler,
12270 &target_ctx,
12271 &mut target_rx,
12272 732,
12273 3,
12274 "target",
12275 Some(ConsumerIdentity {
12276 module_id: "opener".to_string(),
12277 launch_nonce: "opener-nonce".to_string(),
12278 }),
12279 )
12280 .await;
12281 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12282
12283 handler
12284 .handle_control_frame(
12285 &target_ctx,
12286 catalog_update_with_capabilities_frame(
12287 4,
12288 CapabilityDeclarations {
12289 provides: Vec::new(),
12290 requires: Vec::new(),
12291 must_never_reach: Vec::new(),
12292 },
12293 ),
12294 )
12295 .await
12296 .expect("claim removal catalog.update succeeds");
12297 assert_eq!(
12298 forwarding.active_binding_count().unwrap(),
12299 1,
12300 "removing an attested target claim must leave the route census unchanged"
12301 );
12302 assert!(
12303 client_rx.try_recv().is_err(),
12304 "claim removal must not emit route.closed capability_denied"
12305 );
12306 assert!(
12307 target_rx.try_recv().is_err(),
12308 "claim removal must not send the target a route GOODBYE"
12309 );
12310 }
12311
12312 #[tokio::test]
12315 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
12316 let registry = Arc::new(Registry::default());
12317 let forwarding = Arc::new(ForwardingTable::default());
12318 let supervisor = SupervisorHandle::new();
12319 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12320 let handler =
12321 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12322 .with_supervisor(supervisor);
12323 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
12324 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
12325 register_capability_manifest(
12326 &handler,
12327 &target_ctx,
12328 &mut target_rx,
12329 capability_manifest("target", &["credentials-provider/v1"], &[]),
12330 1,
12331 )
12332 .await;
12333 register_capability_manifest(
12334 &handler,
12335 &opener_ctx,
12336 &mut opener_rx,
12337 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12338 2,
12339 )
12340 .await;
12341
12342 let (_client_rx, bind) = open_route_for_capability_test(
12343 &handler,
12344 &target_ctx,
12345 &mut target_rx,
12346 742,
12347 3,
12348 "target",
12349 None,
12350 )
12351 .await;
12352 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12353 panic!("direct scope-honesty route must bind");
12354 };
12355 assert_eq!(principal, Some(Principal::Direct));
12356 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12357 }
12358
12359 #[tokio::test]
12362 async fn must_never_reach_self_route_is_capability_forbidden() {
12363 let registry = Arc::new(Registry::default());
12364 let forwarding = Arc::new(ForwardingTable::default());
12365 let supervisor = SupervisorHandle::new();
12366 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
12367 let handler =
12368 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12369 .with_supervisor(supervisor);
12370 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
12371 register_capability_manifest(
12372 &handler,
12373 &self_ctx,
12374 &mut self_rx,
12375 capability_manifest(
12376 "self-provider",
12377 &["credentials-provider/v1"],
12378 &["credentials-provider/v1"],
12379 ),
12380 1,
12381 )
12382 .await;
12383
12384 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
12385 let replies = handler
12386 .handle_control_frame(
12387 &client_ctx,
12388 route_open_frame_with_admission_facts(
12389 2,
12390 "self-provider",
12391 unique_project_root("admission-facts"),
12392 Some(ConsumerIdentity {
12393 module_id: "self-provider".to_string(),
12394 launch_nonce: "self-nonce".to_string(),
12395 }),
12396 None,
12397 ),
12398 )
12399 .await
12400 .expect("self-route refusal returns a typed frame");
12401 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
12402 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12403 assert!(
12404 self_rx.try_recv().is_err(),
12405 "self denial must not relay route.bind"
12406 );
12407 }
12408
12409 #[test]
12410 fn unsupported_channel_zero_frame_returns_error() {
12411 let handler = ControlHandler::default();
12412 let request = Frame::build(
12413 FrameType::Request,
12414 control_flags(),
12415 0,
12416 0,
12417 21,
12418 b"opaque".to_vec(),
12419 )
12420 .unwrap();
12421
12422 let response = handler
12423 .handle_control(ConnectionId::new(1), request)
12424 .unwrap();
12425
12426 assert_eq!(response[0].header.ty, FrameType::Error);
12427 assert_eq!(
12428 parse_error(&response[0])["code"],
12429 "unsupported_control_frame"
12430 );
12431 }
12432
12433 mod swap {
12438 use super::*;
12439
12440 const INCUMBENT: ConnectionId = ConnectionId::new(30);
12441 const CANDIDATE: ConnectionId = ConnectionId::new(40);
12442
12443 struct Swap {
12444 registry: Arc<Registry>,
12445 forwarding: Arc<ForwardingTable>,
12446 handler: ControlHandler,
12447 incumbent_ctx: RouteCtx,
12448 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12449 candidate_ctx: RouteCtx,
12450 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12451 }
12452
12453 async fn swap_with_incumbent() -> Swap {
12454 let registry = Arc::new(Registry::default());
12455 let forwarding = Arc::new(ForwardingTable::default());
12456 let handler =
12457 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12458 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
12459 hello_via_sink(
12460 &handler,
12461 &incumbent_ctx,
12462 &mut incumbent_rx,
12463 hello_frame("aft", PROTOCOL_VERSION, 7),
12464 )
12465 .await;
12466 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
12467 Swap {
12468 registry,
12469 forwarding,
12470 handler,
12471 incumbent_ctx,
12472 incumbent_rx,
12473 candidate_ctx,
12474 candidate_rx,
12475 }
12476 }
12477
12478 fn register_candidate(swap: &Swap, ready: Option<bool>) {
12479 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
12480 candidate_manifest.ready = ready;
12481 let registration = swap
12482 .registry
12483 .register_candidate_with_control_ops(
12484 candidate_manifest,
12485 PROTOCOL_VERSION,
12486 CANDIDATE,
12487 module_baseline_control_ops(),
12488 )
12489 .unwrap();
12490 swap.forwarding
12491 .register_candidate_module_connection(
12492 CANDIDATE,
12493 "aft".to_string(),
12494 PROTOCOL_VERSION,
12495 manifest_concurrency(®istration.manifest),
12496 swap.candidate_ctx.egress.clone(),
12497 )
12498 .unwrap();
12499 }
12500
12501 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
12502 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
12503 swap.registry.promote_candidate("aft").unwrap().unwrap();
12504 cutover.incumbent.unwrap()
12505 }
12506
12507 fn keyed_total(counters: &Value, key: &str) -> u64 {
12508 counters[key]
12509 .as_object()
12510 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
12511 .unwrap_or(0)
12512 }
12513
12514 #[tokio::test]
12515 async fn replacement_logs_old_end_and_new_admission_once() {
12516 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12517 let mut swap = swap_with_incumbent().await;
12518 swap.handler
12519 .supervisor
12520 .open_swap("aft", "candidate-nonce".to_string());
12521 hello_via_sink(
12522 &swap.handler,
12523 &swap.candidate_ctx,
12524 &mut swap.candidate_rx,
12525 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
12526 )
12527 .await;
12528 cutover(&swap);
12529 swap.handler.cleanup_connection(INCUMBENT).unwrap();
12530
12531 let captured = crate::router::test_log::captured_logs(&logs);
12532 println!("captured replacement registry logs:\n{captured}");
12533 let new_admissions: Vec<_> = captured
12534 .lines()
12535 .filter(|line| {
12536 line.contains("connection_id=40")
12537 && line.contains("swap candidate registered; not routable until cutover")
12538 })
12539 .collect();
12540 assert_eq!(
12541 new_admissions.len(),
12542 1,
12543 "unexpected admission logs: {captured}"
12544 );
12545 assert!(new_admissions[0].contains("module_id=aft"));
12546 assert!(new_admissions[0].contains("connection_id=40"));
12547 assert!(
12548 new_admissions[0].contains("swap candidate registered; not routable until cutover")
12549 );
12550 assert!(new_admissions[0].contains("ready=true"));
12551
12552 let old_admissions: Vec<_> = captured
12553 .lines()
12554 .filter(|line| {
12555 line.contains("module registered module_id=aft ")
12556 && line.contains("connection_id=30")
12557 })
12558 .collect();
12559 assert_eq!(
12560 old_admissions.len(),
12561 1,
12562 "unexpected incumbent admission: {captured}"
12563 );
12564
12565 let old_ends: Vec<_> = captured
12566 .lines()
12567 .filter(|line| {
12568 line.contains("connection_id=30") && line.contains("module registration ended")
12569 })
12570 .collect();
12571 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
12572 assert!(old_ends[0].contains("module_id=aft"));
12573 assert!(old_ends[0].contains("connection_id=30"));
12574 assert!(old_ends[0].contains("reason=replaced"));
12575 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
12576 assert_eq!(
12577 captured
12578 .lines()
12579 .filter(|line| line.contains("module registration promoted"))
12580 .count(),
12581 1,
12582 "unexpected promotion logs: {captured}"
12583 );
12584 }
12585
12586 #[tokio::test]
12592 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
12593 let mut swap = swap_with_incumbent().await;
12594 let handler = swap.handler.clone();
12595
12596 let cotenant = ConnectionId::new(31);
12598 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
12599 let (cotenant_task, cotenant_bind) = relay_route_open(
12600 &handler,
12601 cotenant,
12602 &cotenant_ctx.egress,
12603 &mut swap.incumbent_rx,
12604 100,
12605 "aft",
12606 "swap-cotenant",
12607 )
12608 .await;
12609 handler
12610 .handle_control_frame(
12611 &swap.incumbent_ctx,
12612 route_bind_ack(cotenant_bind.header.corr),
12613 )
12614 .await
12615 .unwrap();
12616 assert!(cotenant_task.await.unwrap().is_empty());
12617 let (cotenant_channel, cotenant_epoch) =
12618 published_route(&cotenant_rx.recv().await.unwrap());
12619
12620 let caller = ConnectionId::new(32);
12622 let (caller_ctx, mut caller_rx) = route_ctx(caller);
12623 let (caller_task, caller_bind) = relay_route_open(
12624 &handler,
12625 caller,
12626 &caller_ctx.egress,
12627 &mut swap.incumbent_rx,
12628 101,
12629 "aft",
12630 "swap-caller",
12631 )
12632 .await;
12633 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
12634
12635 register_candidate(&swap, None);
12636 cutover(&swap);
12637
12638 let ack = handler
12640 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
12641 .await;
12642 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
12643 if module_loop_error.is_some() {
12644 handler.cleanup_connection(INCUMBENT).unwrap();
12647 }
12648
12649 assert!(
12651 cotenant_rx.try_recv().is_err(),
12652 "the co-tenant route on the incumbent was torn down by one late ack: \
12653 {module_loop_error:?}"
12654 );
12655 assert!(matches!(
12656 swap.forwarding
12657 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
12658 .unwrap(),
12659 DataRoute::Client(DataRouteState::Bound(_))
12660 ));
12661 assert_eq!(module_loop_error, None);
12662 assert!(swap
12663 .registry
12664 .get_module_by_connection(INCUMBENT)
12665 .unwrap()
12666 .is_some());
12667
12668 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
12670 .await
12671 .expect("the incumbent is told to drop the abandoned binding")
12672 .unwrap()
12673 .frame;
12674 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12675 assert_eq!(goodbye.header.channel, abandoned_channel);
12676 assert_eq!(goodbye.header.epoch, abandoned_epoch);
12677 assert!(swap.incumbent_rx.try_recv().is_err());
12678
12679 let response = caller_task.await.unwrap();
12681 assert_eq!(response.len(), 1);
12682 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
12683 assert!(caller_rx.try_recv().is_err());
12684
12685 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
12689 let counters = handler.counters().snapshot();
12690 assert_eq!(
12691 keyed_total(&counters, "route_open_accepted_by_principal"),
12692 1
12693 );
12694 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
12695 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
12696 }
12697
12698 #[tokio::test]
12702 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
12703 let mut swap = swap_with_incumbent().await;
12704 register_candidate(&swap, None);
12705 let incumbent = cutover(&swap);
12706 swap.forwarding
12707 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
12708 .unwrap()
12709 .expect("the incumbent is still registered");
12710
12711 let client = ConnectionId::new(33);
12712 let (client_ctx, mut client_rx) = route_ctx(client);
12713 let route_handler = swap.handler.clone();
12714 let open_ctx = RouteCtx {
12715 connection_id: client,
12716 egress: client_ctx.egress.clone(),
12717 };
12718 let mut route_task = tokio::spawn(async move {
12719 route_handler
12720 .handle_control_frame(
12721 &open_ctx,
12722 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
12723 )
12724 .await
12725 .unwrap()
12726 });
12727 let bind = tokio::select! {
12728 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
12729 response = &mut route_task => {
12730 let response = response.unwrap();
12731 panic!(
12732 "post-cutover route.open was refused instead of relayed to the candidate: {}",
12733 parse_error(&response[0])["code"]
12734 );
12735 }
12736 };
12737 swap.handler
12738 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
12739 .await
12740 .unwrap();
12741 assert!(route_task.await.unwrap().is_empty());
12742 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
12743 match swap
12744 .forwarding
12745 .lookup_data_route(client, channel, epoch)
12746 .unwrap()
12747 {
12748 DataRoute::Client(DataRouteState::Bound(route)) => {
12749 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
12750 }
12751 other => panic!("expected a bound route on the candidate, got {other:?}"),
12752 }
12753 assert!(swap.incumbent_rx.try_recv().is_err());
12754 }
12755
12756 #[tokio::test]
12761 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
12762 let swap = swap_with_incumbent().await;
12763 register_candidate(&swap, Some(false));
12764 let update = Frame::build(
12765 FrameType::Request,
12766 control_flags(),
12767 0,
12768 0,
12769 55,
12770 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
12771 provides: manifest("aft", PROTOCOL_VERSION).provides,
12772 capabilities: None,
12773 ready: Some(true),
12774 })
12775 .unwrap(),
12776 )
12777 .unwrap();
12778
12779 let replies = swap
12780 .handler
12781 .handle_control_frame(&swap.candidate_ctx, update)
12782 .await
12783 .unwrap();
12784
12785 assert_eq!(replies.len(), 1);
12786 assert_eq!(
12787 replies[0].header.ty,
12788 FrameType::Response,
12789 "candidate catalog.update was refused: {:?}",
12790 serde_json::from_slice::<Value>(&replies[0].body).ok()
12791 );
12792 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
12793 assert_eq!(
12794 swap.registry
12795 .get_module("aft")
12796 .unwrap()
12797 .unwrap()
12798 .connection_id,
12799 INCUMBENT
12800 );
12801 }
12802 }
12803
12804 mod swap_admission {
12808 use super::*;
12809
12810 const INCUMBENT_NONCE: &str = "incumbent-nonce";
12811 const CANDIDATE_NONCE: &str = "candidate-nonce";
12812
12813 fn handler_with_incumbent(
12814 module_id: &str,
12815 reserved: bool,
12816 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
12817 let registry = Arc::new(Registry::default());
12818 let supervisor = SupervisorHandle::new();
12819 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
12820 if reserved {
12821 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
12822 }
12823 let handler =
12824 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
12825 let incumbent = handler
12826 .handle_control(
12827 ConnectionId::new(1),
12828 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
12829 )
12830 .unwrap();
12831 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
12832 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
12833 (registry, supervisor, handler)
12834 }
12835
12836 #[test]
12842 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
12843 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
12844
12845 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
12846 let replies = handler
12847 .handle_control(
12848 ConnectionId::new(connection),
12849 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
12850 )
12851 .unwrap();
12852 assert_eq!(replies[0].header.ty, FrameType::Error);
12853 assert_eq!(
12854 parse_error(&replies[0])["code"],
12855 "swap_token_invalid",
12856 "nonce {nonce:?}"
12857 );
12858 }
12859 assert!(registry.get_candidate("aft").unwrap().is_none());
12860 assert_eq!(
12861 registry.get_module("aft").unwrap().unwrap().connection_id,
12862 ConnectionId::new(1)
12863 );
12864
12865 let admitted = handler
12868 .handle_control(
12869 ConnectionId::new(4),
12870 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
12871 )
12872 .unwrap();
12873 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
12874 assert_eq!(
12875 registry
12876 .get_candidate("aft")
12877 .unwrap()
12878 .unwrap()
12879 .connection_id,
12880 ConnectionId::new(4)
12881 );
12882 assert_eq!(
12883 registry.get_module("aft").unwrap().unwrap().connection_id,
12884 ConnectionId::new(1),
12885 "the candidate must not take the active slot"
12886 );
12887 let replayed = handler
12888 .handle_control(
12889 ConnectionId::new(5),
12890 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
12891 )
12892 .unwrap();
12893 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
12894
12895 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
12899 let squatter = handler
12900 .handle_control(
12901 ConnectionId::new(6),
12902 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
12903 )
12904 .unwrap();
12905 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
12906 assert!(
12907 registry.get_module("aft").unwrap().is_none(),
12908 "a squatter took the active slot of an id being swapped"
12909 );
12910 }
12911
12912 #[test]
12917 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
12918 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
12919
12920 let replies = handler
12921 .handle_control(
12922 ConnectionId::new(2),
12923 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12924 )
12925 .unwrap();
12926
12927 assert_eq!(
12928 replies[0].header.ty,
12929 FrameType::HelloAck,
12930 "reserved candidate refused: {:?}",
12931 serde_json::from_slice::<Value>(&replies[0].body).ok()
12932 );
12933 assert_eq!(
12934 registry
12935 .get_candidate("vault")
12936 .unwrap()
12937 .unwrap()
12938 .connection_id,
12939 ConnectionId::new(2)
12940 );
12941 }
12942
12943 #[test]
12946 fn without_an_open_swap_the_ordinary_gates_decide() {
12947 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
12948 supervisor.close_swap("vault");
12949
12950 let candidate = handler
12951 .handle_control(
12952 ConnectionId::new(2),
12953 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
12954 )
12955 .unwrap();
12956 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
12957 let duplicate = handler
12958 .handle_control(
12959 ConnectionId::new(3),
12960 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
12961 )
12962 .unwrap();
12963 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12964 assert!(registry.get_candidate("vault").unwrap().is_none());
12965 }
12966 }
12967
12968 mod scopes {
12972 mod delta {
12973 include!("scope_handler_delta_tests.rs");
12974 }
12975
12976 use subc_protocol::scope::{
12977 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
12978 ScopeStatus,
12979 };
12980
12981 use super::*;
12982
12983 const OWNER: &str = "prefrontal-core";
12984
12985 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
12986 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
12987 }
12988
12989 async fn call(
12990 handler: &ControlHandler,
12991 ctx: &RouteCtx,
12992 request: &ModuleControlRequestFromModule,
12993 ) -> Frame {
12994 let body = serde_json::to_vec(request).unwrap();
12995 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
12996 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
12997 assert_eq!(replies.len(), 1, "{replies:?}");
12998 replies.pop().unwrap()
12999 }
13000
13001 async fn sync(
13002 handler: &ControlHandler,
13003 ctx: &RouteCtx,
13004 generation: u64,
13005 scopes: Vec<ScopeRecord>,
13006 ) -> Result<ModuleControlResponseToModule, String> {
13007 let reply = call(
13008 handler,
13009 ctx,
13010 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
13011 )
13012 .await;
13013 match reply.header.ty {
13014 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
13015 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
13016 }
13017 }
13018
13019 async fn describe(
13020 handler: &ControlHandler,
13021 ctx: &RouteCtx,
13022 owner: &str,
13023 scope_ref: &str,
13024 ) -> ModuleControlResponseToModule {
13025 let reply = call(
13026 handler,
13027 ctx,
13028 &ModuleControlRequestFromModule::ScopeDescribe {
13029 owner: Principal::Reserved {
13030 module_id: owner.to_string(),
13031 },
13032 scope_ref: scope_ref.to_string(),
13033 },
13034 )
13035 .await;
13036 assert_eq!(
13037 reply.header.ty,
13038 FrameType::Response,
13039 "{:?}",
13040 parse_error(&reply)
13041 );
13042 serde_json::from_slice(&reply.body).unwrap()
13043 }
13044
13045 async fn module(
13047 handler: &ControlHandler,
13048 connection: u64,
13049 module_id: &str,
13050 nonce: Option<&str>,
13051 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
13052 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
13053 hello_via_sink(
13054 handler,
13055 &ctx,
13056 &mut rx,
13057 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
13058 )
13059 .await;
13060 (ctx, rx)
13061 }
13062
13063 #[tokio::test]
13066 async fn a_client_connection_cannot_sync_or_describe() {
13067 let handler = ControlHandler::new(Arc::new(Registry::default()));
13068 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
13069 for request in [
13070 ModuleControlRequestFromModule::ScopeSync {
13071 generation: 1,
13072 scopes: vec![head("s", 1)],
13073 },
13074 ModuleControlRequestFromModule::ScopeDescribe {
13075 owner: Principal::Direct,
13076 scope_ref: "s".to_string(),
13077 },
13078 ] {
13079 let reply = call(&handler, &ctx, &request).await;
13080 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
13081 }
13082 assert!(
13083 !handler
13084 .scopes
13085 .read()
13086 .unwrap()
13087 .describe(
13088 &Principal::Reserved {
13089 module_id: OWNER.to_string()
13090 },
13091 "s"
13092 )
13093 .owner_synced
13094 );
13095 }
13096
13097 #[tokio::test]
13100 async fn a_module_without_a_supervised_launch_cannot_sync() {
13101 let handler = ControlHandler::new(Arc::new(Registry::default()));
13102 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
13103 assert_eq!(
13104 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
13105 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13106 );
13107 }
13108
13109 #[tokio::test]
13110 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
13111 let supervisor = SupervisorHandle::new();
13112 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13113 let handler = ControlHandler::new(Arc::new(Registry::default()))
13114 .with_supervisor(supervisor.clone());
13115 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
13116 sync(&handler, &incumbent, 1, vec![head("s", 1)])
13117 .await
13118 .expect("the current launch syncs");
13119
13120 supervisor.open_swap(OWNER, "n2".to_string());
13123 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
13124 assert_eq!(
13125 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
13126 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13127 );
13128 sync(&handler, &incumbent, 2, vec![head("s", 1)])
13129 .await
13130 .expect("the serving owner syncs during the swap");
13131
13132 supervisor.close_swap(OWNER);
13135 assert_eq!(
13136 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
13137 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13138 );
13139 sync(&handler, &incumbent, 3, vec![head("s", 1)])
13140 .await
13141 .expect("the serving owner syncs after the rollback");
13142 handler.cleanup_connection(candidate.connection_id).unwrap();
13143
13144 supervisor.open_swap(OWNER, "n3".to_string());
13149 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
13150 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
13151 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
13152 .await
13153 .expect("the promoted launch takes authority");
13154 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
13155 panic!("unexpected reply {reply:?}");
13156 };
13157 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
13158 assert_eq!(
13159 sync(&handler, &incumbent, 4, Vec::new()).await,
13160 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13161 );
13162 }
13163
13164 #[tokio::test]
13167 async fn closing_the_authority_connection_frees_sync_authority() {
13168 let supervisor = SupervisorHandle::new();
13169 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13170 let handler = ControlHandler::new(Arc::new(Registry::default()))
13171 .with_supervisor(supervisor.clone());
13172 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
13173 sync(&handler, &first, 10, vec![head("s", 1)])
13174 .await
13175 .unwrap();
13176 handler.cleanup_connection(first.connection_id).unwrap();
13177
13178 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
13179 sync(&handler, &second, 1, vec![head("s", 1)])
13180 .await
13181 .expect("the next connection takes the released authority");
13182 }
13183
13184 #[tokio::test]
13185 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
13186 let supervisor = SupervisorHandle::new();
13187 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13188 let handler =
13189 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
13190 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
13191 sync(&handler, &first, 10, vec![head("s", 1)])
13192 .await
13193 .unwrap();
13194 handler
13195 .handle_control_frame(
13196 &first,
13197 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
13198 )
13199 .await
13200 .unwrap();
13201 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
13202 sync(&handler, &second, 1, vec![head("s", 1)])
13203 .await
13204 .expect("GOODBYE releases authority even if the old socket remains open");
13205 }
13206
13207 #[tokio::test]
13208 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
13209 let registry = Arc::new(Registry::default());
13210 let supervisor_handle = SupervisorHandle::new();
13211 let supervisor =
13212 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
13213 .with_handle(supervisor_handle.clone())
13214 .with_daemon_incarnation("incarnation-7".to_string());
13215 supervisor
13218 .supervise_configured(
13219 ModuleSpec {
13220 module_id: OWNER.to_string(),
13221 program: PathBuf::from("/nonexistent/prefrontal-core"),
13222 args: Vec::new(),
13223 env: Vec::new(),
13224 reserved: false,
13225 reserved_prefixes: Vec::new(),
13226 protocol: ModuleProtocol::Subc,
13227 overlap: Default::default(),
13228 },
13229 false,
13230 )
13231 .unwrap();
13232 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
13233 let handler =
13234 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13235 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
13236
13237 let ModuleControlResponseToModule::ScopeDescribe {
13239 status,
13240 daemon_incarnation,
13241 owner_synced,
13242 owner_configured,
13243 scope,
13244 ..
13245 } = describe(&handler, &reader, OWNER, "s").await
13246 else {
13247 panic!("not a describe reply");
13248 };
13249 assert_eq!(status, ScopeStatus::NotLive);
13250 assert_eq!(daemon_incarnation, "incarnation-7");
13251 assert!(!owner_synced);
13252 assert!(owner_configured);
13253 assert!(scope.is_none());
13254
13255 let ModuleControlResponseToModule::ScopeDescribe {
13258 status,
13259 owner_configured,
13260 ..
13261 } = describe(&handler, &reader, "ghost", "s").await
13262 else {
13263 panic!("not a describe reply");
13264 };
13265 assert_eq!(status, ScopeStatus::NotLive);
13266 assert!(!owner_configured);
13267
13268 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
13270 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
13271 let ModuleControlResponseToModule::ScopeDescribe {
13272 status,
13273 scope_epoch,
13274 owner_synced,
13275 scope,
13276 ..
13277 } = describe(&handler, &reader, OWNER, "s").await
13278 else {
13279 panic!("not a describe reply");
13280 };
13281 assert_eq!(status, ScopeStatus::Live);
13282 assert_eq!(scope_epoch, Some(4));
13283 assert!(owner_synced);
13284 let stamp = scope.expect("a live scope carries its stamp");
13285 assert!(
13286 stamp.owner_authorized,
13287 "prefrontal-core is the default authority"
13288 );
13289 assert_eq!(stamp.kind, ScopeKind::Head);
13290 }
13291
13292 #[tokio::test]
13293 async fn scope_authority_owners_decides_owner_authorized() {
13294 let supervisor = SupervisorHandle::new();
13295 supervisor.set_spawn_nonce("broca", "b1".to_string());
13296 let handler = ControlHandler::new(Arc::new(Registry::default()))
13297 .with_supervisor(supervisor)
13298 .with_scope_authority_owners(vec!["broca".to_string()]);
13299 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
13300 let mut gated = head("s", 1);
13301 gated.attributes.agent_id = Some("agent".to_string());
13302 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
13303 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
13304 describe(&handler, &broca, "broca", "s").await
13305 else {
13306 panic!("not a describe reply");
13307 };
13308 assert!(scope.unwrap().owner_authorized);
13309 }
13310
13311 #[tokio::test]
13315 async fn scope_ops_and_the_scopes_capability_are_advertised() {
13316 let handler = ControlHandler::new(Arc::new(Registry::default()));
13317 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
13318 let ack = hello_via_sink(
13319 &handler,
13320 &ctx,
13321 &mut rx,
13322 hello_frame("m", PROTOCOL_VERSION, 1),
13323 )
13324 .await;
13325 let ack = parse_ack(&ack);
13326 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
13327 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
13328 }
13329 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
13330
13331 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
13332 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13333 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13334 let reply = handler
13335 .handle_control_frame(&client, frame)
13336 .await
13337 .unwrap()
13338 .pop()
13339 .unwrap();
13340 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13341 serde_json::from_slice(&reply.body).unwrap()
13342 else {
13343 panic!("not a server.describe reply");
13344 };
13345 assert!(
13346 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
13347 "{capabilities:?}"
13348 );
13349 }
13350
13351 const PLEXUS: &str = "plexus";
13354 const OTHER: &str = "other";
13355 const AFT: &str = "aft";
13356 const BROCA: &str = "broca";
13357 const MAGIC: &str = "magic-context";
13358
13359 fn nonce(module_id: &str) -> String {
13360 format!("nonce-{module_id}")
13361 }
13362
13363 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
13364 let (tx, rx) = mpsc::channel(64);
13365 (
13366 RouteCtx {
13367 connection_id: ConnectionId::new(connection),
13368 egress: FrameSink::new(tx),
13369 },
13370 rx,
13371 )
13372 }
13373
13374 struct Rig {
13378 handler: ControlHandler,
13379 forwarding: Arc<ForwardingTable>,
13380 owner: RouteCtx,
13381 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13382 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
13383 generation: u64,
13384 next_connection: u64,
13385 _supervisor: Supervisor,
13386 }
13387
13388 async fn rig() -> Rig {
13389 rig_with_flow_support(true).await
13390 }
13391
13392 async fn rig_with_flow_support(flow_support: bool) -> Rig {
13393 rig_with_scope_capabilities(flow_support, false).await
13394 }
13395
13396 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
13397 let registry = Arc::new(Registry::default());
13398 let forwarding = Arc::new(ForwardingTable::default());
13399 let supervisor_handle = SupervisorHandle::new();
13400 let supervisor =
13401 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
13402 .with_handle(supervisor_handle.clone());
13403 supervisor
13404 .supervise_configured(
13405 ModuleSpec {
13406 module_id: OWNER.to_string(),
13407 program: PathBuf::from("/nonexistent/prefrontal-core"),
13408 args: Vec::new(),
13409 env: Vec::new(),
13410 reserved: false,
13411 reserved_prefixes: Vec::new(),
13412 protocol: ModuleProtocol::Subc,
13413 overlap: Default::default(),
13414 },
13415 false,
13416 )
13417 .unwrap();
13418 for module_id in [OWNER, AFT, BROCA, MAGIC] {
13419 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
13420 }
13421 let handler =
13422 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13423 .with_supervisor(supervisor_handle);
13424 let (owner, mut owner_rx) = wide_ctx(1);
13425 hello_via_sink(
13426 &handler,
13427 &owner,
13428 &mut owner_rx,
13429 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
13430 )
13431 .await;
13432 let mut modules = BTreeMap::new();
13433 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
13434 let (ctx, mut rx) = wide_ctx(connection);
13435 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
13436 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
13437 body["manifest"]["provenance"] =
13441 serde_json::json!({"wire_crate_version": "0.29.0"});
13442 if flow_support || run_support {
13443 let mut provides = Vec::new();
13444 if flow_support {
13445 provides.push("flow-scopes/v1");
13446 }
13447 if run_support {
13448 provides.push("agent-run-scopes/v1");
13449 }
13450 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
13451 }
13452 let hello = Frame::build(
13453 FrameType::Hello,
13454 control_flags(),
13455 0,
13456 0,
13457 connection,
13458 serde_json::to_vec(&body).unwrap(),
13459 )
13460 .unwrap();
13461 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
13462 modules.insert(module_id.to_string(), (ctx, rx));
13463 }
13464 Rig {
13465 handler,
13466 forwarding,
13467 owner,
13468 _owner_rx: owner_rx,
13469 modules,
13470 generation: 0,
13471 next_connection: 100,
13472 _supervisor: supervisor,
13473 }
13474 }
13475
13476 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
13477 ScopeCarrier::new(Principal::Reserved {
13478 module_id: module_id.to_string(),
13479 })
13480 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
13481 }
13482
13483 fn session(scope_epoch: u64) -> ScopeRecord {
13486 let mut record = head("s", scope_epoch);
13487 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
13488 record.attributes.agent_id = Some("agent-1".to_string());
13489 record.attributes.delegates = true;
13490 record
13491 }
13492
13493 impl Rig {
13494 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
13495 self.generation += 1;
13496 sync(&self.handler, &self.owner, self.generation, scopes)
13497 .await
13498 .expect("the owner's sync is accepted");
13499 }
13500
13501 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
13502 ScopeSelector {
13503 owner: Principal::Reserved {
13504 module_id: OWNER.to_string(),
13505 },
13506 scope_ref: scope_ref.to_string(),
13507 scope_epoch,
13508 }
13509 }
13510
13511 fn open_frame(
13512 &mut self,
13513 opener: Option<&str>,
13514 target: &str,
13515 scope: Option<ScopeSelector>,
13516 ) -> (
13517 RouteCtx,
13518 mpsc::Receiver<crate::router::OutboundFrame>,
13519 Frame,
13520 ) {
13521 self.next_connection += 1;
13522 let (ctx, rx) = wide_ctx(self.next_connection);
13523 let root = unique_project_root("scoped-open");
13524 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
13525 target: RouteTarget::ToolProvider {
13526 module_id: target.to_string(),
13527 },
13528 identity: BindIdentity::new(
13529 root.path().to_path_buf(),
13530 "unit".to_string(),
13531 "session".to_string(),
13532 ),
13533 consumer_identity: opener.map(|module_id| ConsumerIdentity {
13534 module_id: module_id.to_string(),
13535 launch_nonce: nonce(module_id),
13536 }),
13537 consumer_capabilities: None,
13538 role_versions: None,
13539 admission_facts: None,
13540 scope,
13541 })
13542 .unwrap();
13543 let frame = Frame::build(
13544 FrameType::Request,
13545 control_flags(),
13546 0,
13547 0,
13548 self.next_connection,
13549 body,
13550 )
13551 .unwrap();
13552 (ctx, rx, frame)
13553 }
13554
13555 async fn refused(
13557 &mut self,
13558 opener: Option<&str>,
13559 target: &str,
13560 scope: Option<ScopeSelector>,
13561 ) -> String {
13562 self.refusal_body(opener, target, scope).await["code"]
13563 .as_str()
13564 .unwrap()
13565 .to_string()
13566 }
13567
13568 async fn refusal_body(
13569 &mut self,
13570 opener: Option<&str>,
13571 target: &str,
13572 scope: Option<ScopeSelector>,
13573 ) -> Value {
13574 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
13575 let replies = tokio::time::timeout(
13576 Duration::from_secs(2),
13577 self.handler.handle_control_frame(&ctx, frame),
13578 )
13579 .await
13580 .expect("the open must be refused before waiting for a bind ack")
13581 .unwrap();
13582 assert_eq!(replies.len(), 1, "{replies:?}");
13583 assert_eq!(replies[0].header.ty, FrameType::Error);
13584 let (_, module_rx) = self.modules.get_mut(target).unwrap();
13585 assert!(
13586 module_rx.try_recv().is_err(),
13587 "a refused open relays nothing"
13588 );
13589 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
13590 parse_error(&replies[0])
13591 }
13592
13593 async fn relayed(
13595 &mut self,
13596 opener: Option<&str>,
13597 target: &str,
13598 scope: Option<ScopeSelector>,
13599 ) -> Relayed {
13600 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
13601 let handler = self.handler.clone();
13602 let task_ctx = ctx.clone();
13603 let task = tokio::spawn(async move {
13604 handler
13605 .handle_control_frame(&task_ctx, frame)
13606 .await
13607 .unwrap()
13608 });
13609 let (_, module_rx) = self.modules.get_mut(target).unwrap();
13610 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
13611 .await
13612 .expect("the target receives the relayed route.bind")
13613 .unwrap()
13614 .frame;
13615 Relayed {
13616 target: target.to_string(),
13617 client: ctx,
13618 client_rx: rx,
13619 task,
13620 bind,
13621 }
13622 }
13623
13624 async fn ack(&self, relayed: &Relayed) {
13625 let (module, _) = &self.modules[&relayed.target];
13626 self.handler
13627 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
13628 .await
13629 .unwrap();
13630 }
13631
13632 async fn bound(
13634 &mut self,
13635 opener: Option<&str>,
13636 target: &str,
13637 scope: Option<ScopeSelector>,
13638 ) -> Bound {
13639 let relayed = self.relayed(opener, target, scope).await;
13640 self.ack(&relayed).await;
13641 let Relayed {
13642 target,
13643 client,
13644 mut client_rx,
13645 task,
13646 bind,
13647 } = relayed;
13648 assert!(
13649 task.await.unwrap().is_empty(),
13650 "the open is answered by commit"
13651 );
13652 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
13653 Bound {
13654 target,
13655 client,
13656 client_rx,
13657 channel,
13658 epoch,
13659 bind,
13660 }
13661 }
13662
13663 fn live(&self, route: &Bound) -> bool {
13664 matches!(
13665 self.forwarding
13666 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
13667 .unwrap(),
13668 DataRoute::Client(DataRouteState::Bound(_))
13669 )
13670 }
13671 }
13672
13673 struct Relayed {
13674 target: String,
13675 client: RouteCtx,
13676 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13677 task: tokio::task::JoinHandle<Vec<Frame>>,
13678 bind: Frame,
13679 }
13680
13681 struct Bound {
13682 target: String,
13683 client: RouteCtx,
13684 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13685 channel: u16,
13686 epoch: u32,
13687 bind: Frame,
13688 }
13689
13690 impl Bound {
13691 fn closed_reason(&mut self) -> RouteCloseReason {
13694 let mut reason = None;
13695 let mut goodbye = false;
13696 while let Ok(outbound) = self.client_rx.try_recv() {
13697 let frame = outbound.frame;
13698 match frame.header.ty {
13699 FrameType::Goodbye => {
13700 assert_eq!(
13701 (frame.header.channel, frame.header.epoch),
13702 (self.channel, self.epoch)
13703 );
13704 goodbye = true;
13705 }
13706 FrameType::Push => {
13707 let ClientControlPush::RouteClosed {
13708 reason: r,
13709 module_id,
13710 ..
13711 } = serde_json::from_slice(&frame.body).unwrap()
13712 else {
13713 panic!("unexpected push");
13714 };
13715 assert_eq!(module_id, self.target);
13716 reason = Some(r);
13717 }
13718 other => panic!("unexpected frame {other:?}"),
13719 }
13720 }
13721 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
13722 reason.expect("the client is told why the route closed")
13723 }
13724
13725 fn untouched(&mut self) -> bool {
13726 self.client_rx.try_recv().is_err()
13727 }
13728
13729 fn stamp(&self) -> Option<ScopeStamp> {
13730 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
13731 ModuleControlRequest::RouteBind { scope, .. } => scope,
13732 other => panic!("expected a route.bind, got {other:?}"),
13733 }
13734 }
13735 }
13736
13737 #[tokio::test]
13738 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
13739 ) {
13740 let mut rig = rig().await;
13741 let mut record = session(1);
13742 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
13743 record.child_owners = vec![Principal::Reserved {
13744 module_id: MAGIC.to_string(),
13745 }];
13746 rig.sync(vec![record]).await;
13747 let scope = || Some(rig_selector("s", Some(1)));
13748
13749 rig.bound(Some(OWNER), PLEXUS, scope()).await;
13752 rig.bound(Some(AFT), OTHER, scope()).await;
13753 rig.bound(Some(BROCA), PLEXUS, scope()).await;
13754
13755 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
13759 assert_eq!(
13760 rig.refused(opener, target, scope()).await,
13761 error_codes::SCOPE_NOT_CARRIER,
13762 "{opener:?} -> {target}"
13763 );
13764 }
13765 }
13766
13767 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
13768 ScopeSelector {
13769 owner: Principal::Reserved {
13770 module_id: OWNER.to_string(),
13771 },
13772 scope_ref: scope_ref.to_string(),
13773 scope_epoch,
13774 }
13775 }
13776
13777 #[tokio::test]
13778 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
13779 let mut rig = rig().await;
13780 rig.sync(vec![session(1)]).await;
13781 for opener in [OWNER, AFT] {
13782 assert_eq!(
13783 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
13784 .await,
13785 error_codes::SCOPE_EPOCH_REQUIRED,
13786 "{opener}"
13787 );
13788 }
13789 }
13790
13791 #[tokio::test]
13792 async fn admission_separates_not_synced_not_live_and_ended() {
13793 let mut rig = rig().await;
13794 let code = rig
13796 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13797 .await;
13798 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
13799 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
13800
13801 let ghost = ScopeSelector {
13803 owner: Principal::Reserved {
13804 module_id: "ghost".to_string(),
13805 },
13806 scope_ref: "s".to_string(),
13807 scope_epoch: Some(1),
13808 };
13809 assert_eq!(
13810 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
13811 error_codes::SCOPE_NOT_LIVE
13812 );
13813
13814 rig.sync(vec![session(2)]).await;
13815 assert_eq!(
13816 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
13817 .await,
13818 error_codes::SCOPE_NOT_LIVE
13819 );
13820 for epoch in [1, 3] {
13821 assert_eq!(
13822 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
13823 .await,
13824 error_codes::SCOPE_ENDED,
13825 "epoch {epoch}"
13826 );
13827 }
13828 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
13830 .await;
13831 }
13832
13833 #[tokio::test]
13834 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
13835 let mut rig = rig().await;
13836 rig.sync(vec![session(1)]).await;
13837 let route = rig
13838 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13839 .await;
13840 let stamp = route.stamp().expect("a scoped bind carries the stamp");
13841 assert_eq!(stamp.scope_ref, "s");
13842 assert_eq!(stamp.scope_epoch, 1);
13843 assert_eq!(stamp.kind, ScopeKind::Head);
13844 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
13845 assert!(stamp.attributes.delegates);
13846 assert!(stamp.owner_authorized);
13847 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13848 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
13849
13850 let (broca, mut broca_rx) = wide_ctx(50);
13853 hello_via_sink(
13854 &rig.handler,
13855 &broca,
13856 &mut broca_rx,
13857 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
13858 )
13859 .await;
13860 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
13861 .await
13862 .unwrap();
13863 let own = ScopeSelector {
13864 owner: Principal::Reserved {
13865 module_id: BROCA.to_string(),
13866 },
13867 scope_ref: "b".to_string(),
13868 scope_epoch: Some(1),
13869 };
13870 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
13871 assert!(!route.stamp().unwrap().owner_authorized);
13872 }
13873
13874 #[tokio::test]
13875 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
13876 let mut rig = rig().await;
13877 let mut record = head("s", 1);
13878 record.carriers = vec![carrier(AFT, None)];
13879 let flow_id = "Flow:run-7/step_2!~";
13880 record.attributes.flow_id = Some(flow_id.to_string());
13881 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
13882 .await
13883 .unwrap();
13884 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
13885 panic!("not a sync reply");
13886 };
13887 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
13888 let route = rig
13889 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13890 .await;
13891 assert!(rig.live(&route), "the stamped bind committed");
13892 let stamp = route.stamp().expect("a flow scope carries a stamp");
13893 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
13894 assert_eq!(stamp.attributes.agent_id, None);
13895 assert!(!stamp.attributes.delegates);
13896 assert!(stamp.owner_authorized);
13897 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13898 assert_eq!(unscoped.stamp(), None);
13899 }
13900
13901 #[tokio::test]
13902 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
13903 let mut rig = rig_with_flow_support(false).await;
13904 let mut record = session(1);
13905 record.attributes.flow_id = Some("flow:7".to_string());
13906 rig.sync(vec![record]).await;
13907 for opener in [OWNER, AFT] {
13908 let body = rig
13909 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13910 .await;
13911 assert_eq!(body["code"], "target_flow_unsupported");
13912 let message = body["message"].as_str().unwrap();
13913 for required in [PLEXUS, "flow-scopes/v1"] {
13914 assert!(message.contains(required), "{message}");
13915 }
13916 }
13917 }
13918
13919 #[tokio::test]
13920 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
13921 let mut rig = rig_with_flow_support(true).await;
13922 let mut record = session(1);
13923 record.attributes.flow_id = Some("flow:7".to_string());
13924 rig.sync(vec![record]).await;
13925 for opener in [OWNER, AFT] {
13926 let route = rig
13927 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
13928 .await;
13929 assert!(rig.live(&route));
13930 assert_eq!(
13931 route.stamp().unwrap().attributes.flow_id.as_deref(),
13932 Some("flow:7")
13933 );
13934 }
13935 }
13936
13937 #[tokio::test]
13938 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
13939 use std::future::Future;
13940
13941 let mut rig = rig_with_flow_support(true).await;
13942 let mut record = session(1);
13943 record.attributes.flow_id = Some("flow:7".to_string());
13944 rig.sync(vec![record]).await;
13945 let (client, mut client_rx, frame) =
13946 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
13947 for _ in 0..64 {
13950 client.egress.try_send(route_bind_ack(1)).unwrap();
13951 }
13952 let handler = rig.handler.clone();
13953 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
13954 std::future::poll_fn(|cx| {
13955 assert!(open.as_mut().poll(cx).is_pending());
13956 std::task::Poll::Ready(())
13957 })
13958 .await;
13959 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13960
13961 let old_connection = rig.modules[PLEXUS].0.connection_id;
13962 rig.handler.cleanup_connection(old_connection).unwrap();
13963 let (replacement, mut replacement_rx) = wide_ctx(200);
13964 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
13965 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
13966 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
13967 let hello = Frame::build(
13968 FrameType::Hello,
13969 control_flags(),
13970 0,
13971 0,
13972 200,
13973 serde_json::to_vec(&body).unwrap(),
13974 )
13975 .unwrap();
13976 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
13977
13978 client_rx.try_recv().unwrap();
13979 let replies = tokio::time::timeout(Duration::from_secs(2), open)
13980 .await
13981 .expect("the replacement is refused without waiting for a bind ack")
13982 .unwrap();
13983 assert_eq!(replies.len(), 1);
13984 let body = parse_error(&replies[0]);
13985 assert_eq!(body["code"], "target_flow_unsupported");
13986 for required in [PLEXUS, "flow-scopes/v1"] {
13987 assert!(body["message"].as_str().unwrap().contains(required));
13988 }
13989 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
13990 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
13991 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
13992 assert!(rig
13993 .handler
13994 .registry
13995 .get_module_by_connection(replacement.connection_id)
13996 .unwrap()
13997 .is_some());
13998 }
13999
14000 #[tokio::test]
14001 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
14002 {
14003 let mut rig = rig_with_flow_support(false).await;
14004 rig.sync(vec![session(1)]).await;
14005 let route = rig
14006 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14007 .await;
14008 assert!(rig.live(&route));
14009 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
14010 let mut record = session(1);
14011 record.attributes.flow_id = Some("flow:7".to_string());
14012 rig.sync(vec![record]).await;
14013 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
14014 assert!(rig.live(&unscoped));
14015 assert_eq!(unscoped.stamp(), None);
14016 }
14017
14018 #[tokio::test]
14019 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
14020 let mut rig = rig().await;
14021 let mut record = session(1);
14022 record.attributes.flow_id = Some("flow:7".to_string());
14023 rig.sync(vec![record.clone()]).await;
14024 let before = rig.forwarding.published_scope_tag(OWNER, "s");
14025 let mut owner_route = rig
14026 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
14027 .await;
14028 let mut carrier_route = rig
14029 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
14030 .await;
14031 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14032 rig.sync(vec![record.clone()]).await;
14033 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
14034 assert!(rig.live(&owner_route) && owner_route.untouched());
14035 assert!(rig.live(&carrier_route) && carrier_route.untouched());
14036
14037 record.attributes.flow_id = Some("flow:8".to_string());
14038 rig.sync(vec![record]).await;
14039 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
14040 let before = before.unwrap();
14041 assert_eq!(after.scope_epoch, before.scope_epoch);
14042 assert!(after.version > before.version);
14043 for route in [&mut owner_route, &mut carrier_route] {
14044 assert!(!rig.live(route));
14045 assert_eq!(
14046 route.closed_reason(),
14047 RouteCloseReason::ScopeDelegationChanged
14048 );
14049 }
14050 assert!(rig.live(&unscoped) && unscoped.untouched());
14051 for target in [PLEXUS, OTHER] {
14054 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
14055 let goodbye = module_rx
14056 .try_recv()
14057 .expect("the provider sees the drain")
14058 .frame;
14059 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14060 assert!(module_rx.try_recv().is_err());
14061 }
14062 let rebound = rig
14063 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14064 .await;
14065 assert_eq!(
14066 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
14067 Some("flow:8")
14068 );
14069 }
14070
14071 #[tokio::test]
14076 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
14077 let mut rig = rig().await;
14078 rig.sync(vec![session(1)]).await;
14079 let mut cotenant = rig
14080 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
14081 .await;
14082
14083 let mut changed = session(1);
14084 changed.child_owners.push(Principal::Reserved {
14085 module_id: MAGIC.to_string(),
14086 });
14087 let mut ended = None;
14088 for (code, next) in [
14089 (error_codes::SCOPE_CHANGED, vec![changed]),
14090 (error_codes::SCOPE_ENDED, Vec::new()),
14091 ] {
14092 let relayed = rig
14093 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14094 .await;
14095 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
14096 ended = Some(next.is_empty());
14097 rig.sync(next).await;
14098 rig.ack(&relayed).await;
14099 let replies = relayed.task.await.unwrap();
14100 assert_eq!(replies.len(), 1, "{replies:?}");
14101 assert_eq!(parse_error(&replies[0])["code"], code);
14102 assert_eq!(
14103 subc_protocol::error_codes::is_retryable_route_open(code),
14104 code == error_codes::SCOPE_CHANGED
14105 );
14106 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14107 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14109 let mut goodbyes = Vec::new();
14112 while let Ok(outbound) = plexus_rx.try_recv() {
14113 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
14114 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
14115 }
14116 assert!(
14117 goodbyes.contains(&(bind_channel, bind_epoch)),
14118 "{goodbyes:?}"
14119 );
14120 assert!(rig
14121 .handler
14122 .registry
14123 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
14124 .unwrap()
14125 .is_some());
14126 }
14127 assert_eq!(ended, Some(true));
14128 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
14131 }
14132
14133 #[tokio::test]
14136 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
14137 struct Case {
14138 name: &'static str,
14139 change: fn(&mut ScopeRecord),
14140 closed: [Option<RouteCloseReason>; 4],
14143 }
14144 use RouteCloseReason::*;
14145 let cases = [
14146 Case {
14147 name: "a carrier entry removed",
14148 change: |r| {
14149 r.carriers.retain(|c| {
14150 c.principal
14151 != Principal::Reserved {
14152 module_id: AFT.to_string(),
14153 }
14154 })
14155 },
14156 closed: [None, Some(ScopeCarrierRemoved), None, None],
14157 },
14158 Case {
14159 name: "a target removed from a carrier",
14160 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
14161 closed: [None, None, None, Some(ScopeCarrierRemoved)],
14162 },
14163 Case {
14164 name: "a bare carrier narrowed to targets",
14165 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
14166 closed: [None, Some(ScopeCarrierRemoved), None, None],
14167 },
14168 Case {
14169 name: "delegates turned off",
14170 change: |r| r.attributes.delegates = false,
14171 closed: [Some(ScopeDelegationChanged); 4],
14172 },
14173 Case {
14174 name: "agent_id changed",
14175 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
14176 closed: [Some(ScopeDelegationChanged); 4],
14177 },
14178 Case {
14179 name: "a carrier added, child owners changed, the record re-sent",
14180 change: |r| {
14181 r.carriers.push(carrier(MAGIC, None));
14182 r.child_owners.push(Principal::Reserved {
14183 module_id: MAGIC.to_string(),
14184 });
14185 },
14186 closed: [None; 4],
14187 },
14188 Case {
14189 name: "a target added",
14190 change: |r| {
14191 r.carriers[1]
14192 .targets
14193 .as_mut()
14194 .unwrap()
14195 .push("third".to_string())
14196 },
14197 closed: [None; 4],
14198 },
14199 Case {
14200 name: "delegates turned on",
14201 change: |r| r.attributes.delegates = true,
14202 closed: [None; 4],
14203 },
14204 ];
14205 for case in cases {
14206 let mut rig = rig().await;
14207 rig.sync(vec![session(1)]).await;
14208 let scope = || Some(rig_selector("s", Some(1)));
14209 let mut routes = [
14210 rig.bound(Some(OWNER), PLEXUS, scope()).await,
14211 rig.bound(Some(AFT), PLEXUS, scope()).await,
14212 rig.bound(Some(BROCA), PLEXUS, scope()).await,
14213 rig.bound(Some(BROCA), OTHER, scope()).await,
14214 ];
14215 let mut record = session(1);
14216 (case.change)(&mut record);
14217 rig.sync(vec![record]).await;
14218 for (index, expected) in case.closed.iter().enumerate() {
14219 let route = &mut routes[index];
14220 match expected {
14221 Some(reason) => {
14222 assert!(!rig.live(route), "{}: route {index} still live", case.name);
14223 assert_eq!(
14224 route.closed_reason(),
14225 *reason,
14226 "{}: route {index}",
14227 case.name
14228 );
14229 }
14230 None => {
14231 assert!(rig.live(route), "{}: route {index} closed", case.name);
14232 assert!(
14233 route.untouched(),
14234 "{}: route {index} was told something",
14235 case.name
14236 );
14237 }
14238 }
14239 }
14240 }
14241 }
14242
14243 #[tokio::test]
14244 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
14245 for next in [Vec::new(), vec![session(2)]] {
14247 let mut rig = rig().await;
14248 rig.sync(vec![session(1)]).await;
14249 let mut route = rig
14250 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14251 .await;
14252 rig.sync(next).await;
14253 assert!(!rig.live(&route));
14254 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
14255 }
14256
14257 let mut rig = rig().await;
14260 let mut child = session(1);
14261 child.scope_ref = "child".to_string();
14262 child.kind = ScopeKind::Worker;
14263 child.parent = Some(ScopeParent::new(
14264 Principal::Reserved {
14265 module_id: OWNER.to_string(),
14266 },
14267 "s".to_string(),
14268 1,
14269 ));
14270 rig.sync(vec![session(1), child.clone()]).await;
14271 let mut child_route = rig
14272 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
14273 .await;
14274 assert_eq!(
14275 child_route.stamp().unwrap().parent_state,
14276 Some(ParentState::Linked)
14277 );
14278 rig.sync(vec![child]).await;
14279 assert!(!rig.live(&child_route));
14280 assert_eq!(
14281 child_route.closed_reason(),
14282 RouteCloseReason::ScopeParentEnded
14283 );
14284 }
14285
14286 #[tokio::test]
14287 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
14288 ) {
14289 let mut rig = rig().await;
14290 rig.sync(vec![session(1)]).await;
14291 let mut route = rig
14292 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14293 .await;
14294 let before = rig.forwarding.published_scope_tag(OWNER, "s");
14295 rig.sync(vec![session(1)]).await;
14296 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
14297 assert!(rig.live(&route) && route.untouched());
14298
14299 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
14303 .forwarding
14304 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
14305 .unwrap()
14306 else {
14307 panic!("the route is bound");
14308 };
14309 binding.flow.acquire_tagged(9, false).await.unwrap();
14310 let mut widened = session(1);
14311 widened.carriers.push(carrier(MAGIC, None));
14312 rig.sync(vec![widened]).await;
14313 assert!(rig.live(&route) && route.untouched());
14314 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14315 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
14316 assert_eq!(binding.flow.in_flight(), 1);
14319 binding
14320 .flow
14321 .acquire_tagged(10, false)
14322 .await
14323 .expect("the flow is still open");
14324 }
14325
14326 #[tokio::test]
14329 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
14330 let mut rig = rig().await;
14331 rig.sync(vec![session(1)]).await;
14332 let mut on_incumbent = rig
14333 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14334 .await;
14335
14336 let (candidate, _candidate_rx) = wide_ctx(9);
14339 let registration = rig
14340 .handler
14341 .registry
14342 .register_candidate_with_control_ops(
14343 manifest(PLEXUS, PROTOCOL_VERSION),
14344 PROTOCOL_VERSION,
14345 candidate.connection_id,
14346 module_baseline_control_ops(),
14347 )
14348 .unwrap();
14349 rig.forwarding
14350 .register_candidate_module_connection(
14351 candidate.connection_id,
14352 PLEXUS.to_string(),
14353 PROTOCOL_VERSION,
14354 manifest_concurrency(®istration.manifest),
14355 candidate.egress.clone(),
14356 )
14357 .unwrap();
14358 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
14359 rig.handler
14360 .registry
14361 .promote_candidate(PLEXUS)
14362 .unwrap()
14363 .unwrap();
14364 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
14365
14366 rig.sync(Vec::new()).await;
14367 assert!(!rig.live(&on_incumbent));
14368 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
14369 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14370 let goodbye = incumbent_rx
14371 .try_recv()
14372 .expect("the superseded endpoint is told")
14373 .frame;
14374 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14375 }
14376 }
14377}
14378
14379#[cfg(test)]
14380mod concurrency_default_exposure_tests {
14381 use super::*;
14382
14383 fn hello_body(role_json: &str) -> Vec<u8> {
14384 format!(
14385 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
14386 )
14387 .into_bytes()
14388 }
14389
14390 fn manifest_from(body: &[u8]) -> ModuleManifest {
14391 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
14392 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
14393 .expect("manifest parses")
14394 }
14395
14396 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
14397
14398 #[test]
14399 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
14400 let body = hello_body(&format!(
14401 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
14402 ));
14403 let manifest = manifest_from(&body);
14404 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
14407 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
14408 }
14409
14410 #[test]
14411 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
14412 let body = hello_body(&format!(
14413 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
14414 ));
14415 let manifest = manifest_from(&body);
14416 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
14417 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
14418 }
14419
14420 #[test]
14421 fn non_management_roles_are_never_reported() {
14422 let body = hello_body(
14423 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
14424 );
14425 let manifest = manifest_from(&body);
14426 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
14427 }
14428}