1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 error_codes,
21 manifest::{
22 validate_hello_capability_grammar, validate_hello_event_declarations,
23 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
24 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
25 },
26 scope::{
27 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
28 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
29 SCOPE_SYNC_OP,
30 },
31 session::{
32 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
33 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
34 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
35 ROLE_VERSIONS_FIELD,
36 },
37 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
38 Priority, RouteTarget, PROTOCOL_VERSION,
39};
40use tokio::time::{timeout_at, Instant};
41use tracing::{debug, info, warn};
42
43use crate::{
44 capability_requirements::{
45 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
46 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
47 RuntimeModule,
48 },
49 daemon_config::RestartRequiredSection,
50 forwarding::{
51 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
52 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
53 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
54 },
55 observability::{
56 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
57 },
58 provenance::{
59 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
60 },
61 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
62 router::{RouteCtx, RouterError},
63 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
64 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
65 stderr_tail::{CaptureState, TailEntry},
66 supervise::{
67 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
68 SupervisorHandle, SwapHelloAdmission,
69 },
70 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
71};
72
73pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
79
80const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
81const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
82const CAP_PING_PONG: &str = "ping_pong_v1";
83const CAP_SESSION_ATTACH: &str = "session_attach_v1";
84const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
85
86const SUBC_CONTROL_OPS: &[&str] = &[
87 ops::SERVER_DESCRIBE,
88 ops::CATALOG_LIST,
89 ops::ROUTE_OPEN,
90 ops::ROUTE_POLL,
91 ops::ROUTE_CLOSING,
92 ops::ROUTE_CLOSED,
93 ops::SUPERVISOR_LIST,
94 ops::SUPERVISOR_RESTART,
95 ops::SUPERVISOR_SWAP,
96 ops::SUPERVISOR_RELOAD,
97 ops::SUPERVISOR_RESCAN,
98 ops::SUPERVISOR_RELEASE_RESERVED,
99 ops::SUPERVISOR_SET_ENABLED,
100 ops::SUPERVISOR_HEALTH_PROBE,
101 ops::SUPERVISOR_HEALTH,
102 ops::SUPERVISOR_STDERR_TAIL,
103 ops::SUPERVISOR_TERMINALS,
104 ops::SUPERVISOR_ROUTES,
105 ops::SUPERVISOR_PROVENANCE,
106 ops::SUPERVISOR_SPAWN_SNAPSHOT,
107 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
108];
109
110const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
111 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
112 "supervisor.live_roots",
113 SCOPE_SYNC_OP,
114 SCOPE_APPLY_OP,
115 SCOPE_DESCRIBE_OP,
116 "operator.confirm",
117];
118
119const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
123
124const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
125
126pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
134
135pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
150
151pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
169
170const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
171const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
172
173fn reload_verdict(
174 configured: &Path,
175 spawned_from: Option<&Path>,
176 image: subc_control::RunningImageAgreement,
177) -> PendingReloadVerdict {
178 let path = match spawned_from {
179 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
180 ReloadPathAgreement::Match
181 }
182 Some(spawned_from) => ReloadPathAgreement::Mismatch {
183 configured: configured.to_path_buf(),
184 spawned_from: spawned_from.to_path_buf(),
185 },
186 None => ReloadPathAgreement::Unavailable {
187 reason: if matches!(
188 image,
189 subc_control::RunningImageAgreement::Unavailable {
190 reason: subc_control::RunningImageUnavailableReason::NotRunning
191 }
192 ) {
193 ReloadPathUnavailableReason::NotRunning
194 } else {
195 ReloadPathUnavailableReason::SpawnedPathUnavailable
196 },
197 },
198 };
199 PendingReloadVerdict { path, image }
200}
201
202fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
203 if configured == spawned_from {
204 return true;
205 }
206 #[cfg(windows)]
207 {
208 match (
214 std::fs::canonicalize(configured),
215 std::fs::canonicalize(spawned_from),
216 ) {
217 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
218 _ => false,
219 }
220 }
221 #[cfg(not(windows))]
222 {
223 false
224 }
225}
226
227#[derive(Clone)]
228struct DaemonProvenanceFacts {
229 build: DaemonBuildProvenance,
230 pid: Option<u32>,
231 started_at_ms: Option<u64>,
232 start_clock: Option<crate::clock::StartClock>,
233 executable_path: Option<PathBuf>,
234 executable_identity: Option<SpawnedFileIdentity>,
235 process_start_time: Option<u64>,
236 probe: ExecutableIdentityProbe,
237}
238
239impl Default for DaemonProvenanceFacts {
240 fn default() -> Self {
241 Self {
242 build: DaemonBuildProvenance {
243 build_git_sha: None,
244 build_lock_digest: None,
245 },
246 pid: None,
247 started_at_ms: None,
248 start_clock: None,
249 executable_path: None,
250 executable_identity: None,
251 process_start_time: None,
252 probe: ExecutableIdentityProbe::default(),
253 }
254 }
255}
256
257#[derive(Debug, Clone)]
258struct SupervisorRescanContext {
259 supervisor: Supervisor,
260 config_path: PathBuf,
261 configured_port: Option<u16>,
262 storage_config: Option<crate::daemon_config::StorageConfig>,
263 admission_facts_carrier_module_id: Option<String>,
264 admission_facts_targets: Option<Vec<String>>,
265 scope_authority_owners: Vec<String>,
266}
267
268const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
289 "reloading",
290 "supervisor_not_live",
291 "registration_not_active",
292 "no_forwarding_connection",
293 "relay_send_failed",
294];
295
296#[derive(Clone)]
298pub struct ControlHandler {
299 registry: Arc<Registry>,
300 forwarding: Arc<ForwardingTable>,
301 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
302 supervisor: SupervisorHandle,
303 subc_capabilities: Arc<[String]>,
304 route_bind_relay_timeout: Duration,
308 route_bind_relay_timeouts: BTreeMap<String, Duration>,
312 route_bind_breakers: RouteBindBreakers,
315 route_bind_concurrency: RouteBindConcurrency,
318 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
323 route_bind_breaker_threshold: u32,
325 route_bind_breaker_cooldown: Duration,
327 health_probe_timeout: Duration,
328 storage_config: Option<crate::daemon_config::StorageConfig>,
331 machine_id: Option<crate::machine_id::MachineId>,
335 admission_facts_carrier_module_id: Option<String>,
336 admission_facts_targets: Option<Vec<String>>,
337 scopes: Arc<RwLock<ScopeTable>>,
341 scope_deadline_added: Arc<tokio::sync::Notify>,
345 scope_authority_owners: Vec<String>,
348 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
351 rescan: Option<SupervisorRescanContext>,
352 connected_clients: ConnectedClients,
353 counters: DaemonCounters,
354 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
355 daemon_provenance: DaemonProvenanceFacts,
356 #[cfg(test)]
357 control_dispatch_delay: Option<Duration>,
358 #[cfg(test)]
359 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
360 #[cfg(test)]
364 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
365}
366
367impl fmt::Debug for ControlHandler {
368 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
369 f.debug_struct("ControlHandler")
370 .field("registry", &self.registry)
371 .field("forwarding", &self.forwarding)
372 .field("process_liveness", &self.process_liveness.is_some())
373 .field("supervisor", &self.supervisor)
374 .field("subc_capabilities", &self.subc_capabilities)
375 .finish()
376 }
377}
378
379struct RouteOpenRequest {
380 target: RouteTarget,
381 identity: BindIdentity,
382 consumer_identity: Option<ConsumerIdentity>,
383 consumer_capabilities: Option<Vec<String>>,
384 role_versions: Option<BTreeMap<String, String>>,
385 admission_facts: Option<serde_json::Value>,
386 scope: Option<ScopeSelector>,
387}
388
389struct RouteBindReservationGuard {
390 forwarding: Arc<ForwardingTable>,
391 endpoint: ModuleEndpointId,
392 relay_corr: u64,
393 armed: bool,
394}
395
396struct ModuleControlRpcGuard {
397 forwarding: Arc<ForwardingTable>,
398 endpoint: ModuleEndpointId,
399 corr: u64,
400 armed: bool,
401}
402
403impl ModuleControlRpcGuard {
404 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
405 Self {
406 forwarding,
407 endpoint,
408 corr,
409 armed: true,
410 }
411 }
412
413 fn disarm(&mut self) {
414 self.armed = false;
415 }
416}
417
418impl Drop for ModuleControlRpcGuard {
419 fn drop(&mut self) {
420 if self.armed {
421 let _ = self
422 .forwarding
423 .cancel_module_control_rpc(self.endpoint, self.corr);
424 }
425 }
426}
427
428impl RouteBindReservationGuard {
429 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
430 Self {
431 forwarding,
432 endpoint,
433 relay_corr,
434 armed: true,
435 }
436 }
437
438 fn release_and_disarm(&mut self) {
439 if !self.armed {
440 return;
441 }
442 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
443 self.endpoint,
444 self.relay_corr,
445 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
446 ) {
447 send_goodbye_target_best_effort(
448 &self.forwarding.counters(),
449 &target,
450 "canceled route.bind",
451 );
452 }
453 self.armed = false;
454 }
455
456 fn disarm(&mut self) {
457 self.armed = false;
458 }
459}
460
461impl Drop for RouteBindReservationGuard {
462 fn drop(&mut self) {
463 self.release_and_disarm();
464 }
465}
466
467#[derive(Debug, Clone, Default)]
497pub(crate) struct RouteBindBreakers {
498 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
499}
500
501#[derive(Debug, Clone, Default)]
502pub(crate) struct RouteBindConcurrency {
503 modules: Arc<Mutex<HashMap<String, usize>>>,
504}
505
506struct RouteBindConcurrencyGuard {
507 concurrency: RouteBindConcurrency,
508 module_id: String,
509}
510
511impl RouteBindConcurrency {
512 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
515 let mut modules = self
516 .modules
517 .lock()
518 .expect("route.bind concurrency mutex poisoned");
519 let in_flight = modules.entry(module_id.to_string()).or_default();
520 if *in_flight >= limit {
521 return Err(*in_flight);
522 }
523 *in_flight += 1;
524 Ok(RouteBindConcurrencyGuard {
525 concurrency: self.clone(),
526 module_id: module_id.to_string(),
527 })
528 }
529}
530
531impl Drop for RouteBindConcurrencyGuard {
532 fn drop(&mut self) {
533 let mut modules = self
534 .concurrency
535 .modules
536 .lock()
537 .expect("route.bind concurrency mutex poisoned");
538 let remove = {
539 let in_flight = modules
540 .get_mut(&self.module_id)
541 .expect("admitted route.bind has a concurrency entry");
542 *in_flight -= 1;
543 *in_flight == 0
544 };
545 if remove {
546 modules.remove(&self.module_id);
547 }
548 }
549}
550
551#[derive(Debug, Default)]
552struct ModuleBreakerState {
553 consecutive_timeouts: u32,
555 cooldown_until: Option<Instant>,
558 probe_in_flight: Option<Arc<()>>,
563}
564
565enum RouteBindAdmission<'a> {
567 Admitted {
568 guard: RouteBindBreakerGuard<'a>,
569 probe: bool,
572 },
573 Refused {
574 consecutive_timeouts: u32,
575 retry_in: Duration,
579 probe_in_flight: bool,
580 },
581}
582
583struct RouteBindBreakerGuard<'a> {
590 breakers: RouteBindBreakers,
591 module_id: &'a str,
592 probe_token: Option<Arc<()>>,
593 settled: bool,
594}
595
596impl RouteBindBreakerGuard<'_> {
597 fn record_accepted(&mut self) -> bool {
601 self.settled = true;
602 self.breakers.record_accepted(self.module_id)
603 }
604
605 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
608 self.settled = true;
609 self.breakers.record_timeout(
610 self.module_id,
611 self.probe_token.as_ref(),
612 threshold,
613 cooldown,
614 )
615 }
616
617 fn record_inconclusive(&mut self) {
626 self.settled = true;
627 self.breakers
628 .record_inconclusive(self.module_id, self.probe_token.as_ref());
629 }
630}
631
632impl Drop for RouteBindBreakerGuard<'_> {
633 fn drop(&mut self) {
634 if !self.settled {
635 self.breakers
636 .record_inconclusive(self.module_id, self.probe_token.as_ref());
637 }
638 }
639}
640
641struct BreakerOpened {
645 consecutive_timeouts: u32,
646 reopened_after_probe: bool,
649}
650
651impl RouteBindBreakers {
652 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
653 self.modules
654 .lock()
655 .expect("route.bind breaker mutex poisoned")
656 }
657
658 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
661 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
662 probe: probe_token.is_some(),
663 guard: RouteBindBreakerGuard {
664 breakers: self.clone(),
665 module_id,
666 probe_token,
667 settled: false,
668 },
669 };
670
671 let mut modules = self.lock();
672 let Some(state) = modules.get_mut(module_id) else {
673 return admitted(None);
674 };
675 let Some(cooldown_until) = state.cooldown_until else {
676 return admitted(None);
677 };
678 if state.probe_in_flight.is_some() {
679 return RouteBindAdmission::Refused {
680 consecutive_timeouts: state.consecutive_timeouts,
681 retry_in: Duration::ZERO,
682 probe_in_flight: true,
683 };
684 }
685 let now = Instant::now();
686 if now < cooldown_until {
687 return RouteBindAdmission::Refused {
688 consecutive_timeouts: state.consecutive_timeouts,
689 retry_in: cooldown_until - now,
690 probe_in_flight: false,
691 };
692 }
693 let token = Arc::new(());
694 state.probe_in_flight = Some(Arc::clone(&token));
695 admitted(Some(token))
696 }
697
698 fn record_accepted(&self, module_id: &str) -> bool {
699 self.lock()
700 .remove(module_id)
701 .is_some_and(|state| state.cooldown_until.is_some())
702 }
703
704 fn record_timeout(
705 &self,
706 module_id: &str,
707 probe_token: Option<&Arc<()>>,
708 threshold: u32,
709 cooldown: Duration,
710 ) -> Option<BreakerOpened> {
711 let mut modules = self.lock();
712 let state = modules.entry(module_id.to_string()).or_default();
713 let was_open = state.cooldown_until.is_some();
714 let was_probe = Self::owns_probe(state, probe_token);
715 if was_probe {
716 state.probe_in_flight = None;
717 }
718 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
719 if state.consecutive_timeouts < threshold {
720 return None;
721 }
722 state.cooldown_until = Some(Instant::now() + cooldown);
723 Some(BreakerOpened {
724 consecutive_timeouts: state.consecutive_timeouts,
725 reopened_after_probe: was_open && was_probe,
726 })
727 }
728
729 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
730 state
734 .probe_in_flight
735 .as_ref()
736 .zip(token)
737 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
738 }
739
740 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
741 if let Some(state) = self.lock().get_mut(module_id) {
742 if Self::owns_probe(state, probe_token) {
743 state.probe_in_flight = None;
744 }
745 }
746 }
747
748 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
763 self.lock()
764 .remove(module_id)
765 .map(|state| state.consecutive_timeouts)
766 .filter(|discarded| *discarded > 0)
767 }
768
769 fn open_snapshot(&self) -> Option<serde_json::Value> {
776 let now = Instant::now();
777 let modules = self.lock();
778 let open = modules
779 .iter()
780 .filter_map(|(module_id, state)| {
781 let cooldown_until = state.cooldown_until?;
782 Some((
783 module_id.clone(),
784 serde_json::json!({
785 "consecutive_timeouts": state.consecutive_timeouts,
786 "cooldown_remaining_ms":
787 cooldown_until.saturating_duration_since(now).as_millis() as u64,
788 "probe_in_flight": state.probe_in_flight.is_some(),
789 }),
790 ))
791 })
792 .collect::<serde_json::Map<String, serde_json::Value>>();
793 (!open.is_empty()).then_some(serde_json::Value::Object(open))
794 }
795}
796
797impl ControlHandler {
798 pub fn new(registry: Arc<Registry>) -> Self {
799 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
800 }
801
802 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
803 let counters = forwarding.counters();
804 let route_bind_breakers = forwarding.route_bind_breakers();
808 let route_bind_concurrency = forwarding.route_bind_concurrency();
809 let route_outages = forwarding.route_outages();
810 Self {
811 registry,
812 forwarding,
813 process_liveness: None,
814 supervisor: SupervisorHandle::new(),
815 subc_capabilities: Arc::from([
816 CAP_MANIFEST_REGISTRATION.to_string(),
817 CAP_CHANNEL_LIFECYCLE.to_string(),
818 CAP_PING_PONG.to_string(),
819 CAP_SESSION_ATTACH.to_string(),
820 CAP_ADMISSION_FACTS_RELAY.to_string(),
821 CAP_SCOPES_V1.to_string(),
822 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
823 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
824 ]),
825 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
826 route_bind_relay_timeouts: BTreeMap::new(),
827 route_bind_breakers,
828 route_bind_concurrency,
829 route_outages,
830 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
831 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
832 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
833 storage_config: None,
834 machine_id: None,
835 admission_facts_carrier_module_id: None,
836 admission_facts_targets: None,
837 scopes: Arc::new(RwLock::new(ScopeTable::new(
838 crate::daemon_config::default_scope_authority_owners(),
839 ))),
840 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
841 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
842 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
843 rescan: None,
844 connected_clients: ConnectedClients::new(),
845 counters,
846 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
847 daemon_provenance: DaemonProvenanceFacts::default(),
848 #[cfg(test)]
849 control_dispatch_delay: None,
850 #[cfg(test)]
851 provenance_probe_override: None,
852 #[cfg(test)]
853 route_bind_relay_sent_hold: None,
854 }
855 }
856
857 pub fn with_storage_config(
860 mut self,
861 storage_config: Option<crate::daemon_config::StorageConfig>,
862 ) -> Self {
863 self.storage_config = storage_config;
864 self
865 }
866
867 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
870 self.machine_id = machine_id;
871 self
872 }
873
874 pub fn with_admission_facts_config(
878 mut self,
879 carrier_module_id: Option<String>,
880 targets: Option<Vec<String>>,
881 ) -> Self {
882 self.admission_facts_carrier_module_id = carrier_module_id;
883 self.admission_facts_targets = targets;
884 self
885 }
886
887 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
891 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
892 self.scope_authority_owners = owners;
893 self
894 }
895
896 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
899 self.route_bind_relay_timeout = timeout;
900 self
901 }
902
903 pub fn with_route_bind_relay_timeouts(
909 mut self,
910 timeouts: impl IntoIterator<Item = (String, Duration)>,
911 ) -> Self {
912 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
913 self
914 }
915
916 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
922 self.route_bind_relay_timeouts
923 .get(module_id)
924 .copied()
925 .unwrap_or(self.route_bind_relay_timeout)
926 }
927
928 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
936 self.route_bind_breaker_threshold = threshold.max(1);
937 self.route_bind_breaker_cooldown = cooldown;
938 self
939 }
940
941 #[cfg(test)]
942 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
943 self.health_probe_timeout = timeout;
944 self
945 }
946
947 #[cfg(test)]
948 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
949 self.control_dispatch_delay = Some(delay);
950 self
951 }
952
953 #[cfg(test)]
957 pub(crate) fn with_route_bind_relay_sent_hold(
958 mut self,
959 hold: Arc<tokio::sync::Barrier>,
960 ) -> Self {
961 self.route_bind_relay_sent_hold = Some(hold);
962 self
963 }
964
965 pub fn with_process_liveness(
966 mut self,
967 process_liveness: Arc<dyn ModuleProcessLiveness>,
968 ) -> Self {
969 self.process_liveness = Some(process_liveness);
970 self
971 }
972
973 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
974 self.supervisor = supervisor;
975 self
976 }
977
978 pub fn with_daemon_provenance(
979 mut self,
980 pid: u32,
981 started_at_ms: u64,
982 executable_path: Option<PathBuf>,
983 build_git_sha: Option<String>,
984 build_lock_digest: Option<String>,
985 ) -> Self {
986 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
987 let process_start_time = process_start_time(pid);
988 self.daemon_provenance = DaemonProvenanceFacts {
989 build: DaemonBuildProvenance {
990 build_git_sha,
991 build_lock_digest,
992 },
993 pid: Some(pid),
994 started_at_ms: Some(started_at_ms),
995 start_clock: None,
996 executable_path,
997 executable_identity,
998 process_start_time,
999 probe: ExecutableIdentityProbe::default(),
1000 };
1001 self
1002 }
1003
1004 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1005 self.daemon_provenance.start_clock = Some(clock);
1006 self
1007 }
1008
1009 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1011 self.scopes
1012 .write()
1013 .unwrap_or_else(|poisoned| poisoned.into_inner())
1014 .set_wall_clock(Arc::new(clock));
1015 self
1016 }
1017
1018 #[cfg(test)]
1019 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1020 self.provenance_probe_override = Some(result);
1021 self
1022 }
1023
1024 pub fn with_capability_config(
1028 self,
1029 modules: impl IntoIterator<Item = (String, bool)>,
1030 reserved_capabilities: BTreeMap<String, String>,
1031 ) -> Self {
1032 self.capability_evaluator
1033 .configure(modules, reserved_capabilities);
1034 self
1035 }
1036
1037 pub fn with_supervisor_rescan(
1038 mut self,
1039 supervisor: Supervisor,
1040 config_path: impl Into<PathBuf>,
1041 configured_port: Option<u16>,
1042 ) -> Self {
1043 self.rescan = Some(SupervisorRescanContext {
1044 supervisor,
1045 config_path: config_path.into(),
1046 configured_port,
1047 storage_config: self.storage_config.clone(),
1048 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1049 admission_facts_targets: self.admission_facts_targets.clone(),
1050 scope_authority_owners: self.scope_authority_owners.clone(),
1051 });
1052 self
1053 }
1054
1055 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1056 self.connected_clients = connected_clients;
1057 self
1058 }
1059
1060 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1061 Arc::clone(&self.forwarding)
1062 }
1063
1064 pub(crate) fn counters(&self) -> DaemonCounters {
1065 self.counters.clone()
1066 }
1067
1068 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1071 tokio::spawn(async move {
1072 loop {
1073 self.capability_evaluator
1074 .wait_for_change_or_deadline()
1075 .await;
1076 self.refresh_capability_requirements();
1077 }
1078 });
1079 }
1080
1081 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1087 tokio::spawn(async move {
1088 loop {
1089 let has_deadlines = self
1090 .scopes
1091 .read()
1092 .unwrap_or_else(|poisoned| poisoned.into_inner())
1093 .has_deadlines();
1094 if !has_deadlines {
1095 self.scope_deadline_added.notified().await;
1096 continue;
1097 }
1098 tokio::time::sleep(Duration::from_secs(1)).await;
1099 if let Err(error) = self.sweep_expired_scopes() {
1100 warn!(%error, "scope expiry sweep failed");
1101 }
1102 }
1103 });
1104 }
1105
1106 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1107 let mut table = self
1108 .scopes
1109 .write()
1110 .unwrap_or_else(|poisoned| poisoned.into_inner());
1111 let swept = table.sweep_expired();
1112 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1113 drop(table);
1114 self.close_scope_drained_routes(drained);
1115 Ok(())
1116 }
1117
1118 fn publish_scope_changes(
1119 &self,
1120 changes: &[crate::scopes::ScopeTagChange],
1121 expired: &[crate::scopes::ScopeExpired],
1122 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1123 if changes.is_empty() {
1124 return Ok(Vec::new());
1125 }
1126 let drained = self
1127 .forwarding
1128 .publish_scope_changes(changes)
1129 .map_err(RouterError::Forwarding)?;
1130 let mut counts = HashMap::new();
1131 for route in &drained {
1132 *counts
1133 .entry((
1134 route.scope.owner.as_str(),
1135 route.scope.scope_ref.as_str(),
1136 route.scope.tag.scope_epoch,
1137 ))
1138 .or_insert(0usize) += 1;
1139 }
1140 for scope in expired {
1141 let routes_closed = counts
1142 .get(&(
1143 scope.owner.as_str(),
1144 scope.scope_ref.as_str(),
1145 scope.scope_epoch,
1146 ))
1147 .copied()
1148 .unwrap_or(0);
1149 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1150 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1151 routes_closed, "scope expired");
1152 }
1153 Ok(drained)
1154 }
1155
1156 fn runtime_capability_snapshot(
1157 &self,
1158 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1159 let runtime = self
1160 .supervisor
1161 .list()
1162 .into_iter()
1163 .map(|module| {
1164 let status = module.status().map_err(|err| {
1165 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1166 })?;
1167 Ok(RuntimeModule {
1168 module_id: status.module_id,
1169 state: status.state,
1170 enabled: status.enabled,
1171 })
1172 })
1173 .collect::<Result<Vec<_>, RouterError>>()?;
1174 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1175 RouterError::backend(
1176 0,
1177 0,
1178 format!("failed to list capability registrations: {err}"),
1179 )
1180 })?;
1181 let registrations = registrations
1182 .into_iter()
1183 .map(|registration| RegisteredModule {
1184 module_id: registration.manifest.module_id,
1185 module_version: registration.manifest.module_version,
1186 capabilities: registration.manifest.capabilities,
1187 })
1188 .collect();
1189 Ok((runtime, registrations))
1190 }
1191
1192 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1199 let cached_registration = RegisteredModule {
1200 module_id: registration.manifest.module_id.clone(),
1201 module_version: registration.manifest.module_version.clone(),
1202 capabilities: registration.manifest.capabilities.clone(),
1203 };
1204 if self.capability_evaluator.record_hello(&cached_registration) {
1205 warn!(
1206 module_id = %cached_registration.module_id,
1207 "capability claims drifted from the cached manifest"
1208 );
1209 }
1210 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1211 self.enforce_capability_denies();
1212 }
1213 self.refresh_capability_requirements();
1214 }
1215
1216 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1220 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1221 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1222 self.supervisor.set_swap_promotion_observer(observer);
1223 }
1224
1225 pub fn refresh_capability_requirements(&self) {
1226 match self.runtime_capability_snapshot() {
1227 Ok((runtime, registrations)) => {
1228 log_requirement_events(
1229 self.capability_evaluator
1230 .evaluate_now(&runtime, ®istrations),
1231 );
1232 }
1233 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1234 }
1235 }
1236
1237 fn enforce_capability_denies(&self) {
1241 let (_, registrations) = match self.registry.list_modules() {
1242 Ok(snapshot) => snapshot,
1243 Err(err) => {
1244 warn!(error = %err, "failed to read registrations for capability deny census");
1245 return;
1246 }
1247 };
1248 let manifests = registrations
1249 .into_iter()
1250 .map(|registration| {
1251 (
1252 registration.manifest.module_id.clone(),
1253 registration.manifest,
1254 )
1255 })
1256 .collect::<BTreeMap<_, _>>();
1257 let census = match self.forwarding.route_census(None) {
1258 Ok(census) => census,
1259 Err(err) => {
1260 warn!(error = %err, "failed to read route census for capability deny enforcement");
1261 return;
1262 }
1263 };
1264
1265 for (target_module_id, routes) in census {
1266 let Some(target_manifest) = manifests.get(&target_module_id) else {
1267 continue;
1268 };
1269 let mut closed_routes = Vec::new();
1270 let mut module_goodbyes = Vec::new();
1271 for route in routes {
1272 let Principal::Reserved {
1273 module_id: opening_module_id,
1274 } = &route.principal
1275 else {
1276 continue;
1277 };
1278 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1279 continue;
1280 };
1281 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1282 continue;
1283 };
1284
1285 match self.forwarding.release_client_route(
1286 route.goodbye_target.connection_id,
1287 route.goodbye_target.channel,
1288 route.goodbye_target.epoch,
1289 ) {
1290 Ok(RouteRelease::Removed(module_goodbye)) => {
1291 warn!(
1292 opening_module_id,
1293 target_module_id,
1294 capability,
1295 "force-closing route because an attested capability deny edge now matches"
1296 );
1297 closed_routes.push(route);
1298 module_goodbyes.push(module_goodbye);
1299 }
1300 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1301 Err(err) => warn!(
1302 opening_module_id,
1303 target_module_id,
1304 capability,
1305 error = %err,
1306 "failed to force-close capability-denied route"
1307 ),
1308 }
1309 }
1310
1311 if closed_routes.is_empty() {
1312 continue;
1313 }
1314 send_route_control_pushes(
1315 &self.forwarding,
1316 closed_routes,
1317 ClientControlPush::RouteClosed {
1318 module_id: target_module_id,
1319 channels: Vec::new(),
1320 reason: RouteCloseReason::CapabilityDenied,
1321 drained: false,
1322 abandoned: 0,
1323 excluded_subscriptions: 0,
1324 terminal: Some(false),
1325 },
1326 );
1327 self.emit_route_goodbyes(module_goodbyes);
1328 }
1329 }
1330
1331 fn not_ready_reason(
1337 &self,
1338 registration: &crate::registry::ModuleRegistration,
1339 ) -> Option<NotReadyReason> {
1340 if !registration.ready {
1341 return Some(NotReadyReason {
1342 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1343 capability: None,
1344 });
1345 }
1346 self.first_unprovided_required_capability(registration)
1347 .map(|capability| NotReadyReason {
1348 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1349 capability: Some(capability),
1350 })
1351 }
1352
1353 fn first_unprovided_required_capability(
1371 &self,
1372 registration: &crate::registry::ModuleRegistration,
1373 ) -> Option<String> {
1374 let required = registration
1375 .manifest
1376 .capabilities
1377 .iter()
1378 .flat_map(|declarations| declarations.requires.iter())
1379 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1380 .map(|requirement| requirement.capability.as_str())
1381 .collect::<BTreeSet<_>>();
1382 if required.is_empty() {
1383 return None;
1384 }
1385 let module_id = registration.manifest.module_id.as_str();
1386 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1387 if required
1388 .iter()
1389 .any(|capability| verdict(capability).is_none())
1390 {
1391 self.refresh_capability_requirements();
1392 }
1393 required
1394 .into_iter()
1395 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1396 .map(str::to_string)
1397 }
1398
1399 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1400 self.capability_evaluator
1401 .statuses()
1402 .into_iter()
1403 .map(capability_requirement_status)
1404 .collect()
1405 }
1406
1407 fn deregister_connection(
1414 &self,
1415 connection_id: ConnectionId,
1416 reason: RegistrationEndReason,
1417 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1418 self.registry
1419 .deregister_connection_with_reason(connection_id, reason)
1420 }
1421
1422 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1423 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1424 return None;
1425 }
1426 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1427 parse_client_control_request(&frame.body)
1428 else {
1429 return None;
1430 };
1431 Some(target_module_id(&target).to_string())
1432 }
1433
1434 pub(crate) fn route_open_capacity_refusal(
1435 &self,
1436 ctx: &RouteCtx,
1437 frame: &Frame,
1438 target_module_id: &str,
1439 in_flight: usize,
1440 limit: usize,
1441 ) -> Result<Frame, RouterError> {
1442 self.route_open_admission_refusal_frame(
1443 ctx,
1444 frame,
1445 target_module_id,
1446 "open_admission_full",
1447 (in_flight, limit),
1448 format!(
1449 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1450 ),
1451 )
1452 }
1453
1454 fn route_open_target_capacity_refusal(
1455 &self,
1456 ctx: &RouteCtx,
1457 frame: &Frame,
1458 target_module_id: &str,
1459 in_flight: usize,
1460 ) -> Result<Frame, RouterError> {
1461 self.route_open_admission_refusal_frame(
1462 ctx,
1463 frame,
1464 target_module_id,
1465 "target_binds_full",
1466 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1467 format!(
1468 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1469 ),
1470 )
1471 }
1472
1473 fn route_open_admission_refusal_frame(
1481 &self,
1482 ctx: &RouteCtx,
1483 frame: &Frame,
1484 target_module_id: &str,
1485 reason: &'static str,
1486 (in_flight, limit): (usize, usize),
1487 message: impl Into<String>,
1488 ) -> Result<Frame, RouterError> {
1489 let code = error_codes::TARGET_UNAVAILABLE;
1490 self.counters.increment_route_open_refused(code);
1491 info!(
1492 target: "control",
1493 code,
1494 reason,
1495 module_id = ?target_module_id,
1496 connection_id = ctx.connection_id.get(),
1497 in_flight,
1498 limit,
1499 "route.open refused"
1500 );
1501 control_error_frame(frame, code, message.into())
1502 }
1503
1504 #[cfg(test)]
1510 pub fn handle_control(
1511 &self,
1512 connection_id: ConnectionId,
1513 frame: Frame,
1514 ) -> Result<Vec<Frame>, RouterError> {
1515 match frame.header.ty {
1516 FrameType::Ping => Ok(vec![pong(&frame)?]),
1517 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1518 FrameType::Goodbye => self.handle_goodbye(connection_id),
1519 ty => Ok(vec![control_error_frame(
1520 &frame,
1521 "unsupported_control_frame",
1522 format!("unsupported channel-0 frame {ty:?}"),
1523 )?]),
1524 }
1525 }
1526
1527 pub async fn handle_control_frame(
1528 &self,
1529 ctx: &RouteCtx,
1530 frame: Frame,
1531 ) -> Result<Vec<Frame>, RouterError> {
1532 self.handle_control_frame_timed(ctx, frame, None).await
1533 }
1534
1535 pub(crate) async fn handle_control_frame_timed(
1536 &self,
1537 ctx: &RouteCtx,
1538 frame: Frame,
1539 dispatch_started_at: Option<StdInstant>,
1540 ) -> Result<Vec<Frame>, RouterError> {
1541 match frame.header.ty {
1542 FrameType::Ping => Ok(vec![pong(&frame)?]),
1543 FrameType::Hello => {
1544 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1545 }
1546 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1547 FrameType::Cancel => {
1548 if self
1551 .forwarding
1552 .operator_confirms()
1553 .cancel(ctx.connection_id, frame.header.corr)
1554 || self
1555 .supervisor
1556 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1557 {
1558 Ok(Vec::new())
1559 } else {
1560 Ok(vec![control_error_frame(
1561 &frame,
1562 "unknown_subscription",
1563 "no supervisor spawn subscription has this correlation id",
1564 )?])
1565 }
1566 }
1567 FrameType::Request => {
1568 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1571 if op
1572 .as_ref()
1573 .is_some_and(|probe| probe.op == "operator.confirm")
1574 {
1575 return self.handle_operator_confirm(ctx, frame);
1576 }
1577 if self
1578 .forwarding
1579 .module_endpoint_for_connection(ctx.connection_id)
1580 .map_err(RouterError::Forwarding)?
1581 .is_some()
1582 {
1583 if !is_known_module_request_op(&frame.body) {
1584 return Ok(vec![control_error_frame(
1585 &frame,
1586 "unsupported_control_frame",
1587 "module-originated channel-0 REQUEST is not supported",
1588 )?]);
1589 }
1590 let request = match parse_module_control_request_from_module(&frame.body) {
1591 Ok(request) => request,
1592 Err((err, ControlRequestBodyError::UnknownOp)) => {
1593 return Ok(vec![control_error_frame(
1594 &frame,
1595 "unsupported_control_frame",
1596 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1597 )?])
1598 }
1599 Err((err, ControlRequestBodyError::InvalidBody)) => {
1600 return Ok(vec![control_error_frame(
1601 &frame,
1602 "invalid_control_body",
1603 format!("malformed module control body: {err}"),
1604 )?])
1605 }
1606 };
1607 let op = module_control_request_op(&request);
1608 let corr = frame.header.corr;
1609 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1610 let result =
1611 self.handle_module_control_request(ctx.connection_id, frame, request);
1612 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1613 return result;
1614 }
1615
1616 if is_known_module_request_op(&frame.body) {
1617 return Ok(vec![control_error_frame(
1618 &frame,
1619 "not_registered",
1620 "catalog.update requires an active module registration owned by this connection",
1621 )?]);
1622 }
1623
1624 let request = match parse_client_control_request(&frame.body) {
1625 Ok(request) => request,
1626 Err((err, ControlRequestBodyError::UnknownOp)) => {
1627 return Ok(vec![control_error_frame(
1628 &frame,
1629 "unknown_control_op",
1630 format!("unknown client control op: {err}"),
1631 )?])
1632 }
1633 Err((err, ControlRequestBodyError::InvalidBody)) => {
1634 return Ok(vec![control_error_frame(
1635 &frame,
1636 "invalid_control_body",
1637 format!("malformed client control body: {err}"),
1638 )?])
1639 }
1640 };
1641 let op = client_control_request_op(&request);
1642 let corr = frame.header.corr;
1643 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1644 #[cfg(test)]
1645 if let Some(delay) = self.control_dispatch_delay {
1646 tokio::time::sleep(delay).await;
1647 }
1648 let result = self
1649 .handle_client_control_request(ctx, frame, request)
1650 .await;
1651 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1652 result
1653 }
1654 FrameType::Push => {
1655 let Some(endpoint) = self
1656 .forwarding
1657 .module_endpoint_for_connection(ctx.connection_id)
1658 .map_err(RouterError::Forwarding)?
1659 else {
1660 return Ok(vec![control_error_frame(
1661 &frame,
1662 "unsupported_control_frame",
1663 "client-originated channel-0 PUSH is not supported",
1664 )?]);
1665 };
1666 self.handle_status_update(endpoint, frame)
1667 }
1668 FrameType::Response | FrameType::Error
1669 if self
1670 .forwarding
1671 .module_endpoint_for_connection(ctx.connection_id)
1672 .map_err(RouterError::Forwarding)?
1673 .is_some() =>
1674 {
1675 self.handle_module_relay_response(ctx.connection_id, frame)
1676 }
1677 ty => Ok(vec![control_error_frame(
1678 &frame,
1679 "unsupported_control_frame",
1680 format!("unsupported channel-0 frame {ty:?}"),
1681 )?]),
1682 }
1683 }
1684
1685 pub fn cleanup_connection(
1686 &self,
1687 connection_id: ConnectionId,
1688 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1689 self.cleanup_connection_with_end_reason(
1690 connection_id,
1691 RegistrationEndReason::ConnectionClosed,
1692 )
1693 }
1694
1695 fn cleanup_connection_with_end_reason(
1696 &self,
1697 connection_id: ConnectionId,
1698 requested_reason: RegistrationEndReason,
1699 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1700 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1701 self.registry
1702 .get_module_by_connection(connection_id)?
1703 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1704 .and_then(|module| module.registration_end_reason().ok().flatten())
1705 .unwrap_or(requested_reason)
1706 } else {
1707 requested_reason
1708 };
1709 let crash_closed = self
1710 .registry
1711 .get_module_by_connection(connection_id)?
1712 .and_then(|registration| {
1713 self.forwarding
1714 .module_endpoint_for_connection(connection_id)
1715 .ok()
1716 .flatten()
1717 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1718 .map(|routes| (registration.manifest.module_id, routes))
1719 });
1720 let crash_closed = crash_closed.map(|(module_id, routes)| {
1721 let terminal = match self.supervisor.get(&module_id) {
1722 None => false,
1723 Some(module) => match module.will_recover_after_connection_loss() {
1724 Ok(will_recover) => !will_recover,
1725 Err(err) => {
1726 warn!(
1727 %module_id,
1728 error = %err,
1729 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1730 );
1731 false
1732 }
1733 },
1734 };
1735 let reason = match self.forwarding.is_daemon_draining() {
1739 Ok(true) => RouteCloseReason::Restart,
1740 Ok(false) => RouteCloseReason::Crash,
1741 Err(err) => {
1742 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1743 RouteCloseReason::Crash
1744 }
1745 };
1746 (module_id, routes, reason, terminal)
1747 });
1748 let registrations = self.deregister_connection(connection_id, end_reason);
1749 let cleanup = if crash_closed.is_some() {
1750 self.forwarding.cleanup_connection_counted(connection_id)
1751 } else {
1752 self.forwarding
1758 .cleanup_connection(connection_id)
1759 .map(|released| crate::forwarding::ConnectionCleanup {
1760 released,
1761 abandoned_relays: 0,
1762 })
1763 };
1764 if let Some((module_id, routes, reason, terminal)) = crash_closed {
1769 let abandoned = cleanup
1770 .as_ref()
1771 .map_or(0, |cleanup| cleanup.abandoned_relays);
1772 send_route_control_pushes(
1773 &self.forwarding,
1774 routes,
1775 ClientControlPush::RouteClosed {
1776 module_id,
1777 channels: Vec::new(),
1778 reason,
1779 drained: false,
1780 abandoned,
1781 excluded_subscriptions: 0,
1782 terminal: Some(terminal),
1783 },
1784 );
1785 }
1786 if let Ok(cleanup) = cleanup {
1787 self.emit_route_goodbyes(cleanup.released);
1788 }
1789 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1793 crate::supervise::notify_registration_release();
1794 self.capability_evaluator.wake_deadline_loop();
1795 self.refresh_capability_requirements();
1796 }
1797 self.supervisor.remove_spawn_subscribers(connection_id);
1798 self.hello_launch_nonces
1801 .lock()
1802 .unwrap_or_else(|poisoned| poisoned.into_inner())
1803 .forget(connection_id);
1804 self.scopes
1805 .write()
1806 .unwrap_or_else(|poisoned| poisoned.into_inner())
1807 .release_connection(connection_id);
1808 registrations
1809 }
1810
1811 pub(crate) fn handle_route_goodbye(
1812 &self,
1813 connection_id: ConnectionId,
1814 route_channel: u16,
1815 route_epoch: u32,
1816 ) -> Result<bool, RouterError> {
1817 debug!(
1818 connection_id = connection_id.get(),
1819 route_channel, route_epoch, "handling route GOODBYE"
1820 );
1821 let RouteRelease::Removed(released_route) = self
1822 .forwarding
1823 .release_client_route(connection_id, route_channel, route_epoch)
1824 .map_err(RouterError::Forwarding)?
1825 else {
1826 return Ok(false);
1827 };
1828 self.emit_route_goodbyes(vec![released_route]);
1829 Ok(true)
1830 }
1831
1832 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1833 for released in released_routes {
1834 let frame = match Frame::build_with_version(
1835 released.negotiated_ver,
1836 FrameType::Goodbye,
1837 control_flags(),
1838 released.channel,
1839 released.epoch,
1840 0,
1841 Vec::new(),
1842 ) {
1843 Ok(frame) => frame,
1844 Err(err) => {
1845 warn!(
1846 route_channel = released.channel,
1847 error = %err,
1848 "failed to build route GOODBYE frame"
1849 );
1850 continue;
1851 }
1852 };
1853 if !released.close_on_delivery_failure() {
1854 crate::forwarding::send_module_route_goodbye(
1855 &self.counters,
1856 &released.sink,
1857 frame,
1858 released.module_id.as_deref(),
1859 "client route released",
1860 );
1861 continue;
1862 }
1863 if let Err(err) = released.sink.try_send(frame) {
1864 warn!(
1865 target_connection_id = released.connection_id.get(),
1866 route_channel = released.channel,
1867 error = %err,
1868 "route GOODBYE was not delivered to client; closing target connection"
1869 );
1870 if self
1871 .forwarding
1872 .escalate_client_delivery_failure(
1873 released.connection_id,
1874 released.channel,
1875 released.epoch,
1876 CloseReason::new(
1877 "route_goodbye_delivery_failed",
1878 format!(
1879 "failed to enqueue route GOODBYE for channel {}: {err}",
1880 released.channel
1881 ),
1882 ),
1883 crate::forwarding::UndeliveredFrame {
1884 module_id: released.module_id.as_deref(),
1885 sink: &released.sink,
1886 },
1887 )
1888 .unwrap_or(false)
1889 {
1890 self.counters.increment_goodbye_relay_client_failed();
1891 }
1892 }
1893 }
1894 }
1895
1896 fn send_abandoned_route_bind_goodbye(
1908 &self,
1909 module_sink: &crate::FrameSink,
1910 negotiated_ver: u8,
1911 module_channel: u16,
1912 module_epoch: u32,
1913 ) {
1914 let frame = match Frame::build_with_version(
1915 negotiated_ver,
1916 FrameType::Goodbye,
1917 control_flags(),
1918 module_channel,
1919 module_epoch,
1920 0,
1921 Vec::new(),
1922 ) {
1923 Ok(frame) => frame,
1924 Err(err) => {
1925 warn!(
1926 route_channel = module_channel,
1927 error = %err,
1928 "failed to build GOODBYE for abandoned route.bind"
1929 );
1930 return;
1931 }
1932 };
1933 crate::forwarding::send_module_route_goodbye(
1934 &self.counters,
1935 module_sink,
1936 frame,
1937 None,
1938 "abandoned route.bind",
1939 );
1940 }
1941
1942 fn handle_hello(
1943 &self,
1944 connection_id: ConnectionId,
1945 sink: Option<crate::FrameSink>,
1946 frame: Frame,
1947 ) -> Result<Vec<Frame>, RouterError> {
1948 debug!(
1949 connection_id = connection_id.get(),
1950 corr = frame.header.corr,
1951 "handling HELLO"
1952 );
1953 if self
1957 .registry
1958 .get_module_by_connection(connection_id)
1959 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
1960 .is_some()
1961 {
1962 return Ok(vec![control_error_frame(
1963 &frame,
1964 "invalid_hello",
1965 "connection is already registered as a module",
1966 )?]);
1967 }
1968 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
1969 Ok(value) => value,
1970 Err(err) => {
1971 return Ok(vec![control_error_frame(
1972 &frame,
1973 "invalid_hello",
1974 format!("malformed HELLO body: {err}"),
1975 )?])
1976 }
1977 };
1978 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
1979 return Ok(vec![control_error_frame(
1980 &frame,
1981 "invalid_capability_grammar",
1982 err.to_string(),
1983 )?]);
1984 }
1985 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
1986 return Ok(vec![control_error_frame(
1987 &frame,
1988 "invalid_manifest",
1989 err.to_string(),
1990 )?]);
1991 }
1992 if let Err(err) = validate_hello_event_declarations(&hello_value) {
1993 return Ok(vec![control_error_frame(
1994 &frame,
1995 "invalid_event_declaration",
1996 err.to_string(),
1997 )?]);
1998 }
1999 if let Some(provenance) = hello_value
2000 .get("manifest")
2001 .and_then(|manifest| manifest.get("provenance"))
2002 {
2003 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2004 return Ok(vec![control_error_frame(
2005 &frame,
2006 "invalid_manifest",
2007 format!("malformed manifest provenance: {err}"),
2008 )?]);
2009 }
2010 }
2011 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2012 Ok(hello) => hello,
2013 Err(err) => {
2014 return Ok(vec![control_error_frame(
2015 &frame,
2016 "invalid_hello",
2017 format!("malformed HELLO body: {err}"),
2018 )?])
2019 }
2020 };
2021
2022 if hello.protocol_ver != hello.manifest.protocol_ver {
2023 return Ok(vec![control_error_frame(
2024 &frame,
2025 "invalid_manifest",
2026 format!(
2027 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2028 hello.protocol_ver, hello.manifest.protocol_ver
2029 ),
2030 )?]);
2031 }
2032
2033 if hello.manifest.module_id.trim().is_empty() {
2034 return Ok(vec![control_error_frame(
2035 &frame,
2036 "invalid_manifest",
2037 "manifest module_id must not be empty",
2038 )?]);
2039 }
2040
2041 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2042 Ok(negotiated_ver) => negotiated_ver,
2043 Err(message) => {
2044 return Ok(vec![control_error_frame(
2045 &frame,
2046 "version_unsupported",
2047 message,
2048 )?])
2049 }
2050 };
2051
2052 let swap_admission = self
2060 .supervisor
2061 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2062 if swap_admission == SwapHelloAdmission::Refused {
2063 warn!(
2064 module_id = %hello.manifest.module_id,
2065 connection_id = connection_id.get(),
2066 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2067 );
2068 return Ok(vec![control_error_frame(
2069 &frame,
2070 "swap_token_invalid",
2071 format!(
2072 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2073 hello.manifest.module_id
2074 ),
2075 )?]);
2076 }
2077 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2078
2079 if let Some(rejection) = (!swap_candidate)
2087 .then(|| {
2088 self.supervisor.reserved_hello_rejection(
2089 &hello.manifest.module_id,
2090 hello.launch_nonce.as_deref(),
2091 )
2092 })
2093 .flatten()
2094 {
2095 let message = match rejection {
2096 ReservedHelloRejection::Exact { module_id } => format!(
2097 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2098 ),
2099 ReservedHelloRejection::Prefix {
2100 prefix,
2101 owner_module_id,
2102 } => format!(
2103 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2104 hello.manifest.module_id
2105 ),
2106 };
2107 return Ok(vec![control_error_frame(
2108 &frame,
2109 "reserved_module",
2110 message,
2111 )?]);
2112 }
2113
2114 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2115 &hello.manifest.module_id,
2116 hello.manifest.capabilities.as_ref(),
2117 );
2118 if let Some(refusal) = reserved_capability_refusals.first() {
2119 let capability = refusal.capability.clone();
2120 let bound_module = refusal.claimants[0].clone();
2121 log_duplicate_claim_events(reserved_capability_refusals);
2122 return Ok(vec![control_error_frame(
2123 &frame,
2124 "reserved_capability",
2125 format!(
2126 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2127 capability, bound_module, hello.manifest.module_id
2128 ),
2129 )?]);
2130 }
2131
2132 if self
2135 .forwarding
2136 .connection_has_client_routes(connection_id)
2137 .map_err(RouterError::Forwarding)?
2138 {
2139 return Ok(vec![control_error_frame(
2140 &frame,
2141 "invalid_hello",
2142 "connection has open client routes and cannot also register as a module",
2143 )?]);
2144 }
2145
2146 self.hello_launch_nonces
2151 .lock()
2152 .unwrap_or_else(|poisoned| poisoned.into_inner())
2153 .record(connection_id, hello.launch_nonce.as_deref());
2154 let control_ops = effective_module_control_ops(hello.control_ops);
2155 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2158 if swap_candidate {
2159 return self.register_swap_candidate(
2160 connection_id,
2161 sink,
2162 &frame,
2163 hello.manifest,
2164 negotiated_ver,
2165 control_ops,
2166 hello_ack,
2167 );
2168 }
2169 let registration = match self.registry.register_with_control_ops(
2170 hello.manifest,
2171 negotiated_ver,
2172 connection_id,
2173 control_ops,
2174 ) {
2175 Ok(registration) => registration,
2176 Err(RegistryError::DuplicateModuleId { module_id }) => {
2177 return Ok(vec![control_error_frame(
2178 &frame,
2179 "duplicate_module_id",
2180 format!(
2181 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2182 ),
2183 )?])
2184 }
2185 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2186 return Ok(vec![control_error_frame(
2187 &frame,
2188 "invalid_module_id",
2189 err.to_string(),
2190 )?])
2191 }
2192 Err(err) => {
2193 return Ok(vec![control_error_frame(
2194 &frame,
2195 "registry_error",
2196 err.to_string(),
2197 )?])
2198 }
2199 };
2200
2201 let reply = if let Some(sink) = sink {
2202 let concurrency = manifest_concurrency(®istration.manifest);
2218 if let Err(err) = self.forwarding.register_module_connection_acked(
2219 connection_id,
2220 registration.manifest.module_id.clone(),
2221 negotiated_ver,
2222 concurrency,
2223 sink,
2224 hello_ack,
2225 ) {
2226 if matches!(
2230 self.deregister_connection(
2231 connection_id,
2232 RegistrationEndReason::RegistrationFailed,
2233 ),
2234 Ok(r) if !r.is_empty()
2235 ) {
2236 crate::supervise::notify_registration_release();
2237 }
2238 return Ok(vec![control_error_frame(
2239 &frame,
2240 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2241 "invalid_hello"
2242 } else {
2243 forwarding_error_code(&err)
2244 },
2245 err.to_string(),
2246 )?]);
2247 }
2248 Vec::new()
2249 } else {
2250 vec![hello_ack]
2253 };
2254
2255 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2264 info!(
2265 module_id = %registration.manifest.module_id,
2266 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2267 );
2268 }
2269
2270 self.apply_registration_capabilities(®istration);
2271
2272 info!(
2273 module_id = %registration.manifest.module_id,
2274 module_version = %registration.manifest.module_version,
2275 negotiated_ver,
2276 routable_provider = manifest_provides_routable_role(®istration.manifest),
2277 connection_id = connection_id.get(),
2278 "module registered"
2279 );
2280
2281 Ok(reply)
2282 }
2283
2284 #[allow(clippy::too_many_arguments)]
2294 fn register_swap_candidate(
2295 &self,
2296 connection_id: ConnectionId,
2297 sink: Option<crate::FrameSink>,
2298 frame: &Frame,
2299 manifest: ModuleManifest,
2300 negotiated_ver: u8,
2301 control_ops: Vec<String>,
2302 hello_ack: Frame,
2303 ) -> Result<Vec<Frame>, RouterError> {
2304 let module_id = manifest.module_id.clone();
2305 let registration = match self.registry.register_candidate_with_control_ops(
2306 manifest,
2307 negotiated_ver,
2308 connection_id,
2309 control_ops,
2310 ) {
2311 Ok(registration) => registration,
2312 Err(RegistryError::DuplicateModuleId { module_id }) => {
2313 return Ok(vec![control_error_frame(
2314 frame,
2315 "duplicate_module_id",
2316 format!(
2317 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2318 ),
2319 )?])
2320 }
2321 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2322 return Ok(vec![control_error_frame(
2323 frame,
2324 "invalid_module_id",
2325 err.to_string(),
2326 )?])
2327 }
2328 Err(err) => {
2329 return Ok(vec![control_error_frame(
2330 frame,
2331 "registry_error",
2332 err.to_string(),
2333 )?])
2334 }
2335 };
2336 let reply = if let Some(sink) = sink {
2337 let concurrency = manifest_concurrency(®istration.manifest);
2341 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2342 connection_id,
2343 module_id.clone(),
2344 negotiated_ver,
2345 concurrency,
2346 sink,
2347 hello_ack,
2348 ) {
2349 if matches!(
2350 self.deregister_connection(
2351 connection_id,
2352 RegistrationEndReason::RegistrationFailed,
2353 ),
2354 Ok(r) if !r.is_empty()
2355 ) {
2356 crate::supervise::notify_registration_release();
2357 }
2358 return Ok(vec![control_error_frame(
2359 frame,
2360 forwarding_error_code(&err),
2361 err.to_string(),
2362 )?]);
2363 }
2364 Vec::new()
2365 } else {
2366 vec![hello_ack]
2367 };
2368 self.supervisor.mark_swap_candidate_admitted(&module_id);
2369 info!(
2370 module_id = %module_id,
2371 module_version = %registration.manifest.module_version,
2372 negotiated_ver,
2373 ready = registration.ready,
2374 connection_id = connection_id.get(),
2375 "swap candidate registered; not routable until cutover"
2376 );
2377 Ok(reply)
2378 }
2379
2380 fn build_hello_ack(
2381 &self,
2382 frame: &Frame,
2383 negotiated_ver: u8,
2384 module_id: &str,
2385 ) -> Result<Frame, RouterError> {
2386 let ack = ModuleHelloAckBody {
2387 negotiated_ver,
2388 subc_ops: module_subc_ops(),
2389 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2390 storage: self
2391 .storage_config
2392 .as_ref()
2393 .map(|cfg| cfg.descriptor_for(module_id)),
2394 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2395 };
2396 let body = serde_json::to_vec(&ack).map_err(|err| {
2397 RouterError::backend(
2398 0,
2399 frame.header.corr,
2400 format!("failed to encode HELLO_ACK: {err}"),
2401 )
2402 })?;
2403
2404 Frame::build_with_version(
2405 negotiated_ver,
2406 FrameType::HelloAck,
2407 control_flags(),
2408 0,
2409 0,
2410 frame.header.corr,
2411 body,
2412 )
2413 .map_err(RouterError::FrameBuild)
2414 }
2415
2416 async fn handle_client_control_request(
2417 &self,
2418 ctx: &RouteCtx,
2419 frame: Frame,
2420 request: ClientControlRequest,
2421 ) -> Result<Vec<Frame>, RouterError> {
2422 match request {
2423 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2424 ClientControlRequest::CatalogList { module_id } => {
2425 self.handle_catalog_list(frame, module_id)
2426 }
2427 ClientControlRequest::RouteOpen {
2428 target,
2429 identity,
2430 consumer_identity,
2431 consumer_capabilities,
2432 role_versions,
2433 admission_facts,
2434 scope,
2435 } => {
2436 self.handle_route_open(
2437 ctx,
2438 frame,
2439 RouteOpenRequest {
2440 target,
2441 identity,
2442 consumer_identity,
2443 consumer_capabilities,
2444 role_versions,
2445 admission_facts,
2446 scope,
2447 },
2448 )
2449 .await
2450 }
2451 ClientControlRequest::RoutePoll {
2452 route_channel,
2453 route_epoch,
2454 kind,
2455 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2456 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2457 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2458 self.handle_supervisor_spawn_snapshot(frame)
2459 }
2460 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2461 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2462 }
2463 ClientControlRequest::SupervisorRestart {
2464 module_id,
2465 drain_timeout_ms,
2466 } => {
2467 self.log_supervisor_request_received(
2468 ctx,
2469 frame.header.corr,
2470 ops::SUPERVISOR_RESTART,
2471 Some(&module_id),
2472 None,
2473 )?;
2474 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2475 .await
2476 }
2477 ClientControlRequest::SupervisorSwap {
2478 module_id,
2479 ready_timeout_ms,
2480 } => {
2481 self.log_supervisor_request_received(
2482 ctx,
2483 frame.header.corr,
2484 ops::SUPERVISOR_SWAP,
2485 Some(&module_id),
2486 None,
2487 )?;
2488 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2489 .await
2490 }
2491 ClientControlRequest::SupervisorReload { module_id } => {
2492 self.log_supervisor_request_received(
2493 ctx,
2494 frame.header.corr,
2495 ops::SUPERVISOR_RELOAD,
2496 Some(&module_id),
2497 None,
2498 )?;
2499 self.handle_supervisor_reload(frame, module_id).await
2500 }
2501 ClientControlRequest::SupervisorRescan { preview } => {
2502 if !preview {
2503 self.log_supervisor_request_received(
2504 ctx,
2505 frame.header.corr,
2506 ops::SUPERVISOR_RESCAN,
2507 None,
2508 None,
2509 )?;
2510 }
2511 self.handle_supervisor_rescan(frame, preview).await
2512 }
2513 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2514 self.log_supervisor_request_received(
2515 ctx,
2516 frame.header.corr,
2517 ops::SUPERVISOR_RELEASE_RESERVED,
2518 Some(&module_id),
2519 None,
2520 )?;
2521 self.handle_supervisor_release_reserved(frame, module_id)
2522 .await
2523 }
2524 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2525 self.log_supervisor_request_received(
2526 ctx,
2527 frame.header.corr,
2528 ops::SUPERVISOR_SET_ENABLED,
2529 Some(&module_id),
2530 Some(enabled),
2531 )?;
2532 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2533 .await
2534 }
2535 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2536 self.handle_supervisor_health_probe(frame, module_id).await
2537 }
2538 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2539 ClientControlRequest::SupervisorRoutes { module_id } => {
2540 self.handle_supervisor_routes(frame, module_id)
2541 }
2542 ClientControlRequest::SupervisorProvenance { module_id } => {
2543 self.handle_supervisor_provenance(frame, module_id).await
2544 }
2545 ClientControlRequest::SupervisorStderrTail {
2546 module_id,
2547 max_lines,
2548 max_bytes,
2549 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2550 ClientControlRequest::SupervisorTerminals { module_id } => {
2551 self.handle_supervisor_terminals(frame, module_id).await
2552 }
2553 }
2554 }
2555
2556 fn log_supervisor_request_received(
2557 &self,
2558 ctx: &RouteCtx,
2559 corr: u64,
2560 op: &'static str,
2561 module_id: Option<&str>,
2562 enabled: Option<bool>,
2563 ) -> Result<(), RouterError> {
2564 let caller = self
2565 .registry
2566 .get_module_by_connection(ctx.connection_id)
2567 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2568 .map(|registration| Principal::Reserved {
2569 module_id: registration.manifest.module_id,
2570 })
2571 .unwrap_or(Principal::Direct);
2572 let caller = principal_label(&caller);
2573 info!(
2576 target: "control",
2577 op,
2578 module_id,
2579 enabled,
2580 connection_id = ctx.connection_id.get(),
2581 caller = %caller,
2582 "supervisor request received"
2583 );
2584 Ok(())
2585 }
2586
2587 fn handle_module_control_request(
2588 &self,
2589 connection_id: ConnectionId,
2590 frame: Frame,
2591 request: ModuleControlRequestFromModule,
2592 ) -> Result<Vec<Frame>, RouterError> {
2593 match request {
2594 ModuleControlRequestFromModule::CatalogUpdate {
2595 provides,
2596 capabilities,
2597 ready,
2598 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2599 ModuleControlRequestFromModule::LiveRoots {} => {
2600 let registered = self
2601 .registry
2602 .get_module_by_connection(connection_id)
2603 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2604 let Some(registration) = registered else {
2605 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2606 };
2607 let response = self
2608 .forwarding
2609 .live_roots(®istration.manifest.module_id)
2610 .map_err(RouterError::Forwarding)?;
2611 Ok(vec![control_response_body_frame(
2612 &frame,
2613 &response,
2614 "ModuleControlResponseToModule::LiveRoots",
2615 )?])
2616 }
2617 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2618 self.handle_scope_sync(connection_id, frame, generation, scopes)
2619 }
2620 ModuleControlRequestFromModule::ScopeApply {
2621 generation,
2622 upsert,
2623 end,
2624 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2625 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2626 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2627 }
2628 }
2629 }
2630
2631 fn handle_operator_confirm(
2632 &self,
2633 ctx: &RouteCtx,
2634 frame: Frame,
2635 ) -> Result<Vec<Frame>, RouterError> {
2636 use crate::operator_confirm::{audit, Outcome};
2637 let registration = self
2638 .registry
2639 .get_module_by_connection(ctx.connection_id)
2640 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2641 let Some(registration) = registration else {
2642 let outcome = Outcome::refusal("not_registered");
2643 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2644 return Ok(vec![outcome.frame(&frame)]);
2645 };
2646 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2647 Ok(request) => request,
2648 Err(_) => {
2649 let outcome = Outcome::refusal("invalid_control_body");
2650 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2651 return Ok(vec![outcome.frame(&frame)]);
2652 }
2653 };
2654 let module_id = registration.manifest.module_id;
2655 let nonce = self
2659 .hello_launch_nonces
2660 .lock()
2661 .unwrap_or_else(|p| p.into_inner())
2662 .nonce(ctx.connection_id)
2663 .map(str::to_owned);
2664 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2665 self.supervisor
2666 .spawned_consumer_authorized(&module_id, nonce)
2667 });
2668 let confirms = self.forwarding.operator_confirms();
2669 self.forwarding
2670 .with_operator_route(
2671 ctx.connection_id,
2672 request.route_channel,
2673 request.route_epoch,
2674 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
2675 )
2676 .map_err(RouterError::Forwarding)
2677 }
2678
2679 fn handle_scope_sync(
2683 &self,
2684 connection_id: ConnectionId,
2685 frame: Frame,
2686 generation: u64,
2687 scopes: Vec<ScopeRecord>,
2688 ) -> Result<Vec<Frame>, RouterError> {
2689 self.handle_scope_change(connection_id, frame, generation, scopes, None)
2690 }
2691
2692 fn handle_scope_change(
2693 &self,
2694 connection_id: ConnectionId,
2695 frame: Frame,
2696 generation: u64,
2697 scopes: Vec<ScopeRecord>,
2698 end: Option<Vec<ScopeEnd>>,
2699 ) -> Result<Vec<Frame>, RouterError> {
2700 let is_apply = end.is_some();
2701 let Some(registration) = self
2702 .registry
2703 .get_module_by_connection(connection_id)
2704 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2705 else {
2706 return Ok(vec![control_error_frame(
2707 &frame,
2708 "not_registered",
2709 "scope.sync and scope.apply require an active module registration owned by this connection",
2710 )?]);
2711 };
2712 let owner = registration.manifest.module_id;
2713 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2714 let is_current_launch = |connection: ConnectionId| {
2715 self.hello_launch_nonces
2716 .lock()
2717 .unwrap_or_else(|poisoned| poisoned.into_inner())
2718 .presented(connection, current_nonce.as_deref())
2719 };
2720 let mut table = self
2725 .scopes
2726 .write()
2727 .unwrap_or_else(|poisoned| poisoned.into_inner());
2728 let outcome = match end {
2729 Some(end) => table.apply(
2730 &owner,
2731 connection_id,
2732 is_current_launch,
2733 generation,
2734 scopes,
2735 end,
2736 ),
2737 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
2738 };
2739 let drained = match &outcome {
2740 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
2741 Err(_) => Vec::new(),
2742 };
2743 let has_deadlines = outcome.is_ok() && table.has_deadlines();
2744 drop(table);
2745 if has_deadlines {
2746 self.scope_deadline_added.notify_one();
2747 }
2748 match outcome {
2749 Ok(applied) => {
2750 let counts = ScopeOutcomeCounts::of(&applied.results);
2751 info!(
2752 owner = %owner,
2753 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2754 generation,
2755 records = applied.results.len(),
2756 created = counts.created,
2757 replaced = counts.replaced,
2758 updated = counts.updated,
2759 unchanged = counts.unchanged,
2760 refused = counts.refused,
2761 ended = applied.ended.len(),
2762 tag_changes = applied.tag_changes.len(),
2763 routes_closed = drained.len(),
2764 "scope change accepted"
2765 );
2766 for refused in applied
2772 .results
2773 .iter()
2774 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2775 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2776 {
2777 warn!(
2778 owner = %owner,
2779 generation,
2780 scope_ref = %refused.scope_ref,
2781 scope_epoch = refused.scope_epoch,
2782 code = refused.code.as_deref().unwrap_or(""),
2783 "scope record refused"
2784 );
2785 }
2786 self.close_scope_drained_routes(drained);
2787 let response = if is_apply {
2788 ModuleControlResponseToModule::ScopeApply {
2789 generation,
2790 results: applied.results,
2791 end_results: applied.end_results,
2792 ended: applied.ended,
2793 }
2794 } else {
2795 ModuleControlResponseToModule::ScopeSync {
2796 generation,
2797 results: applied.results,
2798 ended: applied.ended,
2799 }
2800 };
2801 Ok(vec![control_response_body_frame(
2802 &frame,
2803 &response,
2804 "scope change response",
2805 )?])
2806 }
2807 Err(refusal) => {
2808 info!(
2809 owner = %owner,
2810 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2811 generation,
2812 code = refusal.code,
2813 "scope change refused"
2814 );
2815 Ok(vec![control_error_frame(
2816 &frame,
2817 refusal.code,
2818 refusal.message,
2819 )?])
2820 }
2821 }
2822 }
2823
2824 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2830 if drained.is_empty() {
2831 return;
2832 }
2833 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2834 BTreeMap::new();
2835 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2836 for route in drained {
2837 warn!(
2838 module_id = %route.module_id,
2839 reason = ?route.reason,
2840 client_connection_id = route.client.connection_id.get(),
2841 route_channel = route.client.channel,
2842 "closing route because its scope changed"
2843 );
2844 pushes
2845 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2846 .or_insert_with(|| (route.reason, Vec::new()))
2847 .1
2848 .push(EndpointRoute {
2849 goodbye_target: route.client.clone(),
2850 principal: Principal::Unverified,
2851 bound_at: Instant::now(),
2852 draining: false,
2853 drain_reason: None,
2854 });
2855 goodbyes.push(route.module);
2856 goodbyes.push(route.client);
2857 }
2858 for ((module_id, _), (reason, routes)) in pushes {
2859 send_route_control_pushes(
2860 &self.forwarding,
2861 routes,
2862 ClientControlPush::RouteClosed {
2863 module_id,
2864 channels: Vec::new(),
2865 reason,
2866 drained: false,
2867 abandoned: 0,
2868 excluded_subscriptions: 0,
2869 terminal: Some(false),
2870 },
2871 );
2872 }
2873 self.emit_route_goodbyes(goodbyes);
2874 }
2875
2876 fn handle_scope_describe(
2879 &self,
2880 connection_id: ConnectionId,
2881 frame: Frame,
2882 owner: Principal,
2883 scope_ref: String,
2884 ) -> Result<Vec<Frame>, RouterError> {
2885 let registered = self
2886 .registry
2887 .get_module_by_connection(connection_id)
2888 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2889 if registered.is_none() {
2890 return Ok(vec![control_error_frame(
2891 &frame,
2892 "not_registered",
2893 "scope.describe requires an active module registration owned by this connection",
2894 )?]);
2895 }
2896 let description = self
2897 .scopes
2898 .read()
2899 .unwrap_or_else(|poisoned| poisoned.into_inner())
2900 .describe(&owner, &scope_ref);
2901 let owner_configured = match &owner {
2902 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
2907 _ => false,
2908 };
2909 let response = ModuleControlResponseToModule::ScopeDescribe {
2910 status: description.status,
2911 scope_epoch: description.scope_epoch,
2912 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2913 owner_synced: description.owner_synced,
2914 owner_configured,
2915 scope: description.stamp,
2916 };
2917 Ok(vec![control_response_body_frame(
2918 &frame,
2919 &response,
2920 "ModuleControlResponseToModule::ScopeDescribe",
2921 )?])
2922 }
2923
2924 fn handle_catalog_update(
2925 &self,
2926 connection_id: ConnectionId,
2927 frame: Frame,
2928 provides: Vec<ProviderRole>,
2929 capabilities: Option<CapabilityDeclarations>,
2930 ready: Option<bool>,
2931 ) -> Result<Vec<Frame>, RouterError> {
2932 self.refresh_capability_requirements();
2933 let Some(registration) = self
2934 .registry
2935 .get_module_by_connection(connection_id)
2936 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2937 else {
2938 return Ok(vec![control_error_frame(
2939 &frame,
2940 "not_registered",
2941 "catalog.update requires an active module registration owned by this connection",
2942 )?]);
2943 };
2944
2945 if let Some(message) =
2946 catalog_update_frozen_field_message(®istration.manifest, &provides)
2947 {
2948 return Ok(vec![control_error_frame(
2949 &frame,
2950 "catalog_update_frozen_field",
2951 message,
2952 )?]);
2953 }
2954
2955 let mut candidate = registration.manifest.clone();
2956 candidate.provides = provides.clone();
2957 candidate.capabilities = capabilities
2958 .clone()
2959 .or_else(|| registration.manifest.capabilities.clone());
2960 if let Err(err) = candidate.validate_capability_grammar() {
2961 return Ok(vec![control_error_frame(
2962 &frame,
2963 "invalid_capability_grammar",
2964 err.to_string(),
2965 )?]);
2966 }
2967
2968 let mut conflicts = self
2971 .capability_evaluator
2972 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
2973 if let Some(conflict) = conflicts.first() {
2974 let message = format!(
2975 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2976 conflict.capability, conflict.claimants[0], candidate.module_id
2977 );
2978 for conflict in &mut conflicts {
2979 conflict.source = DuplicateClaimSource::CatalogUpdate;
2980 }
2981 log_duplicate_claim_events(conflicts);
2982 return Ok(vec![control_error_frame(
2983 &frame,
2984 "reserved_capability",
2985 message,
2986 )?]);
2987 }
2988
2989 let updated = self
2990 .registry
2991 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
2992 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2993 if updated.is_none() {
2994 return Ok(vec![control_error_frame(
2995 &frame,
2996 "not_registered",
2997 "catalog.update requires an active module registration owned by this connection",
2998 )?]);
2999 }
3000 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3001 log_duplicate_claim_events(
3002 self.capability_evaluator
3003 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3004 );
3005 }
3006 if capability_census_trigger(
3007 registration.manifest.capabilities.as_ref(),
3008 updated
3009 .as_ref()
3010 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3011 ) {
3012 self.enforce_capability_denies();
3013 }
3014 self.refresh_capability_requirements();
3015
3016 let response = ModuleControlResponseToModule::CatalogUpdate {};
3017 control_response_body_frame(
3018 &frame,
3019 &response,
3020 "ModuleControlResponseToModule::CatalogUpdate",
3021 )
3022 .map(|frame| vec![frame])
3023 }
3024
3025 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3026 self.refresh_capability_requirements();
3027 let mut counters = self.counters.snapshot();
3033 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3034 self.forwarding.client_route_concentration(),
3035 counters.as_object_mut(),
3036 ) {
3037 obj.insert(
3038 "client_connections_with_routes".into(),
3039 connections_with_routes.into(),
3040 );
3041 obj.insert("max_routes_on_one_connection".into(), max.into());
3042 }
3043 if let (Some(open_breakers), Some(obj)) = (
3049 self.route_bind_breakers.open_snapshot(),
3050 counters.as_object_mut(),
3051 ) {
3052 obj.insert("route_bind_breakers_open".into(), open_breakers);
3053 }
3054 let response = ClientControlResponse::ServerDescribe {
3055 protocol_ver: PROTOCOL_VERSION,
3056 subc_ops: subc_ops(),
3057 capabilities: self.subc_capabilities.as_ref().to_vec(),
3058 connected_clients: self.connected_clients.count(),
3059 counters: Some(counters),
3060 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3061 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3062 capability_requirements: self.capability_requirement_statuses(),
3063 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3064 };
3065 Ok(vec![control_response_body_frame(
3066 &frame,
3067 &response,
3068 "ClientControlResponse::ServerDescribe",
3069 )?])
3070 }
3071
3072 fn handle_catalog_list(
3073 &self,
3074 frame: Frame,
3075 module_id: Option<String>,
3076 ) -> Result<Vec<Frame>, RouterError> {
3077 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3078 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3079 })?;
3080 let entries = modules
3081 .into_iter()
3082 .filter(|registration| {
3083 module_id
3084 .as_deref()
3085 .map(|wanted| registration.manifest.module_id == wanted)
3086 .unwrap_or(true)
3087 })
3088 .map(|registration| {
3089 let not_ready = self.not_ready_reason(®istration);
3090 let roles = registration.manifest.provides;
3091 CatalogEntry::new(
3092 registration.manifest.module_id,
3093 roles,
3094 registration.control_ops,
3095 )
3096 .with_ready(not_ready.is_none())
3097 .with_not_ready(not_ready)
3098 .with_module_version(Some(registration.manifest.module_version))
3099 .with_capabilities(registration.manifest.capabilities)
3100 .with_self_signals(registration.manifest.self_signals)
3101 .with_events(registration.manifest.events)
3102 })
3103 .collect();
3104 let response = ClientControlResponse::CatalogList {
3105 generation,
3106 modules: entries,
3107 subc_ops: subc_ops(),
3108 };
3109 Ok(vec![control_response_body_frame(
3110 &frame,
3111 &response,
3112 "ClientControlResponse::CatalogList",
3113 )?])
3114 }
3115
3116 fn route_open_principal(
3117 &self,
3118 frame: &Frame,
3119 consumer_identity: Option<ConsumerIdentity>,
3120 ) -> Result<Result<Principal, Frame>, RouterError> {
3121 let Some(consumer_identity) = consumer_identity else {
3122 return Ok(Ok(Principal::Direct));
3123 };
3124
3125 if self.supervisor.spawned_consumer_authorized(
3126 &consumer_identity.module_id,
3127 &consumer_identity.launch_nonce,
3128 ) {
3129 return Ok(Ok(Principal::Reserved {
3130 module_id: consumer_identity.module_id,
3131 }));
3132 }
3133
3134 Ok(Err(control_error_frame(
3135 frame,
3136 "bad_consumer_identity",
3137 format!(
3138 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3139 consumer_identity.module_id
3140 ),
3141 )?))
3142 }
3143
3144 fn route_open_refusal_frame(
3151 &self,
3152 ctx: &RouteCtx,
3153 frame: &Frame,
3154 module_id: &str,
3155 reason: &'static str,
3156 code: &'static str,
3157 message: impl Into<String>,
3158 ) -> Result<Frame, RouterError> {
3159 self.observe_route_open_refusal(ctx, module_id, reason, code);
3160 control_error_frame(frame, code, message.into())
3161 }
3162
3163 fn route_open_breaker_refusal_frame(
3181 &self,
3182 ctx: &RouteCtx,
3183 frame: &Frame,
3184 module_id: &str,
3185 consecutive_timeouts: u32,
3186 retry_in: Duration,
3187 probe_in_flight: bool,
3188 ) -> Result<Frame, RouterError> {
3189 self.counters
3190 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3191 debug!(
3192 target: "control",
3193 code = "module_timeout",
3194 module_id = ?module_id,
3195 connection_id = ctx.connection_id.get(),
3196 consecutive_timeouts,
3197 retry_in_ms = retry_in.as_millis() as u64,
3198 probe_in_flight,
3199 "route.open refused by open bind-relay breaker"
3200 );
3201 let detail = if probe_in_flight {
3208 "one test route is already being tried; retry once it settles".to_string()
3209 } else {
3210 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3211 };
3212 control_error_frame(
3213 frame,
3214 "module_timeout",
3215 format!(
3216 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3217 timed out in a row); {detail}; its established routes are unaffected"
3218 ),
3219 )
3220 }
3221
3222 fn observe_route_open_refusal(
3235 &self,
3236 ctx: &RouteCtx,
3237 module_id: &str,
3238 reason: &'static str,
3239 code: &'static str,
3240 ) {
3241 self.counters.increment_route_open_refused(code);
3242 info!(
3243 target: "control",
3244 code,
3245 reason,
3246 module_id = ?module_id,
3247 connection_id = ctx.connection_id.get(),
3248 "route.open refused"
3249 );
3250 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3251 self.route_outages.record_not_serving(module_id, reason);
3252 }
3253 }
3254
3255 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3300 self.route_outages.record_accepted(module_id);
3301 self.counters.increment_route_open_accepted(principal);
3302 info!(
3303 target: "control",
3304 principal,
3305 module_id,
3306 connection_id = ctx.connection_id.get(),
3307 "route.open accepted"
3308 );
3309 }
3310
3311 fn supervised_absent_route_open_refusal_frame(
3312 &self,
3313 ctx: &RouteCtx,
3314 frame: &Frame,
3315 module_id: &str,
3316 code: &'static str,
3317 status: &crate::supervise::ModuleStatus,
3318 ) -> Result<Frame, RouterError> {
3319 self.counters.increment_route_open_refused(code);
3320 info!(
3321 target: "control",
3322 code,
3323 reason = "supervised_not_registered",
3324 module_id = ?module_id,
3325 connection_id = ctx.connection_id.get(),
3326 state = %status.state,
3327 enabled = status.enabled,
3328 live = status.live,
3329 "route.open refused"
3330 );
3331 self.route_outages
3335 .record_not_serving(module_id, "supervised_not_registered");
3336 control_error_frame(
3337 frame,
3338 code,
3339 format!(
3340 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3341 status.state, status.enabled, status.live
3342 ),
3343 )
3344 }
3345
3346 async fn handle_route_open(
3347 &self,
3348 ctx: &RouteCtx,
3349 frame: Frame,
3350 request: RouteOpenRequest,
3351 ) -> Result<Vec<Frame>, RouterError> {
3352 let RouteOpenRequest {
3353 target,
3354 mut identity,
3355 consumer_identity,
3356 consumer_capabilities,
3357 role_versions,
3358 admission_facts,
3359 scope,
3360 } = request;
3361 let target_module_id = target_module_id(&target).to_string();
3362 if self
3363 .registry
3364 .get_module_by_connection(ctx.connection_id)
3365 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3366 .is_some()
3367 {
3368 return Ok(vec![control_error_frame(
3369 &frame,
3370 "invalid_request",
3371 "module connections cannot open client routes",
3372 )?]);
3373 }
3374 debug!(
3375 connection_id = ctx.connection_id.get(),
3376 corr = frame.header.corr,
3377 module_id = %target_module_id,
3378 "handling route.open"
3379 );
3380
3381 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3387 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3388 self.observe_route_open_refusal(
3389 ctx,
3390 &target_module_id,
3391 "invalid_role_versions",
3392 error_codes::INVALID_REQUEST,
3393 );
3394 return Ok(vec![control_error_body_frame(
3395 &frame,
3396 ErrorBody {
3397 code: error_codes::INVALID_REQUEST.to_string(),
3398 message: error.to_string(),
3399 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3400 },
3401 )?]);
3402 }
3403
3404 let Some(registration) = self
3423 .registry
3424 .get_module(&target_module_id)
3425 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3426 else {
3427 if let Some((status, warming)) =
3428 self.supervisor_status(&target_module_id, frame.header.corr)?
3429 {
3430 if status.protocol == ModuleProtocol::None {
3437 return Ok(vec![self.route_open_refusal_frame(
3438 ctx,
3439 &frame,
3440 &target_module_id,
3441 "protocol_none",
3442 error_codes::MODULE_NO_PROTOCOL,
3443 format!(
3444 "module_id '{target_module_id}' is declared protocol: none; \
3445 it speaks no subc wire and serves no routes"
3446 ),
3447 )?]);
3448 }
3449 let code = if warming {
3450 "module_warming"
3451 } else {
3452 "target_unavailable"
3453 };
3454 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3455 ctx,
3456 &frame,
3457 &target_module_id,
3458 code,
3459 &status,
3460 )?]);
3461 }
3462 if let Some(removed_ago_ms) =
3463 self.supervisor.removal_tombstone_age_ms(&target_module_id)
3464 {
3465 return Ok(vec![self.route_open_refusal_frame(
3466 ctx,
3467 &frame,
3468 &target_module_id,
3469 "removed",
3470 error_codes::MODULE_REMOVED,
3471 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3472 )?]);
3473 }
3474 return Ok(vec![self.route_open_refusal_frame(
3475 ctx,
3476 &frame,
3477 &target_module_id,
3478 "not_registered",
3479 error_codes::UNKNOWN_MODULE,
3480 format!("module_id '{target_module_id}' is not registered"),
3481 )?]);
3482 };
3483
3484 if !registration.ready {
3488 self.counters
3489 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3490 info!(
3491 target: "control",
3492 code = error_codes::MODULE_WARMING,
3493 module_id = ?target_module_id,
3494 connection_id = ctx.connection_id.get(),
3495 reason = "declared_not_ready",
3496 "route.open refused"
3497 );
3498 self.route_outages
3501 .record_not_serving(&target_module_id, "declared_not_ready");
3502 return Ok(vec![control_error_body_frame(
3503 &frame,
3504 ErrorBody {
3505 code: error_codes::MODULE_WARMING.to_string(),
3506 message: format!(
3507 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3508 ),
3509 detail: Some(serde_json::json!({
3510 "reason": "declared_not_ready"
3511 })),
3512 },
3513 )?]);
3514 }
3515
3516 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3535 self.counters
3536 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3537 info!(
3538 target: "control",
3539 code = error_codes::MODULE_WARMING,
3540 module_id = ?target_module_id,
3541 connection_id = ctx.connection_id.get(),
3542 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3543 capability = %capability,
3544 "route.open refused"
3545 );
3546 return Ok(vec![control_error_body_frame(
3547 &frame,
3548 ErrorBody {
3549 code: error_codes::MODULE_WARMING.to_string(),
3550 message: format!(
3551 "module_id '{target_module_id}' requires capability '{capability}', \
3552 which no registered module provides; retry"
3553 ),
3554 detail: Some(serde_json::json!({
3555 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3556 "capability": capability,
3557 })),
3558 },
3559 )?]);
3560 }
3561
3562 if !target_has_required_role(&target, ®istration.manifest.provides) {
3563 return Ok(vec![self.route_open_refusal_frame(
3564 ctx,
3565 &frame,
3566 &target_module_id,
3567 "role_not_provided",
3568 "target_unavailable",
3569 format!("module_id '{target_module_id}' does not provide the requested target"),
3570 )?]);
3571 }
3572
3573 if registration.state != ChannelState::Active {
3574 return Ok(vec![self.route_open_refusal_frame(
3575 ctx,
3576 &frame,
3577 &target_module_id,
3578 "registration_not_active",
3579 "target_unavailable",
3580 format!("module_id '{target_module_id}' is not active"),
3581 )?]);
3582 }
3583
3584 if self
3585 .forwarding
3586 .module_is_draining(&target_module_id)
3587 .map_err(RouterError::Forwarding)?
3588 {
3589 return Ok(vec![self.route_open_refusal_frame(
3590 ctx,
3591 &frame,
3592 &target_module_id,
3593 "reloading",
3594 "module_reloading",
3595 format!("module_id '{target_module_id}' is reloading"),
3596 )?]);
3597 }
3598
3599 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3600 process_liveness.process_live(&target_module_id) == Some(false)
3601 }) {
3602 if process_liveness.process_replacing(&target_module_id) {
3609 return Ok(vec![self.route_open_refusal_frame(
3610 ctx,
3611 &frame,
3612 &target_module_id,
3613 "reloading",
3614 "module_reloading",
3615 format!("module_id '{target_module_id}' is reloading"),
3616 )?]);
3617 }
3618 return Ok(vec![self.route_open_refusal_frame(
3619 ctx,
3620 &frame,
3621 &target_module_id,
3622 "supervisor_not_live",
3623 "target_unavailable",
3624 format!("module_id '{target_module_id}' is not live"),
3625 )?]);
3626 }
3627
3628 if !self
3629 .forwarding
3630 .has_live_module_connection(&target_module_id)
3631 .map_err(RouterError::Forwarding)?
3632 {
3633 return Ok(vec![self.route_open_refusal_frame(
3634 ctx,
3635 &frame,
3636 &target_module_id,
3637 "no_forwarding_connection",
3638 "target_unavailable",
3639 format!("module_id '{target_module_id}' has no live forwarding connection"),
3640 )?]);
3641 }
3642
3643 if let Some(error) =
3644 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
3645 {
3646 self.observe_route_open_refusal(
3647 ctx,
3648 &target_module_id,
3649 "op_not_allowed",
3650 "op_not_allowed",
3651 );
3652 return Ok(vec![error]);
3653 }
3654
3655 let principal = match self.route_open_principal(&frame, consumer_identity)? {
3656 Ok(principal) => principal,
3657 Err(error) => {
3658 self.observe_route_open_refusal(
3659 ctx,
3660 &target_module_id,
3661 "bad_consumer_identity",
3662 "bad_consumer_identity",
3663 );
3664 return Ok(vec![error]);
3665 }
3666 };
3667
3668 if let Principal::Reserved {
3672 module_id: opening_module_id,
3673 } = &principal
3674 {
3675 if let Some(opening_registration) = self
3676 .registry
3677 .get_module(opening_module_id)
3678 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3679 {
3680 if let Some(capability) =
3681 denied_capability(&opening_registration.manifest, ®istration.manifest)
3682 {
3683 warn!(
3684 opening_module_id,
3685 target_module_id,
3686 capability,
3687 "refusing route.open because an attested capability deny edge matches"
3688 );
3689 return Ok(vec![self.route_open_refusal_frame(
3690 ctx,
3691 &frame,
3692 &target_module_id,
3693 "capability_deny_edge",
3694 "capability_forbidden",
3695 format!(
3696 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
3697 ),
3698 )?]);
3699 }
3700 }
3701 }
3702
3703 if admission_facts.is_some() {
3704 let carrier_matches = matches!(
3705 &principal,
3706 Principal::Reserved { module_id }
3707 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
3708 );
3709 if !carrier_matches {
3710 return Ok(vec![self.route_open_refusal_frame(
3711 ctx,
3712 &frame,
3713 &target_module_id,
3714 "admission_facts_carrier_not_permitted",
3715 "admission_facts_not_permitted",
3716 "admission facts may only be carried by the configured reserved module",
3717 )?]);
3718 }
3719
3720 let target_allowed = self
3721 .admission_facts_targets
3722 .as_ref()
3723 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
3724 if !target_allowed {
3725 return Ok(vec![self.route_open_refusal_frame(
3726 ctx,
3727 &frame,
3728 &target_module_id,
3729 "admission_facts_target_not_listed",
3730 "admission_facts_target_not_allowed",
3731 format!(
3732 "admission facts are not permitted for target module_id '{target_module_id}'"
3733 ),
3734 )?]);
3735 }
3736
3737 }
3741
3742 let (bound_scope, scope_stamp) = match scope {
3748 None => (None, None),
3749 Some(selector) => {
3750 let owner_configured = match &selector.owner {
3751 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3757 _ => false,
3758 };
3759 let admitted = self
3760 .scopes
3761 .read()
3762 .unwrap_or_else(|poisoned| poisoned.into_inner())
3763 .admit(&principal, &target_module_id, &selector, owner_configured)
3764 .and_then(|admission| {
3765 crate::scopes::check_target_flow_support(
3766 &admission.stamp,
3767 &target_module_id,
3768 registration.manifest.capabilities.as_ref(),
3769 )?;
3770 crate::scopes::check_target_agent_run_support(
3771 &admission.stamp,
3772 &target_module_id,
3773 registration.manifest.capabilities.as_ref(),
3774 )?;
3775 Ok(admission)
3776 });
3777 match admitted {
3778 Ok(admission) => (
3779 Some(BoundScope {
3780 owner: admission.owner,
3781 scope_ref: admission.stamp.scope_ref.clone(),
3782 tag: admission.tag,
3783 }),
3784 Some(admission.stamp),
3785 ),
3786 Err(refusal) => {
3787 return Ok(vec![self.route_open_refusal_frame(
3788 ctx,
3789 &frame,
3790 &target_module_id,
3791 refusal.code,
3792 refusal.code,
3793 refusal.message,
3794 )?]);
3795 }
3796 }
3797 }
3798 };
3799
3800 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
3822 Ok(project_root) => project_root,
3823 Err(err) => {
3824 return Ok(vec![control_error_frame(
3825 &frame,
3826 "invalid_project_root",
3827 err.to_string(),
3828 )?])
3829 }
3830 };
3831 identity.project_root = project_root.as_path().to_path_buf();
3832
3833 let _concurrency_guard = match self
3844 .route_bind_concurrency
3845 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
3846 {
3847 Ok(guard) => guard,
3848 Err(in_flight) => {
3849 return Ok(vec![self.route_open_target_capacity_refusal(
3850 ctx,
3851 &frame,
3852 &target_module_id,
3853 in_flight,
3854 )?]);
3855 }
3856 };
3857
3858 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
3861 RouteBindAdmission::Admitted { guard, probe } => {
3862 if probe {
3863 info!(
3864 module_id = %target_module_id,
3865 connection_id = ctx.connection_id.get(),
3866 "route.bind breaker half-open: admitting one probe"
3867 );
3868 }
3869 guard
3870 }
3871 RouteBindAdmission::Refused {
3872 consecutive_timeouts,
3873 retry_in,
3874 probe_in_flight,
3875 } => {
3876 return Ok(vec![self.route_open_breaker_refusal_frame(
3877 ctx,
3878 &frame,
3879 &target_module_id,
3880 consecutive_timeouts,
3881 retry_in,
3882 probe_in_flight,
3883 )?]);
3884 }
3885 };
3886
3887 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
3892 let relay_deadline = Instant::now() + route_bind_relay_timeout;
3893 let pending = match self
3894 .forwarding
3895 .begin_route_bind_relay_for(
3896 ctx.connection_id,
3897 ctx.egress.clone(),
3898 response_version(&frame),
3899 frame.header.corr,
3900 &target_module_id,
3901 principal.clone(),
3902 bound_scope,
3903 Some(project_root),
3904 relay_deadline,
3905 )
3906 .await
3907 {
3908 Ok(pending) => pending,
3909 Err(err) => {
3910 return Ok(vec![self.route_open_refusal_frame(
3911 ctx,
3912 &frame,
3913 &target_module_id,
3914 "relay_reservation_failed",
3915 forwarding_error_code(&err),
3916 err.to_string(),
3917 )?])
3918 }
3919 };
3920 let crate::forwarding::PendingRouteBindRelay {
3921 endpoint,
3922 module_sink,
3923 negotiated_ver,
3924 client_channel,
3925 client_epoch,
3926 module_channel,
3927 module_epoch,
3928 corr: relay_corr,
3929 receiver,
3930 } = pending;
3931 let mut reservation =
3932 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
3933
3934 if let Some(stamp) = scope_stamp
3939 .as_ref()
3940 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
3941 {
3942 let relay_registration = self
3943 .registry
3944 .get_module_by_connection(endpoint.connection_id)
3945 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3946 if let Err(refusal) = crate::scopes::check_target_flow_support(
3947 stamp,
3948 &target_module_id,
3949 relay_registration
3950 .as_ref()
3951 .and_then(|registration| registration.manifest.capabilities.as_ref()),
3952 )
3953 .and_then(|()| {
3954 crate::scopes::check_target_agent_run_support(
3955 stamp,
3956 &target_module_id,
3957 relay_registration
3958 .as_ref()
3959 .and_then(|registration| registration.manifest.capabilities.as_ref()),
3960 )
3961 }) {
3962 reservation.release_and_disarm();
3963 return Ok(vec![self.route_open_refusal_frame(
3964 ctx,
3965 &frame,
3966 &target_module_id,
3967 refusal.code,
3968 refusal.code,
3969 refusal.message,
3970 )?]);
3971 }
3972 }
3973
3974 debug!(
3975 connection_id = ctx.connection_id.get(),
3976 client_channel,
3977 client_epoch,
3978 module_channel,
3979 module_epoch,
3980 "reserved route handle pair"
3981 );
3982 let principal_label = principal_label(&principal);
3985 let relay = ModuleControlRequest::RouteBind {
3986 route_channel: module_channel,
3987 epoch: module_epoch,
3988 target,
3989 identity,
3990 principal: Some(principal),
3991 consumer_capabilities,
3992 role_versions,
3993 admission_facts,
3994 scope: scope_stamp,
3995 };
3996 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
3997 RouterError::backend(
3998 0,
3999 frame.header.corr,
4000 format!("failed to encode route.bind request: {err}"),
4001 )
4002 })?;
4003 let relay_frame = Frame::build_with_version(
4004 negotiated_ver,
4005 FrameType::Request,
4006 control_flags(),
4007 0,
4008 0,
4009 relay_corr,
4010 relay_body,
4011 )
4012 .map_err(RouterError::FrameBuild)?;
4013
4014 if let Err(err) = module_sink.send(relay_frame).await {
4015 reservation.release_and_disarm();
4016 return Ok(vec![self.route_open_refusal_frame(
4017 ctx,
4018 &frame,
4019 &target_module_id,
4020 "relay_send_failed",
4021 "target_unavailable",
4022 err.to_string(),
4023 )?]);
4024 }
4025
4026 #[cfg(test)]
4027 if let Some(hold) = &self.route_bind_relay_sent_hold {
4028 hold.wait().await;
4032 hold.wait().await;
4033 }
4034
4035 if self
4040 .forwarding
4041 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4042 .map_err(RouterError::Forwarding)?
4043 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4044 {
4045 self.send_abandoned_route_bind_goodbye(
4046 &module_sink,
4047 negotiated_ver,
4048 module_channel,
4049 module_epoch,
4050 );
4051 }
4052
4053 match timeout_at(relay_deadline, receiver).await {
4054 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4055 reservation.disarm();
4056 if breaker.record_accepted() {
4057 info!(
4058 module_id = %target_module_id,
4059 "route.bind breaker closed: the probe was accepted"
4060 );
4061 }
4062 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4063 Ok(Vec::new())
4064 }
4065 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4066 reservation.release_and_disarm();
4067 breaker.record_inconclusive();
4071 let scope_code = match body.code.as_str() {
4075 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4076 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4077 _ => None,
4078 };
4079 if let Some(code) = scope_code {
4080 self.observe_route_open_refusal(
4081 ctx,
4082 &target_module_id,
4083 "scope_changed_before_commit",
4084 code,
4085 );
4086 return Ok(vec![control_error_body_frame(&frame, body)?]);
4087 }
4088 self.counters
4089 .increment_route_open_refused("module_rejected");
4090 info!(
4091 target: "control",
4092 code = "module_rejected",
4093 module_code = ?body.code,
4094 module_id = ?target_module_id,
4095 connection_id = ctx.connection_id.get(),
4096 "route.open refused"
4097 );
4098 Ok(vec![control_error_body_frame(&frame, body)?])
4099 }
4100 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4101 reservation.release_and_disarm();
4102 breaker.record_inconclusive();
4103 tracing::warn!(
4110 module_id = %target_module_id,
4111 "route.bind relay abandoned: {message}"
4112 );
4113 Ok(vec![self.route_open_refusal_frame(
4114 ctx,
4115 &frame,
4116 &target_module_id,
4117 "relay_abandoned",
4118 "target_unavailable",
4119 message,
4120 )?])
4121 }
4122 Ok(Err(_)) => {
4123 reservation.release_and_disarm();
4124 breaker.record_inconclusive();
4125 Ok(vec![self.route_open_refusal_frame(
4126 ctx,
4127 &frame,
4128 &target_module_id,
4129 "relay_waiter_canceled",
4130 "target_unavailable",
4131 "route.bind relay waiter was canceled before the module responded",
4132 )?])
4133 }
4134 Err(_) => {
4135 reservation.release_and_disarm();
4136 if let Some(opened) = breaker.record_timeout(
4140 self.route_bind_breaker_threshold,
4141 self.route_bind_breaker_cooldown,
4142 ) {
4143 warn!(
4144 module_id = %target_module_id,
4145 consecutive_timeouts = opened.consecutive_timeouts,
4146 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4147 reopened_after_probe = opened.reopened_after_probe,
4148 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4149 );
4150 }
4151 tracing::warn!(
4159 module_id = %target_module_id,
4160 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4161 "route.bind relay timed out: module did not ack within budget"
4162 );
4163 Ok(vec![self.route_open_refusal_frame(
4164 ctx,
4165 &frame,
4166 &target_module_id,
4167 "relay_timed_out",
4168 "module_timeout",
4169 format!(
4170 "module_id '{target_module_id}' did not answer route.bind within {:?}",
4171 route_bind_relay_timeout
4172 ),
4173 )?])
4174 }
4175 }
4176 }
4177
4178 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4179 let response = ClientControlResponse::SupervisorSpawnSnapshot {
4180 snapshot: self.supervisor.spawn_snapshot(),
4181 };
4182 Ok(vec![control_response_body_frame(
4183 &frame,
4184 &response,
4185 "ClientControlResponse::SupervisorSpawnSnapshot",
4186 )?])
4187 }
4188
4189 fn handle_supervisor_spawn_subscribe(
4190 &self,
4191 ctx: &RouteCtx,
4192 frame: Frame,
4193 since: Option<SpawnCursor>,
4194 ) -> Result<Vec<Frame>, RouterError> {
4195 match self.supervisor.subscribe_spawns(
4196 ctx.connection_id,
4197 frame.header.corr,
4198 response_version(&frame),
4199 since,
4200 ctx.egress.clone(),
4201 ) {
4202 Ok(()) => Ok(Vec::new()),
4203 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4204 Ok(vec![control_error_body_frame(
4205 &frame,
4206 ErrorBody {
4207 code: "spawn_cursor_incarnation_mismatch".to_string(),
4208 message: "spawn cursor belongs to a different daemon incarnation"
4209 .to_string(),
4210 detail: Some(serde_json::json!({
4211 "current_daemon_incarnation": current
4212 })),
4213 },
4214 )?])
4215 }
4216 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4217 &frame,
4218 ErrorBody {
4219 code: "spawn_cursor_too_old".to_string(),
4220 message: "spawn cursor predates the retained event ring".to_string(),
4221 detail: Some(serde_json::json!({
4222 "oldest_retained_cursor": oldest
4223 })),
4224 },
4225 )?]),
4226 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4227 0,
4228 frame.header.corr,
4229 format!("failed to open supervisor spawn subscription: {error}"),
4230 )),
4231 }
4232 }
4233
4234 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4235 let generation = self
4236 .registry
4237 .generation()
4238 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4239 let mut modules = Vec::new();
4240 for module in self.supervisor.list() {
4241 let status = module.status_for_control("list").map_err(|err| {
4242 RouterError::backend(
4243 0,
4244 frame.header.corr,
4245 format!("failed to read supervisor status: {err}"),
4246 )
4247 })?;
4248 let (configured, _) = module.configuration().map_err(|err| {
4249 RouterError::backend(
4250 0,
4251 frame.header.corr,
4252 format!("failed to read module configuration: {err}"),
4253 )
4254 })?;
4255 let image = module.running_image_agreement().await;
4257 let resources = Some(module.child_resource_usage());
4260 let pending_reload = Some(reload_verdict(
4261 &configured.program,
4262 status.spawned_from.as_deref(),
4263 image,
4264 ));
4265 modules.push(
4268 SupervisorEntry::new(
4269 status.module_id,
4270 status.state.to_string(),
4271 status.enabled,
4272 status.live,
4273 status.health.status,
4274 )
4275 .with_launch_nonce_env(Some(!cfg!(unix)))
4276 .with_launch_nonce_source(status.launch_nonce_source)
4277 .with_protocol(status.protocol)
4278 .with_pending_reload(pending_reload)
4279 .with_last_probe_ms(status.health.last_probe_ms)
4280 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
4281 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
4282 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
4283 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
4284 .with_restart_count(Some(status.restart_count))
4285 .with_max_restarts(Some(status.max_restarts))
4286 .with_lifetime_restarts(Some(status.lifetime_restarts))
4287 .with_spawn_generation(Some(status.spawn_generation))
4288 .with_restart_window_secs(Some(status.restart_window.as_secs()))
4289 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
4290 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
4291 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
4292 .with_resources(resources),
4293 );
4294 }
4295 let response = ClientControlResponse::SupervisorList {
4296 generation,
4297 modules,
4298 };
4299 Ok(vec![control_response_body_frame(
4300 &frame,
4301 &response,
4302 "ClientControlResponse::SupervisorList",
4303 )?])
4304 }
4305
4306 fn handle_supervisor_stderr_tail(
4307 &self,
4308 frame: Frame,
4309 module_id: String,
4310 max_lines: Option<u32>,
4311 max_bytes: Option<u32>,
4312 ) -> Result<Vec<Frame>, RouterError> {
4313 let Some(module) = self.supervisor.get(&module_id) else {
4314 return Ok(vec![control_error_frame(
4315 &frame,
4316 "unknown_module",
4317 format!("module_id '{module_id}' is not supervised"),
4318 )?]);
4319 };
4320
4321 let snapshot = module.stderr_tail(
4322 max_lines.map(|value| value as usize),
4323 max_bytes.map(|value| value as usize),
4324 );
4325
4326 let response = ClientControlResponse::SupervisorStderrTail {
4327 module_id,
4328 tail: StderrTail {
4329 capture: match snapshot.capture {
4330 CaptureState::Captured => StderrCaptureState::Captured,
4331 CaptureState::Incomplete { reason } => {
4332 StderrCaptureState::Incomplete { reason }
4333 }
4334 CaptureState::NotCaptured { reason } => {
4335 StderrCaptureState::NotCaptured { reason }
4336 }
4337 },
4338 entries: snapshot
4339 .entries
4340 .into_iter()
4341 .map(|entry| match entry {
4342 TailEntry::Line {
4343 text,
4344 truncated,
4345 at_ms,
4346 } => StderrTailEntry::Line {
4347 text,
4348 truncated,
4349 at_ms,
4350 },
4351 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
4352 })
4353 .collect(),
4354 dropped_lines: snapshot.dropped_lines,
4355 },
4356 };
4357 Ok(vec![control_response_body_frame(
4358 &frame,
4359 &response,
4360 "ClientControlResponse::SupervisorStderrTail",
4361 )?])
4362 }
4363
4364 async fn handle_supervisor_terminals(
4365 &self,
4366 frame: Frame,
4367 module_id: String,
4368 ) -> Result<Vec<Frame>, RouterError> {
4369 let Some(module) = self.supervisor.get(&module_id) else {
4370 return Ok(vec![control_error_frame(
4371 &frame,
4372 "unknown_module",
4373 format!("module_id '{module_id}' is not supervised"),
4374 )?]);
4375 };
4376
4377 let terminals = module
4380 .read_durable_terminal_history()
4381 .await
4382 .map_err(|error| {
4383 RouterError::backend(
4384 0,
4385 frame.header.corr,
4386 format!("failed to read terminal history: {error}"),
4387 )
4388 })?;
4389 let response = ClientControlResponse::SupervisorTerminals {
4390 module_id,
4391 terminals,
4392 };
4393 Ok(vec![control_response_body_frame(
4394 &frame,
4395 &response,
4396 "ClientControlResponse::SupervisorTerminals",
4397 )?])
4398 }
4399
4400 fn handle_supervisor_routes(
4401 &self,
4402 frame: Frame,
4403 module_id: Option<String>,
4404 ) -> Result<Vec<Frame>, RouterError> {
4405 let modules = self
4406 .forwarding
4407 .route_census(module_id.as_deref())
4408 .map_err(RouterError::Forwarding)?
4409 .into_iter()
4410 .map(|(module_id, routes)| SupervisorRouteModule {
4411 module_id,
4412 routes: routes
4413 .into_iter()
4414 .map(|route| SupervisorRoute {
4415 consumer: match route.principal {
4416 Principal::Reserved { module_id } => {
4417 SupervisorRouteConsumer::Reserved { module_id }
4418 }
4419 Principal::Direct | Principal::Unverified => {
4420 SupervisorRouteConsumer::Direct {
4421 connection_id: route.goodbye_target.connection_id.get(),
4422 }
4423 }
4424 },
4425 age_ms: Instant::now()
4426 .saturating_duration_since(route.bound_at)
4427 .as_millis()
4428 .try_into()
4429 .unwrap_or(u64::MAX),
4430 draining: route.draining,
4431 drain_reason: route.drain_reason,
4432 })
4433 .collect(),
4434 })
4435 .collect();
4436 let response = ClientControlResponse::SupervisorRoutes { modules };
4437 Ok(vec![control_response_body_frame(
4438 &frame,
4439 &response,
4440 "ClientControlResponse::SupervisorRoutes",
4441 )?])
4442 }
4443
4444 async fn handle_supervisor_provenance(
4445 &self,
4446 frame: Frame,
4447 module_id: Option<String>,
4448 ) -> Result<Vec<Frame>, RouterError> {
4449 let mut selected = if let Some(module_id) = module_id {
4450 let Some(module) = self.supervisor.get(&module_id) else {
4451 return Ok(vec![control_error_frame(
4452 &frame,
4453 "unknown_module",
4454 format!("module_id '{module_id}' is not supervised"),
4455 )?]);
4456 };
4457 vec![module]
4458 } else {
4459 self.supervisor.list()
4460 };
4461
4462 let mut modules = Vec::with_capacity(selected.len());
4463 for module in selected.drain(..) {
4464 let (status, observed_image) = module
4465 .status_and_running_image_agreement()
4466 .await
4467 .map_err(|err| {
4468 RouterError::backend(
4469 0,
4470 frame.header.corr,
4471 format!("failed to read supervisor status: {err}"),
4472 )
4473 })?;
4474 let module_declared = self
4475 .registry
4476 .get_module(&status.module_id)
4477 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4478 .and_then(|registration| registration.manifest.provenance)
4479 .map(|build| ModuleDeclaredProvenance::Reported { build })
4480 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4481 #[cfg(test)]
4482 let running_image = match &self.provenance_probe_override {
4483 Some(result) => result.clone(),
4484 None => observed_image,
4485 };
4486 #[cfg(not(test))]
4487 let running_image = observed_image;
4488 modules.push(SupervisorModuleProvenance {
4489 module_id: status.module_id,
4490 module_declared,
4491 daemon_observed: SupervisorObservedProcess {
4492 launch_nonce_source: status.launch_nonce_source,
4493 pid: status.pid,
4494 spawned_at_ms: status.spawned_at_ms,
4495 spawned_from: status.spawned_from,
4496 running_image,
4497 },
4498 });
4499 }
4500 let daemon = SupervisorDaemonProvenance {
4501 daemon_build: self.daemon_provenance.build.clone(),
4502 daemon_observed: DaemonObservedProcess {
4503 pid: self.daemon_provenance.pid,
4504 started_at_ms: self
4505 .daemon_provenance
4506 .start_clock
4507 .map(|clock| clock.started_at_ms())
4508 .or(self.daemon_provenance.started_at_ms),
4509 running_image: self
4510 .daemon_provenance
4511 .probe
4512 .observe(
4513 self.daemon_provenance.pid,
4514 self.daemon_provenance.executable_path.as_deref(),
4515 self.daemon_provenance.executable_identity.clone(),
4516 self.daemon_provenance.process_start_time,
4517 )
4518 .await,
4519 },
4520 };
4521 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
4522 Ok(vec![control_response_body_frame(
4523 &frame,
4524 &response,
4525 "ClientControlResponse::SupervisorProvenance",
4526 )?])
4527 }
4528
4529 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4530 self.refresh_capability_requirements();
4531 let generation = self
4532 .registry
4533 .generation()
4534 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4535 let modules = self
4536 .supervisor
4537 .list()
4538 .into_iter()
4539 .map(|module| {
4540 let status = module.status_for_control("health").map_err(|err| {
4541 RouterError::backend(
4542 0,
4543 frame.header.corr,
4544 format!("failed to read supervisor health: {err}"),
4545 )
4546 })?;
4547 let module_id = status.module_id;
4548 let capability_detail = self
4549 .capability_evaluator
4550 .required_problem_detail(&module_id);
4551 Ok(SupervisorHealthEntry {
4552 module_id,
4553 status: status.health.status,
4554 detail: append_capability_problem_detail(
4555 status.health.detail,
4556 capability_detail,
4557 ),
4558 metrics: status.health.metrics,
4559 consecutive_failures: status.health.consecutive_failures,
4560 late_answer_count: status.health.late_answer_count,
4561 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4562 last_action: status.health.last_action,
4563 last_action_ms: status.health.last_action_ms,
4564 last_probe_ms: status.health.last_probe_ms,
4565 })
4566 })
4567 .collect::<Result<Vec<_>, RouterError>>()?;
4568 let response = ClientControlResponse::SupervisorHealth {
4569 generation,
4570 modules,
4571 };
4572 Ok(vec![control_response_body_frame(
4573 &frame,
4574 &response,
4575 "ClientControlResponse::SupervisorHealth",
4576 )?])
4577 }
4578
4579 async fn handle_supervisor_restart(
4580 &self,
4581 frame: Frame,
4582 module_id: String,
4583 drain_timeout_ms: Option<u64>,
4584 ) -> Result<Vec<Frame>, RouterError> {
4585 let operation_lock = self.supervisor.operation_lock();
4586 let _operation_guard = operation_lock.lock().await;
4587 let Some(module) = self.supervisor.get(&module_id) else {
4588 return Ok(vec![control_error_frame(
4589 &frame,
4590 "unknown_module",
4591 format!("module_id '{module_id}' is not supervised"),
4592 )?]);
4593 };
4594
4595 self.route_outages.mark_operator_action(&module_id);
4596 if let Err(err) = module.restart(drain_timeout_ms).await {
4597 self.route_outages
4598 .operator_action_ended_unrefused(&module_id);
4599 let (code, message) = match err {
4600 crate::supervise::SuperviseError::Disabled { .. } => {
4601 ("module_disabled", err.to_string())
4602 }
4603 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4604 ("swap_in_progress", err.to_string())
4605 }
4606 _ => (
4607 "target_unavailable",
4608 format!("failed to restart module_id '{module_id}': {err}"),
4609 ),
4610 };
4611 return Ok(vec![control_error_frame(&frame, code, message)?]);
4612 }
4613
4614 let response = ClientControlResponse::SupervisorAck {
4615 module_id,
4616 applied: true,
4617 };
4618 Ok(vec![control_response_body_frame(
4619 &frame,
4620 &response,
4621 "ClientControlResponse::SupervisorAck",
4622 )?])
4623 }
4624
4625 async fn handle_supervisor_swap(
4629 &self,
4630 frame: Frame,
4631 module_id: String,
4632 ready_timeout_ms: Option<u64>,
4633 ) -> Result<Vec<Frame>, RouterError> {
4634 let module = {
4641 let operation_lock = self.supervisor.operation_lock();
4642 let _operation_guard = operation_lock.lock().await;
4643 self.supervisor.get(&module_id)
4644 };
4645 let Some(module) = module else {
4646 return Ok(vec![control_error_frame(
4647 &frame,
4648 "unknown_module",
4649 format!("module_id '{module_id}' is not supervised"),
4650 )?]);
4651 };
4652
4653 self.route_outages.mark_operator_action(&module_id);
4654 if let Err(err) = module
4655 .swap(ready_timeout_ms.map(Duration::from_millis))
4656 .await
4657 {
4658 self.route_outages
4659 .operator_action_ended_unrefused(&module_id);
4660 use crate::supervise::SuperviseError;
4661 let message = err.to_string();
4662 let error = match err {
4663 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
4664 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
4665 code: "swap_refused".to_string(),
4666 message,
4667 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
4668 },
4669 SuperviseError::SwapFailed {
4670 arm,
4671 candidate_exit,
4672 ..
4673 } => ErrorBody {
4674 code: "swap_failed".to_string(),
4675 message,
4676 detail: Some(serde_json::json!({
4677 "arm": arm.as_str(),
4678 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
4679 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
4680 })),
4681 },
4682 _ => ErrorBody::new(
4683 "target_unavailable",
4684 format!("failed to swap module_id '{module_id}': {message}"),
4685 ),
4686 };
4687 return Ok(vec![control_error_body_frame(&frame, error)?]);
4688 }
4689 self.route_outages
4693 .operator_action_ended_unrefused(&module_id);
4694
4695 let response = ClientControlResponse::SupervisorAck {
4696 module_id,
4697 applied: true,
4698 };
4699 Ok(vec![control_response_body_frame(
4700 &frame,
4701 &response,
4702 "ClientControlResponse::SupervisorAck",
4703 )?])
4704 }
4705
4706 async fn handle_supervisor_reload(
4707 &self,
4708 frame: Frame,
4709 module_id: String,
4710 ) -> Result<Vec<Frame>, RouterError> {
4711 let operation_lock = self.supervisor.operation_lock();
4712 let _operation_guard = operation_lock.lock().await;
4713 let Some(module) = self.supervisor.get(&module_id) else {
4714 return Ok(vec![control_error_frame(
4715 &frame,
4716 "unknown_module",
4717 format!("module_id '{module_id}' is not supervised"),
4718 )?]);
4719 };
4720
4721 self.route_outages.mark_operator_action(&module_id);
4722 if let Err(err) = module.reload().await {
4723 self.route_outages
4724 .operator_action_ended_unrefused(&module_id);
4725 let (code, message) = match err {
4726 crate::supervise::SuperviseError::Disabled { .. } => {
4727 ("module_disabled", err.to_string())
4728 }
4729 crate::supervise::SuperviseError::SwapInProgress { .. } => {
4730 ("swap_in_progress", err.to_string())
4731 }
4732 _ => (
4733 "reload_failed",
4734 format!("failed to reload module_id '{module_id}': {err}"),
4735 ),
4736 };
4737 return Ok(vec![control_error_frame(&frame, code, message)?]);
4738 }
4739
4740 let response = ClientControlResponse::SupervisorAck {
4741 module_id,
4742 applied: true,
4743 };
4744 Ok(vec![control_response_body_frame(
4745 &frame,
4746 &response,
4747 "ClientControlResponse::SupervisorAck",
4748 )?])
4749 }
4750
4751 async fn handle_supervisor_rescan(
4752 &self,
4753 frame: Frame,
4754 preview: bool,
4755 ) -> Result<Vec<Frame>, RouterError> {
4756 let Some(context) = self.rescan.clone() else {
4757 return Ok(vec![control_error_frame(
4758 &frame,
4759 "rescan_unavailable",
4760 "the daemon was not started with a reloadable config path".to_string(),
4761 )?]);
4762 };
4763
4764 let operation_lock = self.supervisor.operation_lock();
4765 let _operation_guard = operation_lock.lock().await;
4766 let loaded = match crate::daemon_config::load(&context.config_path) {
4767 Ok(config) => config,
4768 Err(err) => {
4769 return Ok(vec![control_error_frame(
4770 &frame,
4771 "invalid_daemon_config",
4772 format!("supervisor rescan rejected daemon config: {err}"),
4773 )?])
4774 }
4775 };
4776 let Some(config) = loaded else {
4784 return Ok(vec![control_error_frame(
4785 &frame,
4786 "invalid_daemon_config",
4787 format!(
4788 "daemon config not found at {}; refusing to rescan (an absent config would \
4789 retire every supervised module)",
4790 context.config_path.display()
4791 ),
4792 )?]);
4793 };
4794 let (
4795 configured_port,
4796 storage_config,
4797 admission_facts_carrier_module_id,
4798 admission_facts_targets,
4799 scope_authority_owners,
4800 modules,
4801 reserved_capabilities,
4802 ) = (
4803 config.port,
4804 config.storage,
4805 config.admission_facts_carrier_module_id,
4806 config.admission_facts_targets,
4807 config.scope_authority_owners,
4808 config.modules,
4809 config.reserved_capabilities,
4810 );
4811
4812 let mut restart_required = Vec::new();
4821 for section in RestartRequiredSection::ALL {
4822 let changed = match section {
4823 RestartRequiredSection::Port => configured_port != context.configured_port,
4824 RestartRequiredSection::Storage => storage_config != context.storage_config,
4825 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
4826 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
4827 }
4828 RestartRequiredSection::AdmissionFactsTargets => {
4829 admission_facts_targets != context.admission_facts_targets
4830 }
4831 RestartRequiredSection::ScopeAuthorityOwners => {
4832 scope_authority_owners != context.scope_authority_owners
4833 }
4834 };
4835 if changed {
4836 restart_required.push(section.label().to_string());
4837 }
4838 }
4839 if !restart_required.is_empty() {
4840 warn!(
4841 config_path = %context.config_path.display(),
4842 sections = %restart_required.join(", "),
4843 "daemon config changed outside the modules section; restart the daemon to apply those changes"
4844 );
4845 }
4846
4847 for configured in &modules {
4848 if let Err(err) = validate_spec(&configured.module_spec()) {
4849 return Ok(vec![control_error_frame(
4850 &frame,
4851 "invalid_daemon_config",
4852 format!("supervisor rescan rejected daemon config: {err}"),
4853 )?]);
4854 }
4855 }
4856
4857 let configured_capabilities = modules
4858 .iter()
4859 .map(|module| (module.module_id.clone(), module.enabled))
4860 .collect::<Vec<_>>();
4861 let preview_capability_warnings = if preview {
4862 let (_, registrations) = self.runtime_capability_snapshot()?;
4863 let current_modules = self
4864 .supervisor
4865 .list()
4866 .into_iter()
4867 .map(|module| module.module_id().to_string())
4868 .collect::<BTreeSet<_>>();
4869 let resulting_modules = configured_capabilities.clone();
4870 let removed = current_modules
4871 .into_iter()
4872 .filter(|module_id| {
4873 !resulting_modules
4874 .iter()
4875 .any(|(configured_id, _)| configured_id == module_id)
4876 })
4877 .collect::<Vec<_>>();
4878 self.capability_evaluator.preview_removal_warnings(
4879 resulting_modules,
4880 &removed,
4881 ®istrations,
4882 )
4883 } else {
4884 Vec::new()
4885 };
4886 let result = match self
4887 .reconcile_supervised_modules(&context.supervisor, modules, preview)
4888 .await
4889 {
4890 Ok(result) => result,
4891 Err(message) => {
4892 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
4893 }
4894 };
4895 if !preview {
4896 self.capability_evaluator
4897 .configure(configured_capabilities, reserved_capabilities);
4898 self.capability_evaluator.wake_deadline_loop();
4899 self.refresh_capability_requirements();
4900 }
4901 let mut result = result;
4902 result.restart_required = restart_required;
4903 result.capability_warnings = preview_capability_warnings;
4904 let response = ClientControlResponse::SupervisorRescan { result };
4905 Ok(vec![control_response_body_frame(
4906 &frame,
4907 &response,
4908 "ClientControlResponse::SupervisorRescan",
4909 )?])
4910 }
4911
4912 async fn handle_supervisor_release_reserved(
4913 &self,
4914 frame: Frame,
4915 module_id: String,
4916 ) -> Result<Vec<Frame>, RouterError> {
4917 let Some(context) = self.rescan.clone() else {
4918 return Ok(vec![control_error_frame(
4919 &frame,
4920 "release_unavailable",
4921 "reserved-id release requires a daemon started with a reloadable config path",
4922 )?]);
4923 };
4924 let operation_lock = self.supervisor.operation_lock();
4925 let _operation_guard = operation_lock.lock().await;
4926 let loaded = match crate::daemon_config::load(&context.config_path) {
4927 Ok(Some(config)) => config,
4928 Ok(None) => {
4929 return Ok(vec![control_error_frame(
4930 &frame,
4931 "invalid_daemon_config",
4932 format!(
4933 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
4934 context.config_path.display()
4935 ),
4936 )?])
4937 }
4938 Err(err) => {
4939 return Ok(vec![control_error_frame(
4940 &frame,
4941 "invalid_daemon_config",
4942 format!("unable to verify reserved-id release against daemon config: {err}"),
4943 )?])
4944 }
4945 };
4946 if loaded
4947 .modules
4948 .iter()
4949 .any(|configured| configured.module_id == module_id)
4950 {
4951 return Ok(vec![control_error_frame(
4952 &frame,
4953 "reserved_module_configured",
4954 format!(
4955 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
4956 ),
4957 )?]);
4958 }
4959 if !self.supervisor.release_retained_reserved_gate(&module_id) {
4960 return Ok(vec![control_error_frame(
4961 &frame,
4962 "reserved_gate_not_retained",
4963 format!(
4964 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
4965 ),
4966 )?]);
4967 }
4968
4969 let response = ClientControlResponse::SupervisorAck {
4970 module_id,
4971 applied: true,
4972 };
4973 Ok(vec![control_response_body_frame(
4974 &frame,
4975 &response,
4976 "ClientControlResponse::SupervisorAck",
4977 )?])
4978 }
4979
4980 async fn reconcile_supervised_modules(
4989 &self,
4990 supervisor: &Supervisor,
4991 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
4992 preview: bool,
4993 ) -> Result<SupervisorRescanResult, String> {
4994 let mut current = BTreeMap::new();
4995 for module in self.supervisor.list() {
4996 let (spec, health) = module.configuration().map_err(|err| {
4997 format!(
4998 "failed to read configuration for module_id '{}': {err}",
4999 module.module_id()
5000 )
5001 })?;
5002 let enabled = module
5003 .status()
5004 .map_err(|err| {
5005 format!(
5006 "failed to read status for module_id '{}': {err}",
5007 module.module_id()
5008 )
5009 })?
5010 .enabled;
5011 current.insert(
5012 module.module_id().to_string(),
5013 (module, spec, health, enabled),
5014 );
5015 }
5016 let configured = configured_modules
5017 .into_iter()
5018 .map(|module| (module.module_id.clone(), module))
5019 .collect::<BTreeMap<_, _>>();
5020
5021 let added = configured
5022 .keys()
5023 .filter(|module_id| !current.contains_key(*module_id))
5024 .cloned()
5025 .collect::<Vec<_>>();
5026 let removed = current
5027 .keys()
5028 .filter(|module_id| !configured.contains_key(*module_id))
5029 .cloned()
5030 .collect::<Vec<_>>();
5031 let mut changed_pending_reload = Vec::new();
5032 let mut configuration_changes = BTreeSet::new();
5033 let mut enabled_changes = BTreeSet::new();
5034 let mut unchanged = 0_u32;
5035
5036 for (module_id, configured_module) in &configured {
5037 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5038 else {
5039 continue;
5040 };
5041 let launch_changed = *current_spec != configured_module.module_spec();
5045 let configuration_changed =
5046 launch_changed || *current_health != configured_module.health;
5047 let enabled_changed = *current_enabled != configured_module.enabled;
5048 if configuration_changed {
5049 configuration_changes.insert(module_id.clone());
5050 }
5051 if launch_changed {
5052 changed_pending_reload.push(module_id.clone());
5053 }
5054 if enabled_changed {
5055 enabled_changes.insert(module_id.clone());
5056 }
5057 if !configuration_changed && !enabled_changed {
5058 unchanged = unchanged.saturating_add(1);
5059 }
5060 }
5061
5062 if preview {
5068 return Ok(SupervisorRescanResult {
5069 added,
5070 removed,
5071 changed_pending_reload,
5072 enabled_changes: enabled_changes.iter().cloned().collect(),
5073 unchanged,
5074 preview: true,
5075 restart_required: Vec::new(),
5079 capability_warnings: Vec::new(),
5080 });
5081 }
5082
5083 for module_id in &removed {
5084 let module = ¤t
5085 .get(module_id)
5086 .expect("removed module came from current supervisor state")
5087 .0;
5088 module.retire().await.map_err(|err| {
5089 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5090 })?;
5091 self.supervisor.record_rescan_removal(module_id);
5125 self.supervisor.retire(module_id);
5126 self.route_outages.forget(module_id);
5127 }
5128
5129 for module_id in configured.keys() {
5130 let Some((module, _, _, _)) = current.get(module_id) else {
5131 continue;
5132 };
5133 let configured_module = configured
5134 .get(module_id)
5135 .expect("configured module id came from configured map");
5136 if configuration_changes.contains(module_id) {
5137 module
5138 .update_configuration(
5139 configured_module.module_spec(),
5140 configured_module.health.clone(),
5141 configured_module.drain_timeout_ms,
5142 )
5143 .await
5144 .map_err(|err| {
5145 format!(
5146 "failed to update module_id '{module_id}' configuration during rescan: {err}"
5147 )
5148 })?;
5149 }
5150 if enabled_changes.contains(module_id) {
5151 self.route_outages.mark_operator_action(module_id);
5154 module
5155 .set_enabled(configured_module.enabled)
5156 .await
5157 .map_err(|err| {
5158 self.route_outages.operator_action_ended_unrefused(module_id);
5159 format!(
5160 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5161 configured_module.enabled
5162 )
5163 })?;
5164 }
5165 }
5166
5167 for module_id in &added {
5168 let configured_module = configured
5169 .get(module_id)
5170 .expect("added module id came from configured map");
5171 supervisor
5172 .supervise_configured_with_health(
5173 configured_module.module_spec(),
5174 configured_module.enabled,
5175 configured_module.health.clone(),
5176 configured_module.drain_timeout_ms,
5177 configured_module.restart,
5178 )
5179 .map_err(|err| {
5180 format!("failed to add module_id '{module_id}' during rescan: {err}")
5181 })?;
5182 }
5183
5184 Ok(SupervisorRescanResult {
5185 added,
5186 removed,
5187 changed_pending_reload,
5188 enabled_changes: enabled_changes.iter().cloned().collect(),
5189 unchanged,
5190 preview: false,
5191 restart_required: Vec::new(),
5194 capability_warnings: Vec::new(),
5195 })
5196 }
5197
5198 async fn handle_supervisor_set_enabled(
5199 &self,
5200 frame: Frame,
5201 module_id: String,
5202 enabled: bool,
5203 ) -> Result<Vec<Frame>, RouterError> {
5204 let operation_lock = self.supervisor.operation_lock();
5205 let _operation_guard = operation_lock.lock().await;
5206 let Some(module) = self.supervisor.get(&module_id) else {
5207 return Ok(vec![control_error_frame(
5208 &frame,
5209 "unknown_module",
5210 format!("module_id '{module_id}' is not supervised"),
5211 )?]);
5212 };
5213
5214 self.route_outages.mark_operator_action(&module_id);
5217 let applied = match module.set_enabled(enabled).await {
5218 Ok(applied) => applied,
5219 Err(err) => {
5220 self.route_outages
5221 .operator_action_ended_unrefused(&module_id);
5222 return Ok(vec![control_error_frame(
5223 &frame,
5224 "target_unavailable",
5225 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
5226 )?]);
5227 }
5228 };
5229 if !applied {
5230 self.route_outages
5233 .operator_action_ended_unrefused(&module_id);
5234 }
5235
5236 self.capability_evaluator.wake_deadline_loop();
5237 self.refresh_capability_requirements();
5238 let response = ClientControlResponse::SupervisorAck { module_id, applied };
5239 Ok(vec![control_response_body_frame(
5240 &frame,
5241 &response,
5242 "ClientControlResponse::SupervisorAck",
5243 )?])
5244 }
5245
5246 async fn handle_supervisor_health_probe(
5247 &self,
5248 frame: Frame,
5249 module_id: String,
5250 ) -> Result<Vec<Frame>, RouterError> {
5251 self.refresh_capability_requirements();
5252 let Some(registration) = self
5253 .registry
5254 .get_module(&module_id)
5255 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5256 else {
5257 return Ok(vec![control_error_frame(
5258 &frame,
5259 "unknown_module",
5260 format!("module_id '{module_id}' is not registered"),
5261 )?]);
5262 };
5263
5264 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
5277 {
5278 return Ok(vec![control_error_frame(
5279 &frame,
5280 "health_not_advertised",
5281 format!("module_id '{module_id}' did not advertise health.check"),
5282 )?]);
5283 }
5284
5285 let deadline = Instant::now() + self.health_probe_timeout;
5286 let pending = match self.forwarding.begin_module_control_rpc_for(
5287 &module_id,
5288 MODULE_CONTROL_OP_HEALTH_CHECK,
5289 deadline,
5290 ) {
5291 Ok(pending) => pending,
5292 Err(err) => {
5293 return Ok(vec![control_error_frame(
5294 &frame,
5295 forwarding_error_code(&err),
5296 err.to_string(),
5297 )?])
5298 }
5299 };
5300
5301 let PendingModuleControlRpc {
5302 endpoint,
5303 module_sink,
5304 negotiated_ver,
5305 corr: probe_corr,
5306 receiver,
5307 } = pending;
5308 let mut guard =
5309 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
5310 let probe_body =
5311 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5312 RouterError::backend(
5313 0,
5314 frame.header.corr,
5315 format!("failed to encode health.check request: {err}"),
5316 )
5317 })?;
5318 let probe_frame = Frame::build_with_version(
5319 negotiated_ver,
5320 FrameType::Request,
5321 control_flags(),
5322 0,
5323 0,
5324 probe_corr,
5325 probe_body,
5326 )
5327 .map_err(RouterError::FrameBuild)?;
5328
5329 if let Err(err) = module_sink.send(probe_frame).await {
5330 return Ok(vec![control_error_frame(
5331 &frame,
5332 "target_unavailable",
5333 err.to_string(),
5334 )?]);
5335 }
5336
5337 match timeout_at(deadline, receiver).await {
5338 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5339 guard.disarm();
5340 let Some(report) = response.health_report() else {
5341 return Ok(vec![control_error_frame(
5342 &frame,
5343 "invalid_control_body",
5344 "health.check RPC returned a non-health response",
5345 )?]);
5346 };
5347 let HealthReport {
5352 status,
5353 detail,
5354 metrics,
5355 } = report;
5356 let capability_detail = self
5357 .capability_evaluator
5358 .required_problem_detail(&module_id);
5359 let response = ClientControlResponse::SupervisorHealthProbe {
5360 module_id,
5361 status,
5362 detail: append_capability_problem_detail(detail, capability_detail),
5363 metrics,
5364 };
5365 Ok(vec![control_response_body_frame(
5366 &frame,
5367 &response,
5368 "ClientControlResponse::SupervisorHealthProbe",
5369 )?])
5370 }
5371 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
5372 guard.disarm();
5373 Ok(vec![control_error_body_frame(&frame, body)?])
5374 }
5375 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5376 guard.disarm();
5377 Ok(vec![control_error_frame(
5378 &frame,
5379 "target_unavailable",
5380 message,
5381 )?])
5382 }
5383 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5384 guard.disarm();
5385 Ok(vec![control_error_frame(
5386 &frame,
5387 "invalid_control_body",
5388 message,
5389 )?])
5390 }
5391 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5392 guard.disarm();
5393 Ok(vec![control_error_frame(
5394 &frame,
5395 "invalid_control_body",
5396 format!("expected module-control op '{expected}', got '{actual}'"),
5397 )?])
5398 }
5399 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
5400 guard.disarm();
5401 Ok(vec![control_error_frame(
5402 &frame,
5403 "module_timeout",
5404 format!(
5405 "module_id '{module_id}' answered health.check after {:?}",
5406 self.health_probe_timeout
5407 ),
5408 )?])
5409 }
5410 Ok(Err(_)) => Ok(vec![control_error_frame(
5411 &frame,
5412 "target_unavailable",
5413 "health.check waiter was canceled before the module responded",
5414 )?]),
5415 Err(_) => Ok(vec![control_error_frame(
5416 &frame,
5417 "module_timeout",
5418 format!(
5419 "module_id '{module_id}' did not answer health.check within {:?}",
5420 self.health_probe_timeout
5421 ),
5422 )?]),
5423 }
5424 }
5425
5426 fn supervisor_status(
5427 &self,
5428 module_id: &str,
5429 corr: u64,
5430 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5431 self.supervisor
5432 .get(module_id)
5433 .map(|module| {
5434 let warming = module.is_warming_for_control("status").map_err(|err| {
5435 RouterError::backend(
5436 0,
5437 corr,
5438 format!(
5439 "failed to read supervisor warming state for module_id '{module_id}': {err}"
5440 ),
5441 )
5442 })?;
5443 module.status_for_control("status").map_err(|err| {
5444 RouterError::backend(
5445 0,
5446 corr,
5447 format!(
5448 "failed to read supervisor status for module_id '{module_id}': {err}"
5449 ),
5450 )
5451 }).map(|status| (status, warming))
5452 })
5453 .transpose()
5454 }
5455
5456 fn guard_module_control_op(
5457 &self,
5458 frame: &Frame,
5459 module_id: &str,
5460 op: &str,
5461 ) -> Result<Option<Frame>, RouterError> {
5462 if self.module_grants_op(module_id, op, frame.header.corr)? {
5463 return Ok(None);
5464 }
5465
5466 Ok(Some(control_error_frame(
5467 frame,
5468 "op_not_allowed",
5469 format!("module_id '{module_id}' did not grant control op '{op}'"),
5470 )?))
5471 }
5472
5473 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
5474 let Some(registration) = self
5475 .registry
5476 .get_module(module_id)
5477 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
5478 else {
5479 return Ok(false);
5480 };
5481 Ok(module_registration_grants_op(®istration.control_ops, op))
5482 }
5483
5484 fn handle_status_update(
5485 &self,
5486 endpoint: ModuleEndpointId,
5487 frame: Frame,
5488 ) -> Result<Vec<Frame>, RouterError> {
5489 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
5490 Ok(update) => update,
5491 Err(err) => {
5492 if is_known_module_push_op(&frame.body) {
5497 return Ok(vec![control_error_frame(
5498 &frame,
5499 "invalid_control_body",
5500 format!("malformed module control push body: {err}"),
5501 )?]);
5502 }
5503 return Ok(Vec::new());
5504 }
5505 };
5506
5507 match update {
5508 ModuleControlPush::RouteStatus {
5509 route_channel,
5510 route_epoch,
5511 status,
5512 } => {
5513 self.forwarding
5514 .cache_status(endpoint, route_channel, route_epoch, status)
5515 .map_err(RouterError::Forwarding)?;
5516 }
5517 }
5518 Ok(Vec::new())
5519 }
5520
5521 fn handle_route_poll(
5522 &self,
5523 ctx: &RouteCtx,
5524 frame: Frame,
5525 route_channel: u16,
5526 route_epoch: u32,
5527 kind: PollKind,
5528 ) -> Result<Vec<Frame>, RouterError> {
5529 let snapshot = self
5530 .forwarding
5531 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
5532 .map_err(RouterError::Forwarding)?;
5533 let response = match (kind, snapshot) {
5534 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
5535 ClientControlResponse::RoutePoll {
5536 route_channel,
5537 route_epoch,
5538 status,
5539 live: None,
5540 }
5541 }
5542 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5543 route_channel,
5544 route_epoch,
5545 status: None,
5546 live: None,
5547 },
5548 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5549 let live = self
5575 .process_liveness
5576 .as_ref()
5577 .and_then(|source| source.process_live(&module_id))
5578 .unwrap_or(true);
5579 ClientControlResponse::RoutePoll {
5580 route_channel,
5581 route_epoch,
5582 status: None,
5583 live: Some(live),
5584 }
5585 }
5586 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5587 route_channel,
5588 route_epoch,
5589 status: None,
5590 live: Some(false),
5591 },
5592 };
5593
5594 Ok(vec![control_response_body_frame(
5595 &frame,
5596 &response,
5597 "ClientControlResponse::RoutePoll",
5598 )?])
5599 }
5600
5601 pub(crate) fn observe_module_control_completion(
5602 &self,
5603 completion: ModuleControlRpcCompletion,
5604 ) -> bool {
5605 match completion {
5606 ModuleControlRpcCompletion::Unknown => false,
5607 ModuleControlRpcCompletion::Settled => true,
5608 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
5609 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
5610 info!(
5611 module_id = %module_id,
5612 latency_ms,
5613 "late health.check answer proves the module is alive"
5614 );
5615 match self
5616 .supervisor
5617 .record_late_health_answer(&module_id, latency_ms)
5618 {
5619 Ok(true) => {}
5620 Ok(false) => debug!(
5621 module_id = %module_id,
5622 latency_ms,
5623 "late health.check answer has no active supervisor snapshot"
5624 ),
5625 Err(err) => warn!(
5626 module_id = %module_id,
5627 latency_ms,
5628 error = %err,
5629 "failed to record late health.check answer"
5630 ),
5631 }
5632 true
5633 }
5634 }
5635 }
5636
5637 fn refuse_to_end_module_connection_for_a_client(
5658 &self,
5659 module_connection_id: ConnectionId,
5660 corr: u64,
5661 err: ForwardingError,
5662 ) -> Result<(), RouterError> {
5663 if let ForwardingError::ConnectionClosing { connection_id } = err {
5664 if connection_id != module_connection_id {
5665 warn!(
5666 module_connection_id = module_connection_id.get(),
5667 client_connection_id = connection_id.get(),
5668 corr,
5669 "dropping a route.bind response for a closing client; the module connection keeps serving"
5670 );
5671 return Ok(());
5672 }
5673 }
5674 Err(RouterError::Forwarding(err))
5675 }
5676
5677 fn handle_module_relay_response(
5678 &self,
5679 connection_id: ConnectionId,
5680 frame: Frame,
5681 ) -> Result<Vec<Frame>, RouterError> {
5682 let mut secondary_error = None;
5683 let outcome = match frame.header.ty {
5684 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
5685 Ok(probe) if probe.op == "route.bind" => {
5686 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
5687 Ok(ModuleControlResponse::RouteBindAck {}) => {
5688 RouteBindRelayOutcome::Accepted
5689 }
5690 Ok(other) => {
5691 let message =
5692 format!("route.bind response carried unexpected body: {other:?}");
5693 secondary_error = Some(control_error_frame(
5694 &frame,
5695 "invalid_control_body",
5696 message.clone(),
5697 )?);
5698 RouteBindRelayOutcome::ModuleGone(message)
5699 }
5700 Err(err) => {
5701 let message = format!("malformed route.bind response body: {err}");
5702 secondary_error = Some(control_error_frame(
5703 &frame,
5704 "invalid_control_body",
5705 message.clone(),
5706 )?);
5707 RouteBindRelayOutcome::ModuleGone(message)
5708 }
5709 }
5710 }
5711 Ok(probe) => {
5712 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
5713 {
5714 Ok(response) => ModuleControlRpcOutcome::Response(response),
5715 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5716 "malformed {} response body: {err}",
5717 probe.op
5718 )),
5719 };
5720 let completion = self
5721 .forwarding
5722 .complete_module_control_rpc(
5723 connection_id,
5724 frame.header.corr,
5725 Some(&probe.op),
5726 outcome,
5727 )
5728 .map_err(RouterError::Forwarding)?;
5729 if !self.observe_module_control_completion(completion) {
5730 debug!(
5731 connection_id = connection_id.get(),
5732 corr = frame.header.corr,
5733 op = %probe.op,
5734 "dropping late or unknown module-control RPC response"
5735 );
5736 }
5737 return Ok(Vec::new());
5738 }
5739 Err(err) => {
5740 if let Some(expected_op) = self
5741 .forwarding
5742 .pending_module_control_op(connection_id, frame.header.corr)
5743 .map_err(RouterError::Forwarding)?
5744 {
5745 let completion = self
5746 .forwarding
5747 .complete_module_control_rpc(
5748 connection_id,
5749 frame.header.corr,
5750 None,
5751 ModuleControlRpcOutcome::MalformedResponse(format!(
5752 "malformed {expected_op} response body: {err}"
5753 )),
5754 )
5755 .map_err(RouterError::Forwarding)?;
5756 if !self.observe_module_control_completion(completion) {
5757 debug!(
5758 connection_id = connection_id.get(),
5759 corr = frame.header.corr,
5760 "dropping late malformed module-control RPC response"
5761 );
5762 }
5763 return Ok(Vec::new());
5764 }
5765 let message = format!("malformed route.bind response body: {err}");
5766 secondary_error = Some(control_error_frame(
5767 &frame,
5768 "invalid_control_body",
5769 message.clone(),
5770 )?);
5771 RouteBindRelayOutcome::ModuleGone(message)
5772 }
5773 },
5774 FrameType::Error => {
5775 if self
5776 .forwarding
5777 .pending_module_control_op(connection_id, frame.header.corr)
5778 .map_err(RouterError::Forwarding)?
5779 .is_some()
5780 {
5781 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
5782 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
5783 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
5784 "malformed module-control ERROR body: {err}"
5785 )),
5786 };
5787 let completion = self
5788 .forwarding
5789 .complete_module_control_rpc(
5790 connection_id,
5791 frame.header.corr,
5792 None,
5793 outcome,
5794 )
5795 .map_err(RouterError::Forwarding)?;
5796 if !self.observe_module_control_completion(completion) {
5797 debug!(
5798 connection_id = connection_id.get(),
5799 corr = frame.header.corr,
5800 "dropping late or unknown module-control RPC error"
5801 );
5802 }
5803 return Ok(Vec::new());
5804 }
5805 match serde_json::from_slice::<ErrorBody>(&frame.body) {
5806 Ok(body) => RouteBindRelayOutcome::Rejected(body),
5807 Err(err) => {
5808 let message = format!("malformed route.bind ERROR body: {err}");
5809 secondary_error = Some(control_error_frame(
5810 &frame,
5811 "invalid_control_body",
5812 message.clone(),
5813 )?);
5814 RouteBindRelayOutcome::ModuleGone(message)
5815 }
5816 }
5817 }
5818 ty => {
5819 return Ok(vec![control_error_frame(
5820 &frame,
5821 "unsupported_control_frame",
5822 format!("unsupported module channel-0 frame {ty:?}"),
5823 )?])
5824 }
5825 };
5826
5827 let settled =
5828 self.forwarding
5829 .complete_pending_relay(connection_id, frame.header.corr, outcome);
5830 let completion = match settled {
5831 Ok(completion) => completion,
5832 Err(err) => {
5833 self.refuse_to_end_module_connection_for_a_client(
5834 connection_id,
5835 frame.header.corr,
5836 err,
5837 )?;
5838 return Ok(secondary_error.into_iter().collect());
5839 }
5840 };
5841 if let Some(target) = completion.abandoned.as_ref() {
5842 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
5843 }
5844 if !completion.settled {
5845 debug!(
5846 connection_id = connection_id.get(),
5847 corr = frame.header.corr,
5848 frame_type = ?frame.header.ty,
5849 "dropping late or unknown route.bind relay response"
5850 );
5851 }
5852 Ok(secondary_error.into_iter().collect())
5853 }
5854
5855 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
5856 debug!(connection_id = connection_id.get(), "handling GOODBYE");
5857 self.cleanup_connection_with_end_reason(
5861 connection_id,
5862 RegistrationEndReason::ExplicitGoodbye,
5863 )
5864 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
5865 Ok(Vec::new())
5866 }
5867}
5868
5869impl Default for ControlHandler {
5870 fn default() -> Self {
5871 Self::new(Arc::new(Registry::default()))
5872 }
5873}
5874
5875impl crate::supervise::SwapPromotionObserver for ControlHandler {
5876 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
5877 self.apply_registration_capabilities(registration);
5878 }
5879}
5880
5881fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
5882 CapabilityRequirementStatus {
5883 consumer: status.consumer,
5884 capability: status.capability,
5885 need: match status.need {
5886 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
5887 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
5888 },
5889 verdict: status.verdict.as_str().to_string(),
5890 episode_seq: status.episode_seq,
5891 config_satisfiable: status.config_satisfiable,
5892 runtime_available: status.runtime_available,
5893 detail: status.detail,
5894 }
5895}
5896
5897fn append_capability_problem_detail(
5898 detail: Option<String>,
5899 capability_detail: Option<String>,
5900) -> Option<String> {
5901 match (detail, capability_detail) {
5902 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
5903 (Some(detail), None) => Some(detail),
5904 (None, Some(capability_detail)) => Some(capability_detail),
5905 (None, None) => None,
5906 }
5907}
5908
5909fn subc_ops() -> Vec<String> {
5910 SUBC_CONTROL_OPS
5911 .iter()
5912 .map(|op| (*op).to_string())
5913 .collect()
5914}
5915
5916fn module_subc_ops() -> Vec<String> {
5917 SUBC_CONTROL_OPS
5918 .iter()
5919 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
5920 .map(|op| (*op).to_string())
5921 .collect()
5922}
5923
5924#[cfg(test)]
5925fn module_baseline_control_ops() -> Vec<String> {
5926 MODULE_BASELINE_CONTROL_OPS
5927 .iter()
5928 .map(|op| (*op).to_string())
5929 .collect()
5930}
5931
5932fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
5933 let mut seen = HashSet::new();
5934 let mut effective = Vec::new();
5935 for op in MODULE_BASELINE_CONTROL_OPS {
5936 if seen.insert((*op).to_string()) {
5937 effective.push((*op).to_string());
5938 }
5939 }
5940 for op in declared.unwrap_or_default() {
5941 if seen.insert(op.clone()) {
5942 effective.push(op);
5943 }
5944 }
5945 effective
5946}
5947
5948fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
5949 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
5950}
5951
5952fn target_module_id(target: &RouteTarget) -> &str {
5953 match target {
5954 RouteTarget::ToolProvider { module_id }
5955 | RouteTarget::ManagementSurface { module_id }
5956 | RouteTarget::InternalService { module_id, .. } => module_id,
5957 }
5958}
5959
5960fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
5961 roles.iter().any(|role| match (target, role) {
5962 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
5963 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
5964 (
5965 RouteTarget::InternalService { service_id, .. },
5966 ProviderRole::InternalService {
5967 service_id: provided,
5968 ..
5969 },
5970 ) => service_id == provided,
5971 _ => false,
5972 })
5973}
5974
5975fn is_routable_role(role: &ProviderRole) -> bool {
5976 matches!(
5977 role,
5978 ProviderRole::ToolProvider { .. }
5979 | ProviderRole::ManagementSurface { .. }
5980 | ProviderRole::InternalService { .. }
5981 )
5982}
5983
5984#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5985enum ControlRequestBodyError {
5986 UnknownOp,
5987 InvalidBody,
5988}
5989
5990#[derive(Debug, Deserialize)]
5991struct ControlOpProbe {
5992 op: String,
5993}
5994
5995const MODULE_PUSH_OPS: &[&str] = &["route.status"];
5998
5999fn is_known_module_push_op(body: &[u8]) -> bool {
6000 serde_json::from_slice::<ControlOpProbe>(body)
6001 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6002 .unwrap_or(false)
6003}
6004
6005fn is_known_module_request_op(body: &[u8]) -> bool {
6006 serde_json::from_slice::<ControlOpProbe>(body)
6007 .map(|probe| is_module_to_subc_op(&probe.op))
6008 .unwrap_or(false)
6009}
6010
6011fn is_module_to_subc_op(op: &str) -> bool {
6012 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6013}
6014
6015fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6016 debug!(
6017 op = %op,
6018 connection_id = connection_id.get(),
6019 corr,
6020 "control dispatch"
6021 );
6022}
6023
6024fn log_slow_control_dispatch(
6025 dispatch_started_at: Option<StdInstant>,
6026 op: &'static str,
6027 connection_id: ConnectionId,
6028 corr: u64,
6029) {
6030 let Some(dispatch_started_at) = dispatch_started_at else {
6031 return;
6032 };
6033 let elapsed = dispatch_started_at.elapsed();
6034 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6035 warn!(
6036 op = %op,
6037 connection_id = connection_id.get(),
6038 corr,
6039 elapsed_ms = elapsed.as_millis() as u64,
6040 "slow control dispatch"
6041 );
6042 }
6043}
6044
6045fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6046 match request {
6047 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6048 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6049 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6050 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6051 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6052 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6053 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6054 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6055 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6056 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6057 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6058 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6059 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6060 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6061 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6062 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6063 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6064 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6065 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6066 }
6067}
6068
6069fn principal_label(principal: &Principal) -> String {
6070 match principal {
6071 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6072 Principal::Direct => "direct".to_string(),
6073 other => format!("{other:?}"),
6074 }
6075}
6076
6077fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6078 match request {
6079 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6080 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6081 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6082 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6083 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6084 }
6085}
6086
6087fn parse_client_control_request(
6088 body: &[u8],
6089) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6090 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6091 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6092 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6093 ControlRequestBodyError::InvalidBody
6094 }
6095 Ok(_) => ControlRequestBodyError::UnknownOp,
6096 Err(_) => ControlRequestBodyError::InvalidBody,
6097 };
6098 (err, classification)
6099 })
6100}
6101
6102fn parse_module_control_request_from_module(
6103 body: &[u8],
6104) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6105 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6106 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6107 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6108 Ok(_) => ControlRequestBodyError::UnknownOp,
6109 Err(_) => ControlRequestBodyError::InvalidBody,
6110 };
6111 (err, classification)
6112 })
6113}
6114
6115#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6116enum ProviderRoleKind {
6117 ToolProvider,
6118 PipelineStage,
6119 ManagementSurface,
6120 InternalService,
6121}
6122
6123fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6124 match role {
6125 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6126 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6127 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6128 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
6129 }
6130}
6131
6132fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
6133 roles.iter().map(provider_role_kind).collect()
6134}
6135
6136const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
6139
6140#[derive(Debug, Default, PartialEq, Eq)]
6142struct ScopeOutcomeCounts {
6143 created: usize,
6144 replaced: usize,
6145 updated: usize,
6146 unchanged: usize,
6147 refused: usize,
6148}
6149
6150impl ScopeOutcomeCounts {
6151 fn of(results: &[ScopeRecordResult]) -> Self {
6152 let mut counts = Self::default();
6153 for result in results {
6154 let slot = match result.outcome {
6155 ScopeRecordOutcome::Created => &mut counts.created,
6156 ScopeRecordOutcome::Replaced => &mut counts.replaced,
6157 ScopeRecordOutcome::Updated => &mut counts.updated,
6158 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
6159 ScopeRecordOutcome::Refused => &mut counts.refused,
6160 };
6161 *slot += 1;
6162 }
6163 counts
6164 }
6165}
6166
6167#[cfg(test)]
6168mod scope_outcome_count_tests {
6169 use super::*;
6170
6171 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
6172 ScopeRecordResult {
6173 scope_ref: "r".to_string(),
6174 scope_epoch: 1,
6175 outcome,
6176 code: None,
6177 message: None,
6178 version: None,
6179 parent_state: None,
6180 }
6181 }
6182
6183 #[test]
6186 fn every_outcome_is_counted_in_its_own_field() {
6187 let results = [
6188 result(ScopeRecordOutcome::Created),
6189 result(ScopeRecordOutcome::Created),
6190 result(ScopeRecordOutcome::Replaced),
6191 result(ScopeRecordOutcome::Updated),
6192 result(ScopeRecordOutcome::Unchanged),
6193 result(ScopeRecordOutcome::Refused),
6194 result(ScopeRecordOutcome::Refused),
6195 result(ScopeRecordOutcome::Refused),
6196 ];
6197 assert_eq!(
6198 ScopeOutcomeCounts::of(&results),
6199 ScopeOutcomeCounts {
6200 created: 2,
6201 replaced: 1,
6202 updated: 1,
6203 unchanged: 1,
6204 refused: 3,
6205 }
6206 );
6207 }
6208}
6209
6210fn capability_census_trigger(
6214 old: Option<&CapabilityDeclarations>,
6215 new: Option<&CapabilityDeclarations>,
6216) -> bool {
6217 let old_provides = old
6218 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
6219 .unwrap_or_default();
6220 let old_denies = old
6221 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
6222 .unwrap_or_default();
6223 let new = new.cloned().unwrap_or(CapabilityDeclarations {
6224 provides: Vec::new(),
6225 requires: Vec::new(),
6226 must_never_reach: Vec::new(),
6227 });
6228
6229 new.provides
6230 .iter()
6231 .any(|capability| !old_provides.contains(capability))
6232 || new
6233 .must_never_reach
6234 .iter()
6235 .any(|capability| !old_denies.contains(capability))
6236}
6237
6238fn denied_capability<'a>(
6241 opening_manifest: &'a ModuleManifest,
6242 target_manifest: &ModuleManifest,
6243) -> Option<&'a str> {
6244 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
6245 let target_capabilities = target_manifest.capabilities.as_ref()?;
6246 opening_capabilities
6247 .must_never_reach
6248 .iter()
6249 .find(|denied| {
6250 target_capabilities
6251 .provides
6252 .iter()
6253 .any(|provided| provided == *denied)
6254 })
6255 .map(String::as_str)
6256}
6257
6258fn catalog_update_frozen_field_message(
6259 registered: &ModuleManifest,
6260 provides: &[ProviderRole],
6261) -> Option<String> {
6262 let old_has_provides = !registered.provides.is_empty();
6263 let new_has_provides = !provides.is_empty();
6264 if old_has_provides != new_has_provides {
6265 return Some(format!(
6266 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
6267 registered.module_id
6268 ));
6269 }
6270
6271 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
6272 return Some(format!(
6273 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
6274 registered.module_id
6275 ));
6276 }
6277
6278 let registered_concurrency = manifest_concurrency(registered);
6279 let mut candidate = registered.clone();
6280 candidate.provides = provides.to_vec();
6281 let candidate_concurrency = manifest_concurrency(&candidate);
6282 if candidate_concurrency != registered_concurrency {
6283 return Some(format!(
6284 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
6285 registered.module_id, registered_concurrency, candidate_concurrency
6286 ));
6287 }
6288
6289 None
6292}
6293
6294fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
6295 manifest.provides.iter().any(is_routable_role)
6296}
6297
6298fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
6304 manifest
6305 .provides
6306 .iter()
6307 .find_map(|provider| match provider {
6308 ProviderRole::ToolProvider { concurrency, .. }
6309 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
6310 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
6311 })
6312 .unwrap_or(Concurrency::ModuleManaged)
6313}
6314
6315fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
6322 let has_management_surface = manifest
6323 .provides
6324 .iter()
6325 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
6326 if !has_management_surface {
6327 return false;
6328 }
6329 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
6330 return false;
6331 };
6332 let Some(provides) = raw
6333 .get("manifest")
6334 .and_then(|manifest| manifest.get("provides"))
6335 .and_then(serde_json::Value::as_array)
6336 else {
6337 return false;
6338 };
6339 provides.iter().any(|role| {
6344 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
6345 && role.get("concurrency").is_none()
6346 })
6347}
6348
6349fn negotiate_version(peer_version: u8) -> Result<u8, String> {
6350 if peer_version != PROTOCOL_VERSION {
6351 return Err(format!(
6352 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
6353 ));
6354 }
6355 Ok(PROTOCOL_VERSION)
6356}
6357
6358fn pong(frame: &Frame) -> Result<Frame, RouterError> {
6359 Frame::build_with_version(
6360 response_version(frame),
6361 FrameType::Pong,
6362 frame.header.flags,
6363 0,
6364 0,
6365 frame.header.corr,
6366 Vec::new(),
6367 )
6368 .map_err(RouterError::FrameBuild)
6369}
6370
6371fn control_error_frame(
6372 frame: &Frame,
6373 code: &'static str,
6374 message: impl Into<String>,
6375) -> Result<Frame, RouterError> {
6376 control_error_body_frame(
6377 frame,
6378 ErrorBody {
6379 code: code.to_string(),
6380 message: message.into(),
6381 detail: None,
6382 },
6383 )
6384}
6385
6386fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
6387 let body = serde_json::to_vec(&error).map_err(|err| {
6388 RouterError::backend(
6389 0,
6390 frame.header.corr,
6391 format!("failed to encode control ERROR: {err}"),
6392 )
6393 })?;
6394
6395 Frame::build_with_version(
6396 response_version(frame),
6397 FrameType::Error,
6398 control_flags(),
6399 0,
6400 0,
6401 frame.header.corr,
6402 body,
6403 )
6404 .map_err(RouterError::FrameBuild)
6405}
6406
6407fn control_response_body_frame<T: Serialize>(
6408 frame: &Frame,
6409 reply: &T,
6410 label: &'static str,
6411) -> Result<Frame, RouterError> {
6412 let body = serde_json::to_vec(reply).map_err(|err| {
6413 RouterError::backend(
6414 0,
6415 frame.header.corr,
6416 format!("failed to encode {label}: {err}"),
6417 )
6418 })?;
6419
6420 Frame::build_with_version(
6421 response_version(frame),
6422 FrameType::Response,
6423 control_flags(),
6424 0,
6425 0,
6426 frame.header.corr,
6427 body,
6428 )
6429 .map_err(RouterError::FrameBuild)
6430}
6431
6432fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6462 match err {
6463 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6464 ForwardingError::NoModuleConnection => "target_unavailable",
6465 ForwardingError::ModuleReloading { .. } => "module_reloading",
6466 ForwardingError::ClientRouteChannelExhausted { .. }
6467 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6468 ForwardingError::StaleModuleEndpoint
6469 | ForwardingError::UnknownReservation { .. }
6470 | ForwardingError::ConnectionClosing { .. }
6471 | ForwardingError::ClientEgressClosed { .. }
6472 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
6473 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
6476 ForwardingError::RelayCorrelationExhausted
6477 | ForwardingError::RouteOpenBuild(_)
6478 | ForwardingError::Poisoned => "forwarding_error",
6479 }
6480}
6481
6482fn response_version(frame: &Frame) -> u8 {
6483 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
6484 frame.header.ver
6485 } else {
6486 PROTOCOL_VERSION
6487 }
6488}
6489
6490fn control_flags() -> Flags {
6491 Flags::new(false, Priority::Passive, false)
6492}
6493
6494fn send_goodbye_target_best_effort(
6499 counters: &DaemonCounters,
6500 target: &GoodbyeTarget,
6501 context: &'static str,
6502) {
6503 let Ok(frame) = Frame::build_with_version(
6504 target.negotiated_ver,
6505 FrameType::Goodbye,
6506 control_flags(),
6507 target.channel,
6508 target.epoch,
6509 0,
6510 Vec::new(),
6511 ) else {
6512 return;
6513 };
6514 crate::forwarding::send_module_route_goodbye(
6515 counters,
6516 &target.sink,
6517 frame,
6518 target.module_id.as_deref(),
6519 context,
6520 );
6521}
6522
6523pub(crate) fn send_route_control_pushes(
6524 forwarding: &ForwardingTable,
6525 routes: Vec<EndpointRoute>,
6526 push: ClientControlPush,
6527) {
6528 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
6529 for route in routes {
6530 let target = route.goodbye_target;
6531 if let Some((existing, channels)) = targets
6532 .iter_mut()
6533 .find(|(existing, _)| existing.connection_id == target.connection_id)
6534 {
6535 debug_assert_eq!(
6536 existing.negotiated_ver, target.negotiated_ver,
6537 "one connection cannot negotiate multiple frame versions"
6538 );
6539 if !channels.contains(&target.channel) {
6540 channels.push(target.channel);
6541 }
6542 continue;
6543 }
6544 let channel = target.channel;
6545 targets.push((target, vec![channel]));
6546 }
6547 for (target, mut channels) in targets {
6548 channels.sort_unstable();
6549 let mut push = push.clone();
6550 match &mut push {
6551 ClientControlPush::RouteClosing {
6552 channels: covered, ..
6553 }
6554 | ClientControlPush::RouteClosed {
6555 channels: covered, ..
6556 } => *covered = channels,
6557 }
6558 let body = match serde_json::to_vec(&push) {
6559 Ok(body) => body,
6560 Err(err) => {
6561 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
6562 continue;
6563 }
6564 };
6565 let frame = match Frame::build_with_version(
6566 target.negotiated_ver,
6567 FrameType::Push,
6568 control_flags(),
6569 0,
6570 0,
6571 0,
6572 body.clone(),
6573 ) {
6574 Ok(frame) => frame,
6575 Err(err) => {
6576 warn!(
6577 route_channel = target.channel,
6578 error = %err,
6579 "failed to build route lifecycle control PUSH frame"
6580 );
6581 continue;
6582 }
6583 };
6584 if let Err(err) = target.sink.try_send(frame) {
6585 if target.close_on_delivery_failure() {
6586 warn!(
6587 target_connection_id = target.connection_id.get(),
6588 route_channel = target.channel,
6589 error = %err,
6590 "route lifecycle control PUSH was not delivered to client; closing target connection"
6591 );
6592 let _ = forwarding.escalate_client_delivery_failure(
6593 target.connection_id,
6594 target.channel,
6595 target.epoch,
6596 CloseReason::new(
6597 "route_lifecycle_push_delivery_failed",
6598 format!(
6599 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
6600 target.channel
6601 ),
6602 ),
6603 crate::forwarding::UndeliveredFrame {
6604 module_id: target.module_id.as_deref(),
6605 sink: &target.sink,
6606 },
6607 );
6608 }
6609 }
6610 }
6611}
6612
6613#[cfg(test)]
6614mod tests {
6615 #[cfg(unix)]
6616 #[tokio::test]
6617 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
6618 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
6619 let path = dir.join("subc.jsonc");
6620 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
6621 "program":"/bin/sleep","args":["60"],"protocol":"none",
6622 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
6623 }}}).to_string()).unwrap();
6624 let mut configured = crate::daemon_config::load(&path)
6625 .unwrap()
6626 .unwrap()
6627 .modules
6628 .pop()
6629 .unwrap();
6630 let registry = std::sync::Arc::new(crate::Registry::default());
6631 let handle = crate::SupervisorHandle::new();
6632 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
6633 .with_handle(handle.clone());
6634 let module = supervisor
6635 .supervise_configured_with_health(
6636 configured.module_spec(),
6637 true,
6638 configured.health.clone(),
6639 None,
6640 configured.restart,
6641 )
6642 .unwrap();
6643 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
6644 let before = module.status().unwrap().pid;
6645 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
6646 configured.health.cadence = std::time::Duration::from_secs(3600);
6647 let health_only = handler
6648 .reconcile_supervised_modules(&supervisor, vec![configured.clone()], false)
6649 .await
6650 .unwrap();
6651 assert!(
6652 health_only.changed_pending_reload.is_empty(),
6653 "health policy is already applied live"
6654 );
6655 assert_eq!(module.status().unwrap().pid, before);
6656 assert_eq!(
6657 module.configuration().unwrap().1.http,
6658 configured.health.http
6659 );
6660 configured.args = vec!["61".into()];
6661 let launch = handler
6662 .reconcile_supervised_modules(&supervisor, vec![configured], false)
6663 .await
6664 .unwrap();
6665 assert_eq!(launch.changed_pending_reload, ["stock"]);
6666 assert_eq!(
6667 module.status().unwrap().pid,
6668 before,
6669 "a launch edit is stored until reload"
6670 );
6671 module.drain().await.unwrap();
6672 }
6673 use cortexkit_test_support::ScratchDir;
6674 use std::{
6675 collections::BTreeMap,
6676 fmt,
6677 path::PathBuf,
6678 sync::{Arc, Mutex},
6679 time::Duration,
6680 };
6681
6682 use serde_json::{json, Value};
6683 use subc_protocol::{
6684 manifest::{
6685 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
6686 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
6687 },
6688 session::HealthStatus,
6689 FrameType,
6690 };
6691
6692 use super::*;
6693 use crate::{
6694 forwarding::{DataRoute, DataRouteState},
6695 registry::ChannelState,
6696 router::FrameSink,
6697 stderr_tail::DEFAULT_MAX_LINE_BYTES,
6698 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
6699 RouteCtx, Router,
6700 };
6701 use tokio::{
6702 sync::mpsc,
6703 time::{sleep, Instant},
6704 };
6705 use tracing::{
6706 field::{Field, Visit},
6707 Event, Subscriber,
6708 };
6709 use tracing_subscriber::{layer::Context, prelude::*, Layer};
6710
6711 fn fake_aft_stub_path() -> PathBuf {
6730 let mut path = std::env::current_exe().expect("current_exe available in tests");
6731 path.pop(); path.pop(); path.push(if cfg!(windows) {
6734 "fake-aft-stub.exe"
6735 } else {
6736 "fake-aft-stub"
6737 });
6738 assert!(
6739 path.exists(),
6740 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
6741 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
6742 path.display()
6743 );
6744 path
6745 }
6746
6747 fn client_retries(code: &str) -> bool {
6752 subc_protocol::error_codes::is_retryable_route_open(code)
6753 }
6754
6755 #[test]
6766 fn retryability_of_forwarding_codes_matches_the_failure() {
6767 let transient = [
6770 ForwardingError::NoModuleConnection,
6771 ForwardingError::ModuleReloading {
6772 module_id: "m".into(),
6773 },
6774 ForwardingError::StaleModuleEndpoint,
6775 ForwardingError::UnknownReservation {
6776 client_channel: 1,
6777 module_channel: 1,
6778 },
6779 ForwardingError::ConnectionClosing {
6780 connection_id: ConnectionId::new(1),
6781 },
6782 ForwardingError::ClientEgressClosed {
6783 connection_id: ConnectionId::new(1),
6784 },
6785 ForwardingError::ModuleEgressUnavailable {
6786 connection_id: ConnectionId::new(1),
6787 },
6788 ];
6789 for err in transient {
6790 let code = forwarding_error_code(&err);
6791 assert!(
6792 client_retries(code),
6793 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
6794 );
6795 }
6796
6797 let permanent = [
6802 ForwardingError::ConnectionRoleConflict {
6803 connection_id: ConnectionId::new(1),
6804 },
6805 ForwardingError::ClientRouteChannelExhausted {
6806 connection_id: ConnectionId::new(1),
6807 },
6808 ForwardingError::ModuleRouteChannelExhausted {
6809 endpoint: ModuleEndpointId {
6810 connection_id: ConnectionId::new(1),
6811 generation: 1,
6812 },
6813 },
6814 ForwardingError::RelayCorrelationExhausted,
6815 ForwardingError::RouteOpenBuild("x".into()),
6816 ForwardingError::Poisoned,
6817 ];
6818 for err in permanent {
6819 let code = forwarding_error_code(&err);
6820 assert!(
6821 !client_retries(code),
6822 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
6823 );
6824 }
6825 }
6826
6827 #[tokio::test]
6855 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
6856 let handler = ControlHandler::default();
6857 let frame =
6858 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
6859
6860 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
6864 assert_eq!(
6865 stamped,
6866 Principal::Direct,
6867 "a caller that proved nothing must not be stamped as a supervised module"
6868 );
6869
6870 let forged = handler
6875 .route_open_principal(
6876 &frame,
6877 Some(ConsumerIdentity {
6878 module_id: "aft".to_string(),
6879 launch_nonce: "not-a-real-nonce".to_string(),
6880 }),
6881 )
6882 .unwrap();
6883 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
6884 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
6885 }
6886
6887 #[test]
6904 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
6905 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
6906 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
6907 let ClientControlRequest::RouteOpen {
6908 consumer_identity, ..
6909 } = parsed
6910 else {
6911 panic!("route.open body must parse as RouteOpen");
6912 };
6913 assert_eq!(
6914 consumer_identity,
6915 Some(ConsumerIdentity {
6916 module_id: "aft".to_string(),
6917 launch_nonce: "n".to_string(),
6918 }),
6919 "the wire field name must reach the value route_open_principal reads"
6920 );
6921 }
6922
6923 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
6924 ModuleManifest::builder(module_id, "0.1.0")
6925 .protocol_ver(protocol_ver)
6926 .provides(vec![ProviderRole::ToolProvider {
6927 tools: vec![Tool {
6928 name: "read".to_string(),
6929 description: None,
6930 execution_mode: ExecutionMode::Pure,
6931 schema: json!({"type": "object"}),
6932 }],
6933 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
6934 concurrency: Concurrency::ModuleManaged,
6935 emits_push: true,
6936 sub_supervises: true,
6937 }])
6938 .build()
6939 }
6940
6941 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
6942 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
6943 }
6944
6945 fn hello_frame_with_control_ops(
6946 module_id: &str,
6947 protocol_ver: u8,
6948 corr: u64,
6949 control_ops: Option<Vec<String>>,
6950 ) -> Frame {
6951 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
6952 }
6953
6954 fn hello_frame_with_nonce(
6955 module_id: &str,
6956 protocol_ver: u8,
6957 corr: u64,
6958 launch_nonce: Option<&str>,
6959 ) -> Frame {
6960 hello_frame_full(
6961 module_id,
6962 protocol_ver,
6963 corr,
6964 None,
6965 launch_nonce.map(ToOwned::to_owned),
6966 )
6967 }
6968
6969 fn hello_frame_full(
6970 module_id: &str,
6971 protocol_ver: u8,
6972 corr: u64,
6973 control_ops: Option<Vec<String>>,
6974 launch_nonce: Option<String>,
6975 ) -> Frame {
6976 let body = serde_json::to_vec(&ModuleHelloBody {
6977 manifest: manifest(module_id, protocol_ver),
6978 protocol_ver,
6979 control_ops,
6980 launch_nonce,
6981 })
6982 .unwrap();
6983 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
6984 }
6985
6986 fn non_routable_hello_frame_with_control_ops(
6987 module_id: &str,
6988 corr: u64,
6989 control_ops: Option<Vec<String>>,
6990 ) -> Frame {
6991 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
6992 manifest.provides.clear();
6993 let body = serde_json::to_vec(&ModuleHelloBody {
6994 manifest,
6995 protocol_ver: PROTOCOL_VERSION,
6996 control_ops,
6997 launch_nonce: None,
6998 })
6999 .unwrap();
7000 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7001 }
7002
7003 fn capability_grammar_hello_frame(
7004 capabilities: Value,
7005 runtime_computed: Option<Value>,
7006 corr: u64,
7007 ) -> Frame {
7008 let mut body = serde_json::to_value(ModuleHelloBody {
7009 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
7010 protocol_ver: PROTOCOL_VERSION,
7011 control_ops: None,
7012 launch_nonce: None,
7013 })
7014 .expect("HELLO body serializes");
7015 body["manifest"]["capabilities"] = capabilities;
7016 if let Some(runtime_computed) = runtime_computed {
7017 body["runtime_computed"] = runtime_computed;
7018 }
7019 Frame::build(
7020 FrameType::Hello,
7021 control_flags(),
7022 0,
7023 0,
7024 corr,
7025 serde_json::to_vec(&body).expect("HELLO body reserializes"),
7026 )
7027 .expect("HELLO frame builds")
7028 }
7029
7030 fn channel_request(channel: u16, corr: u64) -> Frame {
7031 Frame::build(
7032 FrameType::Request,
7033 Flags::new(true, Priority::Interactive, false),
7034 channel,
7035 0,
7036 corr,
7037 b"opaque".to_vec(),
7038 )
7039 .unwrap()
7040 }
7041
7042 fn route_ctx(
7043 connection_id: ConnectionId,
7044 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7045 let (tx, rx) = mpsc::channel(8);
7046 (
7047 RouteCtx {
7048 connection_id,
7049 egress: FrameSink::new(tx),
7050 },
7051 rx,
7052 )
7053 }
7054
7055 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7056 serde_json::from_slice(&frame.body).unwrap()
7057 }
7058
7059 async fn hello_via_sink(
7064 handler: &ControlHandler,
7065 ctx: &RouteCtx,
7066 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7067 hello: Frame,
7068 ) -> Frame {
7069 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7070 assert!(
7071 replies.is_empty(),
7072 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7073 );
7074 let ack = rx
7075 .try_recv()
7076 .expect("HELLO_ACK is queued on the module sink")
7077 .frame;
7078 assert_eq!(ack.header.ty, FrameType::HelloAck);
7079 ack
7080 }
7081
7082 fn parse_error(frame: &Frame) -> Value {
7083 serde_json::from_slice(&frame.body).unwrap()
7084 }
7085
7086 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7087 serde_json::from_slice(&frame.body).unwrap()
7088 }
7089
7090 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7091 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7092 route_channel,
7093 route_epoch: 0,
7094 kind,
7095 })
7096 .unwrap();
7097 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7098 }
7099
7100 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7101 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7102 module_id: module_id.to_string(),
7103 })
7104 .unwrap();
7105 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7106 }
7107
7108 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7109 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7110 }
7111
7112 fn route_open_frame_with_consumer_capabilities(
7113 corr: u64,
7114 module_id: &str,
7115 project_root: ScratchDir,
7116 consumer_capabilities: Option<Vec<String>>,
7117 ) -> Frame {
7118 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7119 target: RouteTarget::ToolProvider {
7120 module_id: module_id.to_string(),
7121 },
7122 identity: BindIdentity::new(
7123 project_root.path().to_path_buf(),
7124 "unit".to_string(),
7125 "session".to_string(),
7126 ),
7127 consumer_identity: None,
7128 consumer_capabilities,
7129 role_versions: None,
7130 admission_facts: None,
7131 scope: None,
7132 })
7133 .unwrap();
7134 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7135 }
7136
7137 fn route_open_frame_with_role_versions(
7138 corr: u64,
7139 module_id: &str,
7140 project_root: ScratchDir,
7141 role_versions: Option<BTreeMap<String, String>>,
7142 ) -> Frame {
7143 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7144 target: RouteTarget::ToolProvider {
7145 module_id: module_id.to_string(),
7146 },
7147 identity: BindIdentity::new(
7148 project_root.path().to_path_buf(),
7149 "unit".to_string(),
7150 format!("session-{corr}"),
7151 ),
7152 consumer_identity: None,
7153 consumer_capabilities: None,
7154 role_versions,
7155 admission_facts: None,
7156 scope: None,
7157 })
7158 .unwrap();
7159 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7160 }
7161
7162 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
7163 entries
7164 .iter()
7165 .map(|(role, version)| (role.to_string(), version.to_string()))
7166 .collect()
7167 }
7168
7169 fn route_open_frame_with_admission_facts(
7170 corr: u64,
7171 module_id: &str,
7172 project_root: ScratchDir,
7173 consumer_identity: Option<subc_control::ConsumerIdentity>,
7174 facts: Option<Value>,
7175 ) -> Frame {
7176 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7177 target: RouteTarget::ToolProvider {
7178 module_id: module_id.to_string(),
7179 },
7180 identity: BindIdentity::new(
7181 project_root.path().to_path_buf(),
7182 "unit".to_string(),
7183 format!("session-{corr}"),
7184 ),
7185 consumer_identity,
7186 consumer_capabilities: None,
7187 role_versions: None,
7188 admission_facts: facts,
7189 scope: None,
7190 })
7191 .unwrap();
7192 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7193 }
7194
7195 #[derive(Clone, Default)]
7196 struct EventCapture {
7197 events: Arc<Mutex<Vec<CapturedEvent>>>,
7198 }
7199
7200 #[derive(Clone, Debug)]
7201 struct CapturedEvent {
7202 target: String,
7203 level: tracing::Level,
7204 fields: BTreeMap<String, String>,
7205 }
7206
7207 impl EventCapture {
7208 fn events(&self) -> Vec<CapturedEvent> {
7209 self.events.lock().unwrap().clone()
7210 }
7211 }
7212
7213 impl<S> Layer<S> for EventCapture
7214 where
7215 S: Subscriber,
7216 {
7217 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
7218 let mut visitor = EventFieldVisitor::default();
7219 event.record(&mut visitor);
7220 self.events.lock().unwrap().push(CapturedEvent {
7221 target: event.metadata().target().to_string(),
7222 level: *event.metadata().level(),
7223 fields: visitor.fields,
7224 });
7225 }
7226 }
7227
7228 #[derive(Default)]
7229 struct EventFieldVisitor {
7230 fields: BTreeMap<String, String>,
7231 }
7232
7233 impl Visit for EventFieldVisitor {
7234 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
7235 self.fields
7236 .insert(field.name().to_string(), format!("{value:?}"));
7237 }
7238 }
7239
7240 fn health_response(corr: u64, status: HealthStatus) -> Frame {
7241 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
7242 status,
7243 detail: Some("warming".to_string()),
7244 metrics: Some(json!({"queue_depth": 3})),
7245 })
7246 .unwrap();
7247 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7248 }
7249
7250 fn route_bind_ack(corr: u64) -> Frame {
7251 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
7252 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7253 }
7254
7255 fn unique_project_root(label: &str) -> ScratchDir {
7256 ScratchDir::new(label)
7257 }
7258
7259 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
7260 match parse_route_poll(frame) {
7261 ClientControlResponse::RoutePoll {
7262 status: None,
7263 live: Some(live),
7264 ..
7265 } => assert_eq!(live, expected_live),
7266 other => panic!("unexpected route.poll response: {other:?}"),
7267 }
7268 }
7269
7270 fn bind_liveness_route(
7271 registry: &Registry,
7272 forwarding: &ForwardingTable,
7273 module_id: &str,
7274 ) -> (RouteCtx, u16, u32) {
7275 let module_connection = ConnectionId::new(101);
7276 let client_connection = ConnectionId::new(202);
7277 let registration = registry
7278 .register_with_control_ops(
7279 manifest(module_id, PROTOCOL_VERSION),
7280 PROTOCOL_VERSION,
7281 module_connection,
7282 module_baseline_control_ops(),
7283 )
7284 .unwrap();
7285 let (module_tx, _module_rx) = mpsc::channel(8);
7286 let endpoint = forwarding
7287 .register_module_connection(
7288 module_connection,
7289 module_id.to_string(),
7290 PROTOCOL_VERSION,
7291 manifest_concurrency(®istration.manifest),
7292 FrameSink::new(module_tx),
7293 )
7294 .unwrap();
7295 let (client_ctx, _client_rx) = route_ctx(client_connection);
7296 let pending = forwarding
7297 .begin_route_bind_relay_for_test(
7298 client_connection,
7299 client_ctx.egress.clone(),
7300 1,
7301 module_id,
7302 )
7303 .unwrap();
7304 assert_eq!(pending.endpoint, endpoint);
7305 let route_channel = pending.client_channel;
7306 let route_epoch = pending.client_epoch;
7307 forwarding
7308 .complete_pending_relay(
7309 module_connection,
7310 pending.corr,
7311 RouteBindRelayOutcome::Accepted,
7312 )
7313 .unwrap();
7314 (client_ctx, route_channel, route_epoch)
7315 }
7316
7317 struct FakeProcessLiveness {
7318 live: Option<bool>,
7319 }
7320
7321 impl ModuleProcessLiveness for FakeProcessLiveness {
7322 fn process_live(&self, _module_id: &str) -> Option<bool> {
7323 self.live
7324 }
7325 }
7326
7327 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7328 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
7329 {
7330 let registry = Arc::new(Registry::default());
7331 let supervisor_handle = SupervisorHandle::new();
7332 let supervisor = Supervisor::new_for_test(
7333 Arc::clone(®istry),
7334 RestartPolicy::new(1, Duration::from_millis(10)),
7335 )
7336 .with_handle(supervisor_handle.clone());
7337 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
7338 let module = supervisor
7339 .spawn(ModuleSpec {
7340 module_id: "stderr-tail-wire".to_string(),
7341 program: fake_aft_stub_path(),
7342 args: Vec::new(),
7343 env: vec![
7344 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
7345 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
7346 ],
7347 reserved: false,
7348 reserved_prefixes: Vec::new(),
7349 protocol: ModuleProtocol::Subc,
7350 overlap: Default::default(),
7351 })
7352 .unwrap();
7353
7354 let deadline = Instant::now() + Duration::from_secs(5);
7355 loop {
7356 let tail = module.stderr_tail(None, None);
7357 if tail
7358 .entries
7359 .iter()
7360 .any(|entry| matches!(entry, TailEntry::ProcessStart))
7361 && tail.entries.iter().any(|entry| {
7362 matches!(
7363 entry,
7364 TailEntry::Line {
7365 truncated: true,
7366 ..
7367 }
7368 )
7369 })
7370 {
7371 break;
7372 }
7373 assert!(
7374 Instant::now() < deadline,
7375 "module did not produce a truncated line and restart boundary: {tail:?}"
7376 );
7377 sleep(Duration::from_millis(10)).await;
7378 }
7379
7380 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7381 let request = ClientControlRequest::SupervisorStderrTail {
7382 module_id: "stderr-tail-wire".to_string(),
7383 max_lines: None,
7384 max_bytes: None,
7385 };
7386 let frame = Frame::build(
7387 FrameType::Request,
7388 control_flags(),
7389 0,
7390 0,
7391 1,
7392 serde_json::to_vec(&request).unwrap(),
7393 )
7394 .unwrap();
7395 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7396 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7397 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
7398 serde_json::from_slice(&responses[0].body).unwrap()
7399 else {
7400 panic!("expected supervisor.stderr_tail response");
7401 };
7402
7403 assert!(
7404 tail.entries
7405 .iter()
7406 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
7407 "the control response lost the restart boundary"
7408 );
7409 let Some(StderrTailEntry::Line {
7410 text,
7411 truncated,
7412 at_ms,
7413 }) = tail.entries.iter().find(|entry| {
7414 matches!(
7415 entry,
7416 StderrTailEntry::Line {
7417 truncated: true,
7418 ..
7419 }
7420 )
7421 })
7422 else {
7423 panic!("the control response lost the truncated line");
7424 };
7425 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
7426 assert!(*truncated);
7427 assert!(
7428 at_ms.is_some(),
7429 "the control response lost the line's capture time"
7430 );
7431 }
7432
7433 #[tokio::test(flavor = "current_thread")]
7438 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
7439 let dir = ScratchDir::new("terminals-off-worker");
7440 let journal_path = dir.join("terminals.jsonl");
7441 let registry = Arc::new(Registry::default());
7442 let supervisor_handle = SupervisorHandle::new();
7443 let supervisor =
7444 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7445 .with_handle(supervisor_handle.clone())
7446 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7447 let module = supervisor
7448 .spawn(ModuleSpec {
7449 module_id: "terminal-off-worker".to_string(),
7450 program: fake_aft_stub_path(),
7451 args: Vec::new(),
7452 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7453 reserved: false,
7454 reserved_prefixes: Vec::new(),
7455 protocol: ModuleProtocol::Subc,
7456 overlap: Default::default(),
7457 })
7458 .unwrap();
7459 let deadline = Instant::now() + Duration::from_secs(5);
7460 while module.terminal_history().entries.len() != 2 {
7461 assert!(Instant::now() < deadline, "module did not record two exits");
7462 sleep(Duration::from_millis(10)).await;
7463 }
7464
7465 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
7466 let handler =
7467 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
7468 let frame = Frame::build(
7469 FrameType::Request,
7470 control_flags(),
7471 0,
7472 0,
7473 1,
7474 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
7475 module_id: "terminal-off-worker".to_string(),
7476 })
7477 .unwrap(),
7478 )
7479 .unwrap();
7480 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7481 let spawned_at = std::time::Instant::now();
7482 let read = tokio::spawn({
7483 let handler = Arc::clone(&handler);
7484 async move { handler.handle_control_frame(&ctx, frame).await }
7485 });
7486 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
7489 .await
7490 .unwrap()
7491 .expect("the history read reached its pause");
7492 let elapsed = spawned_at.elapsed();
7493 assert!(
7494 elapsed < Duration::from_secs(2) && !read.is_finished(),
7495 "this task could not run while the history read was paused \
7496 (resumed after {elapsed:?}, read finished: {})",
7497 read.is_finished()
7498 );
7499
7500 drop(release);
7501 let responses = read.await.unwrap().unwrap();
7502 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7503 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
7504 panic!("expected supervisor.terminals response");
7505 };
7506 assert_eq!(terminals.entries.len(), 2);
7507 assert_eq!(terminals.journal_skipped_lines, 0);
7508 assert_eq!(terminals.journal_read_errors, 0);
7509 }
7510
7511 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7512 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
7513 let registry = Arc::new(Registry::default());
7514 let supervisor_handle = SupervisorHandle::new();
7515 let supervisor =
7516 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7517 .with_handle(supervisor_handle.clone());
7518 let module = supervisor
7519 .spawn(ModuleSpec {
7520 module_id: "terminal-golden".to_string(),
7521 program: fake_aft_stub_path(),
7522 args: Vec::new(),
7523 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7524 reserved: false,
7525 reserved_prefixes: Vec::new(),
7526 protocol: ModuleProtocol::Subc,
7527 overlap: Default::default(),
7528 })
7529 .unwrap();
7530
7531 let deadline = Instant::now() + Duration::from_secs(5);
7532 while module.terminal_history().entries.len() != 2 {
7533 assert!(
7534 Instant::now() < deadline,
7535 "module did not retain two terminal exits: {:?}",
7536 module.terminal_history()
7537 );
7538 sleep(Duration::from_millis(10)).await;
7539 }
7540
7541 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7542 let request = ClientControlRequest::SupervisorTerminals {
7543 module_id: "terminal-golden".to_string(),
7544 };
7545 let frame = Frame::build(
7546 FrameType::Request,
7547 control_flags(),
7548 0,
7549 0,
7550 1,
7551 serde_json::to_vec(&request).unwrap(),
7552 )
7553 .unwrap();
7554 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7555 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7556 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7557 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
7558 panic!("expected supervisor.terminals response");
7559 };
7560 assert_eq!(terminals.entries.len(), 2);
7561 assert_eq!(terminals.dropped, 0);
7562
7563 let mut rendered = serde_json::to_value(response).unwrap();
7564 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
7567 for (index, entry) in rendered["entries"]
7568 .as_array_mut()
7569 .expect("terminal response entries array")
7570 .iter_mut()
7571 .enumerate()
7572 {
7573 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
7574 }
7575
7576 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7577 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
7578 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
7579 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7580 std::fs::write(&golden_path, &serialized).unwrap();
7581 }
7582 let expected: Value =
7583 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
7584 assert_eq!(rendered, expected);
7585 }
7586
7587 #[test]
7588 fn hello_registers_manifest_and_returns_ack() {
7589 let registry = Arc::new(Registry::default());
7590 let handler = ControlHandler::new(Arc::clone(®istry));
7591 let conn = ConnectionId::new(1);
7592
7593 let responses = handler
7594 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
7595 .unwrap();
7596
7597 assert_eq!(responses.len(), 1);
7598 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
7599 assert_eq!(responses[0].header.channel, 0);
7600 assert_eq!(responses[0].header.corr, 7);
7601 let ack = parse_ack(&responses[0]);
7602 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
7603 assert!(ack
7604 .subc_capabilities
7605 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
7606 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
7607 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
7608 assert!(ack
7609 .subc_ops
7610 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
7611 assert!(ack
7612 .subc_ops
7613 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
7614
7615 let registration = registry.get_module("aft").unwrap().unwrap();
7616 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
7617 assert_eq!(registration.state, ChannelState::Active);
7618 assert_eq!(registration.connection_id, conn);
7619 assert_eq!(registration.control_ops, module_baseline_control_ops());
7620 }
7621
7622 #[test]
7623 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
7624 let invalid_identifiers = [
7625 ("case_change", "credentials-Provider/v1"),
7626 ("leading_zero", "credentials-provider/v01"),
7627 ("trailing_hyphen", "credentials-provider-/v1"),
7628 ("consecutive_hyphens", "credentials--provider/v1"),
7629 ("uppercase", "Credentials-provider/v1"),
7630 ("missing_v", "credentials-provider/1"),
7631 ("whitespace", "credentials provider/v1"),
7632 ("zero_version", "credentials-provider/v0"),
7633 ("out_of_range_version", "credentials-provider/v4294967296"),
7634 (
7635 "overlength_name",
7636 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
7637 ),
7638 ];
7639 let mut cases = invalid_identifiers
7640 .into_iter()
7641 .map(|(name, identifier)| {
7642 (
7643 format!("identifier_{name}"),
7644 "capabilities.provides[0]".to_string(),
7645 identifier.to_string(),
7646 json!({ "provides": [identifier] }),
7647 None,
7648 )
7649 })
7650 .collect::<Vec<_>>();
7651 cases.extend([
7652 (
7653 "unknown_need".to_string(),
7654 "capabilities.requires[0].need".to_string(),
7655 "deferred".to_string(),
7656 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
7657 None,
7658 ),
7659 (
7660 "duplicate_provides".to_string(),
7661 "capabilities.provides[1]".to_string(),
7662 "credentials-provider/v1".to_string(),
7663 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
7664 None,
7665 ),
7666 (
7667 "duplicate_must_never_reach".to_string(),
7668 "capabilities.must_never_reach[1]".to_string(),
7669 "credentials-provider/v1".to_string(),
7670 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
7671 None,
7672 ),
7673 (
7674 "duplicate_requires_same_need".to_string(),
7675 "capabilities.requires[1]".to_string(),
7676 "credentials-provider/v1".to_string(),
7677 json!({ "requires": [
7678 { "capability": "credentials-provider/v1", "need": "required" },
7679 { "capability": "credentials-provider/v1", "need": "required" }
7680 ] }),
7681 None,
7682 ),
7683 (
7684 "duplicate_requires_conflicting_need".to_string(),
7685 "capabilities.requires[1]".to_string(),
7686 "credentials-provider/v1".to_string(),
7687 json!({ "requires": [
7688 { "capability": "credentials-provider/v1", "need": "required" },
7689 { "capability": "credentials-provider/v1", "need": "optional" }
7690 ] }),
7691 None,
7692 ),
7693 (
7694 "capabilities_root_pointer".to_string(),
7695 "runtime_computed[0]".to_string(),
7696 "/capabilities".to_string(),
7697 json!({}),
7698 Some(json!(["/capabilities"])),
7699 ),
7700 (
7701 "capabilities_descendant_pointer".to_string(),
7702 "runtime_computed[0]".to_string(),
7703 "/capabilities/provides".to_string(),
7704 json!({}),
7705 Some(json!(["/capabilities/provides"])),
7706 ),
7707 (
7708 "malformed_pointer_without_leading_slash".to_string(),
7709 "runtime_computed[0]".to_string(),
7710 "capabilities".to_string(),
7711 json!({}),
7712 Some(json!(["capabilities"])),
7713 ),
7714 (
7715 "malformed_pointer_escape".to_string(),
7716 "runtime_computed[0]".to_string(),
7717 "/roles/~2/tools".to_string(),
7718 json!({}),
7719 Some(json!(["/roles/~2/tools"])),
7720 ),
7721 (
7722 "unknown_capabilities_field".to_string(),
7723 "capabilities.future".to_string(),
7724 "<array>".to_string(),
7725 json!({ "future": [] }),
7726 None,
7727 ),
7728 ]);
7729
7730 for (index, (name, field, value, capabilities, runtime_computed)) in
7731 cases.into_iter().enumerate()
7732 {
7733 let registry = Arc::new(Registry::default());
7734 let handler = ControlHandler::new(Arc::clone(®istry));
7735 let response = handler
7736 .handle_control(
7737 ConnectionId::new((index + 1) as u64),
7738 capability_grammar_hello_frame(
7739 capabilities,
7740 runtime_computed,
7741 index as u64 + 1,
7742 ),
7743 )
7744 .expect("invalid HELLO returns a refusal");
7745
7746 assert_eq!(response.len(), 1, "{name} must emit one refusal");
7747 let error = parse_error(&response[0]);
7748 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
7749 let message = error["message"]
7750 .as_str()
7751 .expect("error message is a string");
7752 assert!(
7753 message.contains(&field),
7754 "{name}: field missing from {message}"
7755 );
7756 assert!(
7757 message.contains(&value),
7758 "{name}: value missing from {message}"
7759 );
7760 assert_eq!(
7761 registry
7762 .active_registration_count()
7763 .expect("registry reads"),
7764 0,
7765 "{name}: refused HELLO must not create a catalog entry"
7766 );
7767 }
7768 }
7769
7770 #[test]
7771 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
7772 let registry = Arc::new(Registry::default());
7773 let handler = ControlHandler::new(Arc::clone(®istry));
7774 let capabilities = json!({
7775 "provides": ["credentials-provider/v1"],
7776 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
7777 "must_never_reach": ["federation-transport/v1"]
7778 });
7779 let response = handler
7780 .handle_control(
7781 ConnectionId::new(99),
7782 capability_grammar_hello_frame(
7783 capabilities.clone(),
7784 Some(json!(["/roles/0/tools"])),
7785 99,
7786 ),
7787 )
7788 .expect("valid HELLO registers");
7789 assert_eq!(response[0].header.ty, FrameType::HelloAck);
7790
7791 let request = Frame::build(
7792 FrameType::Request,
7793 control_flags(),
7794 0,
7795 0,
7796 100,
7797 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7798 .expect("catalog request serializes"),
7799 )
7800 .expect("catalog request frame builds");
7801 let response = handler
7802 .handle_catalog_list(request, None)
7803 .expect("catalog list succeeds");
7804 let ClientControlResponse::CatalogList { modules, .. } =
7805 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
7806 else {
7807 panic!("catalog request must return catalog.list");
7808 };
7809 assert_eq!(modules.len(), 1);
7810 assert_eq!(
7811 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
7812 capabilities
7813 );
7814 }
7815
7816 #[test]
7817 fn catalog_list_mirrors_management_operation_description() {
7818 let registry = Arc::new(Registry::default());
7819 let handler = ControlHandler::new(Arc::clone(®istry));
7820 let description = "List managed records and return their identifiers and metadata.";
7821 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
7822 manifest.provides = vec![ProviderRole::ManagementSurface {
7823 operations: vec![ManagementOperation {
7824 name: "records.list".to_string(),
7825 kind: ManagementOperationKind::Query,
7826 description: Some(description.to_string()),
7827 }],
7828 config_schema: json!({"type": "object"}),
7829 observability: vec![ObservabilitySurface {
7830 name: "records.stats".to_string(),
7831 kind: ObservabilityKind::Snapshot,
7832 }],
7833 identity_scope: vec![IdentityScope::Project],
7834 concurrency: Concurrency::ModuleManaged,
7835 }];
7836 registry
7837 .register_with_control_ops(
7838 manifest,
7839 PROTOCOL_VERSION,
7840 ConnectionId::new(99),
7841 Vec::new(),
7842 )
7843 .expect("described management manifest registers");
7844
7845 let request = Frame::build(
7846 FrameType::Request,
7847 control_flags(),
7848 0,
7849 0,
7850 100,
7851 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
7852 .expect("catalog request serializes"),
7853 )
7854 .expect("catalog request frame builds");
7855 let response = handler
7856 .handle_catalog_list(request, None)
7857 .expect("catalog list succeeds");
7858 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
7859 assert_eq!(
7860 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
7861 "catalog.list must preserve the declared operation description verbatim"
7862 );
7863 }
7864
7865 #[test]
7866 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
7867 let registry = Arc::new(Registry::default());
7868 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7869 [("vault".to_string(), true), ("squatter".to_string(), true)],
7870 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7871 );
7872 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
7873 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
7874 provides: vec!["credentials-provider/v1".to_string()],
7875 requires: Vec::new(),
7876 must_never_reach: Vec::new(),
7877 });
7878 let frame = Frame::build(
7879 FrameType::Hello,
7880 control_flags(),
7881 0,
7882 0,
7883 77,
7884 serde_json::to_vec(&ModuleHelloBody {
7885 manifest: squatter,
7886 protocol_ver: PROTOCOL_VERSION,
7887 control_ops: None,
7888 launch_nonce: None,
7889 })
7890 .expect("HELLO serializes"),
7891 )
7892 .expect("HELLO frame builds");
7893 let response = handler
7894 .handle_control(ConnectionId::new(77), frame)
7895 .expect("reserved claim receives a typed refusal");
7896 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
7897 assert_eq!(
7898 registry
7899 .active_registration_count()
7900 .expect("registry reads"),
7901 0,
7902 "a reserved capability refusal must not leave a catalog entry"
7903 );
7904 }
7905
7906 #[test]
7907 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
7908 for settlement in ["timeout", "inconclusive", "drop"] {
7909 let breakers = RouteBindBreakers::default();
7910 let RouteBindAdmission::Admitted {
7911 guard: mut old,
7912 probe: false,
7913 } = breakers.admit("prov")
7914 else {
7915 panic!("ordinary relay admitted")
7916 };
7917 let RouteBindAdmission::Admitted {
7918 guard: mut opener, ..
7919 } = breakers.admit("prov")
7920 else {
7921 panic!("second relay admitted")
7922 };
7923 assert!(
7924 !opener
7925 .record_timeout(1, Duration::ZERO)
7926 .unwrap()
7927 .reopened_after_probe
7928 );
7929 let RouteBindAdmission::Admitted {
7930 guard: mut probe,
7931 probe: true,
7932 } = breakers.admit("prov")
7933 else {
7934 panic!("one cooldown probe admitted")
7935 };
7936 match settlement {
7937 "timeout" => assert!(
7938 !old.record_timeout(1, Duration::ZERO)
7939 .unwrap()
7940 .reopened_after_probe
7941 ),
7942 "inconclusive" => old.record_inconclusive(),
7943 "drop" => drop(old),
7944 _ => unreachable!(),
7945 }
7946 assert!(
7947 matches!(
7948 breakers.admit("prov"),
7949 RouteBindAdmission::Refused {
7950 probe_in_flight: true,
7951 ..
7952 }
7953 ),
7954 "{settlement} of a pre-open relay cannot release the real probe"
7955 );
7956 assert!(
7957 probe
7958 .record_timeout(1, Duration::ZERO)
7959 .unwrap()
7960 .reopened_after_probe
7961 );
7962 assert!(matches!(
7963 breakers.admit("prov"),
7964 RouteBindAdmission::Admitted { probe: true, .. }
7965 ));
7966 }
7967 let breakers = RouteBindBreakers::default();
7968 let admit = || match breakers.admit("prov") {
7969 RouteBindAdmission::Admitted { guard, .. } => guard,
7970 _ => panic!("relay admitted"),
7971 };
7972 admit().record_timeout(1, Duration::ZERO);
7973 let mut old_probe = admit();
7974 breakers.reset_for_new_module_connection("prov");
7975 admit().record_timeout(1, Duration::ZERO);
7976 let _new_probe = admit();
7977 old_probe.record_inconclusive();
7978 assert!(matches!(
7979 breakers.admit("prov"),
7980 RouteBindAdmission::Refused {
7981 probe_in_flight: true,
7982 ..
7983 }
7984 ));
7985 }
7986
7987 #[tokio::test]
7988 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
7989 for candidate in [false, true] {
7990 let registry = Arc::new(Registry::default());
7991 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
7992 [("vault".to_string(), true), ("squatter".to_string(), true)],
7993 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
7994 );
7995 let conn = ConnectionId::new(77);
7996 let (ctx, mut rx) = route_ctx(conn);
7997 let initial = capability_manifest("squatter", &[], &[]);
7998 if candidate {
7999 registry
8000 .register_candidate_with_control_ops(
8001 initial.clone(),
8002 PROTOCOL_VERSION,
8003 conn,
8004 module_baseline_control_ops(),
8005 )
8006 .unwrap();
8007 handler
8008 .forwarding
8009 .register_candidate_module_connection(
8010 conn,
8011 "squatter".to_string(),
8012 PROTOCOL_VERSION,
8013 manifest_concurrency(&initial),
8014 ctx.egress.clone(),
8015 )
8016 .unwrap();
8017 } else {
8018 hello_via_sink(
8019 &handler,
8020 &ctx,
8021 &mut rx,
8022 hello_frame_with_manifest(initial.clone(), 1),
8023 )
8024 .await;
8025 }
8026 let response = handler
8027 .handle_control_frame(
8028 &ctx,
8029 catalog_update_with_capabilities_frame(
8030 2,
8031 capability_manifest("squatter", &["credentials-provider/v1"], &[])
8032 .capabilities
8033 .unwrap(),
8034 ),
8035 )
8036 .await
8037 .unwrap();
8038 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8039 assert_eq!(
8040 registry
8041 .get_module_by_connection(conn)
8042 .unwrap()
8043 .unwrap()
8044 .manifest,
8045 initial
8046 );
8047 }
8048 }
8049
8050 #[test]
8051 fn server_describe_surfaces_required_capability_verdict_fields() {
8052 let registry = Arc::new(Registry::default());
8053 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8054 [
8055 ("consumer".to_string(), true),
8056 ("provider".to_string(), false),
8057 ],
8058 BTreeMap::new(),
8059 );
8060 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
8061 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8062 provides: Vec::new(),
8063 requires: vec![subc_protocol::manifest::CapabilityRequirement {
8064 capability: "credentials-provider/v1".to_string(),
8065 need: subc_protocol::manifest::CapabilityNeed::Required,
8066 }],
8067 must_never_reach: Vec::new(),
8068 });
8069 let hello = Frame::build(
8070 FrameType::Hello,
8071 control_flags(),
8072 0,
8073 0,
8074 78,
8075 serde_json::to_vec(&ModuleHelloBody {
8076 manifest: consumer,
8077 protocol_ver: PROTOCOL_VERSION,
8078 control_ops: None,
8079 launch_nonce: None,
8080 })
8081 .expect("HELLO serializes"),
8082 )
8083 .expect("HELLO frame builds");
8084 handler
8085 .handle_control(ConnectionId::new(78), hello)
8086 .expect("consumer registers");
8087 let describe = Frame::build(
8088 FrameType::Request,
8089 control_flags(),
8090 0,
8091 0,
8092 79,
8093 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8094 .expect("request serializes"),
8095 )
8096 .expect("describe frame builds");
8097 let response = handler
8098 .handle_server_describe(describe)
8099 .expect("server.describe succeeds");
8100 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8101 let requirement = &rendered["capability_requirements"][0];
8102 assert_eq!(requirement["consumer"], "consumer");
8103 assert_eq!(requirement["verdict"], "never_provided");
8104 assert_eq!(requirement["episode_seq"], 1);
8105 assert_eq!(requirement["config_satisfiable"], false);
8106 assert_eq!(requirement["runtime_available"], false);
8107 assert!(requirement["detail"]
8108 .as_str()
8109 .expect("detail string")
8110 .contains("credentials-provider/v1"));
8111 }
8112
8113 #[test]
8114 fn catalog_list_omits_capabilities_for_legacy_manifest() {
8115 let registry = Arc::new(Registry::default());
8116 let handler = ControlHandler::new(Arc::clone(®istry));
8117 let hello = handler
8118 .handle_control(
8119 ConnectionId::new(101),
8120 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
8121 )
8122 .expect("legacy HELLO registers");
8123 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
8124
8125 let request = Frame::build(
8126 FrameType::Request,
8127 control_flags(),
8128 0,
8129 0,
8130 102,
8131 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8132 .expect("catalog request serializes"),
8133 )
8134 .expect("catalog request frame builds");
8135 let response = handler
8136 .handle_catalog_list(request, None)
8137 .expect("catalog list succeeds");
8138 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8139 assert!(
8140 body["modules"][0].get("capabilities").is_none(),
8141 "legacy manifest must retain an absent capabilities field on catalog.list"
8142 );
8143 }
8144
8145 #[test]
8146 fn hello_ack_omits_storage_when_no_storage_config() {
8147 let registry = Arc::new(Registry::default());
8148 let handler = ControlHandler::new(Arc::clone(®istry));
8149 let responses = handler
8150 .handle_control(
8151 ConnectionId::new(1),
8152 hello_frame("aft", PROTOCOL_VERSION, 7),
8153 )
8154 .unwrap();
8155 let ack = parse_ack(&responses[0]);
8156 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
8157 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
8158 }
8159
8160 #[tokio::test]
8161 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
8162 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
8163 let registry = Arc::new(Registry::default());
8164 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
8165 let responses = handler
8166 .handle_control(
8167 ConnectionId::new(1),
8168 hello_frame("aft", PROTOCOL_VERSION, 7),
8169 )
8170 .unwrap();
8171 let ack = parse_ack(&responses[0]);
8172 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
8173
8174 let described = handler
8175 .handle_control_frame(
8176 &route_ctx(ConnectionId::new(2)).0,
8177 Frame::build(
8178 FrameType::Request,
8179 control_flags(),
8180 0,
8181 0,
8182 9,
8183 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8184 )
8185 .unwrap(),
8186 )
8187 .await
8188 .unwrap();
8189 let ClientControlResponse::ServerDescribe { machine_id, .. } =
8190 serde_json::from_slice(&described[0].body).unwrap()
8191 else {
8192 panic!("server.describe answered with another shape");
8193 };
8194 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
8195 }
8196
8197 #[test]
8198 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
8199 let registry = Arc::new(Registry::default());
8202 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
8203 crate::daemon_config::StorageConfig::Sqlite {
8204 data_home: std::path::PathBuf::from("/data"),
8205 },
8206 ));
8207
8208 let responses = handler
8209 .handle_control(
8210 ConnectionId::new(1),
8211 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
8212 )
8213 .unwrap();
8214 let ack = parse_ack(&responses[0]);
8215 assert_eq!(
8216 ack.storage,
8217 Some(serde_json::json!({
8218 "module_id": "alfonso-routing",
8219 "storage_namespace": "default",
8220 "isolation": { "kind": "module" },
8221 "backend": {
8222 "backend": "sqlite",
8223 "path": "/data/cortexkit/alfonso-routing/store.db"
8224 }
8225 })),
8226 "the delivered descriptor is the module's own sqlite store path"
8227 );
8228 }
8229
8230 #[test]
8231 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
8232 let registry = Arc::new(Registry::default());
8233 let handler = ControlHandler::new(Arc::clone(®istry));
8234 let conn = ConnectionId::new(1);
8235 let responses = handler
8236 .handle_control(
8237 conn,
8238 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8239 )
8240 .unwrap();
8241 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8242 let registration = registry.get_module("aft").unwrap().unwrap();
8243 assert_eq!(registration.control_ops, module_baseline_control_ops());
8244
8245 let frame =
8246 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
8247 assert!(handler
8248 .guard_module_control_op(&frame, "aft", "route.bind")
8249 .unwrap()
8250 .is_none());
8251 let error = handler
8252 .guard_module_control_op(&frame, "aft", "test.synthetic")
8253 .unwrap()
8254 .expect("synthetic ungranted op should be rejected");
8255 assert_eq!(error.header.ty, FrameType::Error);
8256 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
8257 }
8258
8259 #[test]
8260 fn hello_control_ops_some_adds_optional_grants() {
8261 let registry = Arc::new(Registry::default());
8262 let handler = ControlHandler::new(Arc::clone(®istry));
8263 handler
8264 .handle_control(
8265 ConnectionId::new(1),
8266 hello_frame_with_control_ops(
8267 "aft",
8268 PROTOCOL_VERSION,
8269 7,
8270 Some(vec![
8271 "future.synthetic".to_string(),
8272 "route.bind".to_string(),
8273 ]),
8274 ),
8275 )
8276 .unwrap();
8277 let registration = registry.get_module("aft").unwrap().unwrap();
8278 assert_eq!(
8279 registration.control_ops,
8280 vec![
8281 "route.bind".to_string(),
8282 "route.status".to_string(),
8283 "future.synthetic".to_string(),
8284 ]
8285 );
8286 let frame =
8287 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
8288 assert!(handler
8289 .guard_module_control_op(&frame, "aft", "future.synthetic")
8290 .unwrap()
8291 .is_none());
8292 }
8293
8294 #[tokio::test]
8295 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
8296 let registry = Arc::new(Registry::default());
8297 let forwarding = Arc::new(ForwardingTable::default());
8298 let handler =
8299 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8300 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
8301 hello_via_sink(
8302 &handler,
8303 &module_ctx,
8304 &mut module_rx,
8305 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8306 )
8307 .await;
8308
8309 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
8310 let responses = handler
8311 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
8312 .await
8313 .unwrap();
8314 assert_eq!(responses.len(), 1);
8315 assert_eq!(responses[0].header.ty, FrameType::Error);
8316 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
8317 assert!(module_rx.try_recv().is_err());
8318 }
8319
8320 #[tokio::test]
8321 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
8322 let registry = Arc::new(Registry::default());
8323 let forwarding = Arc::new(ForwardingTable::default());
8324 let handler =
8325 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8326 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
8327 hello_via_sink(
8328 &handler,
8329 &module_ctx,
8330 &mut module_rx,
8331 hello_frame_with_control_ops(
8332 "aft",
8333 PROTOCOL_VERSION,
8334 7,
8335 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8336 ),
8337 )
8338 .await;
8339
8340 let project_root = unique_project_root("demux");
8341 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
8342 let route_handler = handler.clone();
8343 let route_task = tokio::spawn(async move {
8344 route_handler
8345 .handle_control_frame(
8346 &route_client_ctx,
8347 route_open_frame(100, "aft", project_root),
8348 )
8349 .await
8350 .unwrap()
8351 });
8352 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8353 .await
8354 .unwrap()
8355 .unwrap();
8356 assert!(matches!(
8357 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
8358 ModuleControlRequest::RouteBind { .. }
8359 ));
8360
8361 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
8362 let health_handler = handler.clone();
8363 let health_task = tokio::spawn(async move {
8364 health_handler
8365 .handle_control_frame(
8366 &health_client_ctx,
8367 supervisor_health_probe_frame(101, "aft"),
8368 )
8369 .await
8370 .unwrap()
8371 });
8372 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8373 .await
8374 .unwrap()
8375 .unwrap();
8376 assert_eq!(
8377 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8378 ModuleControlRequest::HealthCheck {}
8379 );
8380
8381 handler
8382 .handle_control_frame(
8383 &module_ctx,
8384 health_response(health_frame.header.corr, HealthStatus::Degraded),
8385 )
8386 .await
8387 .unwrap();
8388 let health_response = health_task.await.unwrap();
8389 assert_eq!(health_response.len(), 1);
8390 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8391 ClientControlResponse::SupervisorHealthProbe {
8392 module_id,
8393 status,
8394 detail,
8395 metrics,
8396 } => {
8397 assert_eq!(module_id, "aft");
8398 assert_eq!(status, HealthStatus::Degraded);
8399 assert_eq!(detail.as_deref(), Some("warming"));
8400 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
8401 }
8402 other => panic!("unexpected health response: {other:?}"),
8403 }
8404
8405 handler
8406 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8407 .await
8408 .unwrap();
8409 let route_response = route_task.await.unwrap();
8410 assert!(route_response.is_empty());
8411 let published = route_client_rx.recv().await.unwrap();
8412 assert!(matches!(
8413 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8414 ClientControlResponse::RouteOpen { .. }
8415 ));
8416 }
8417
8418 async fn relay_route_open(
8423 handler: &ControlHandler,
8424 client_connection: ConnectionId,
8425 client_egress: &FrameSink,
8426 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8427 corr: u64,
8428 module_id: &str,
8429 project_root_label: &str,
8430 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
8431 let ctx = RouteCtx {
8432 connection_id: client_connection,
8433 egress: client_egress.clone(),
8434 };
8435 let handler = handler.clone();
8436 let project_root = unique_project_root(project_root_label);
8437 let module_id = module_id.to_string();
8438 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
8439 let task = tokio::spawn(async move {
8440 let _guard = tracing::dispatcher::set_default(&dispatch);
8441 handler
8442 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8443 .await
8444 .unwrap()
8445 });
8446 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8447 .await
8448 .expect("module receives the relayed route.bind")
8449 .expect("module egress is open");
8450 (task, bind.frame)
8451 }
8452
8453 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8454 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8455 ModuleControlRequest::RouteBind {
8456 route_channel,
8457 epoch,
8458 ..
8459 } => (route_channel, epoch),
8460 other => panic!("expected a route.bind request, got {other:?}"),
8461 }
8462 }
8463
8464 fn published_route(frame: &Frame) -> (u16, u32) {
8465 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
8466 ClientControlResponse::RouteOpen {
8467 route_channel,
8468 route_epoch,
8469 } => (route_channel, route_epoch),
8470 other => panic!("expected a route.open response, got {other:?}"),
8471 }
8472 }
8473
8474 #[tokio::test]
8494 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
8495 let registry = Arc::new(Registry::default());
8496 let forwarding = Arc::new(ForwardingTable::default());
8497 let handler =
8498 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8499
8500 let module_connection = ConnectionId::new(30);
8501 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8502 hello_via_sink(
8503 &handler,
8504 &module_ctx,
8505 &mut module_rx,
8506 hello_frame("aft", PROTOCOL_VERSION, 7),
8507 )
8508 .await;
8509
8510 let dying_client = ConnectionId::new(31);
8511 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
8512
8513 let (first_task, first_bind) = relay_route_open(
8516 &handler,
8517 dying_client,
8518 &dying_ctx.egress,
8519 &mut module_rx,
8520 100,
8521 "aft",
8522 "closing-first",
8523 )
8524 .await;
8525 handler
8526 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
8527 .await
8528 .unwrap();
8529 assert!(first_task.await.unwrap().is_empty());
8530 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
8531
8532 let (second_task, second_bind) = relay_route_open(
8534 &handler,
8535 dying_client,
8536 &dying_ctx.egress,
8537 &mut module_rx,
8538 101,
8539 "aft",
8540 "closing-second",
8541 )
8542 .await;
8543 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
8544
8545 assert!(forwarding
8548 .escalate_client_delivery_failure(
8549 dying_client,
8550 first_channel,
8551 first_epoch,
8552 CloseReason::new(
8553 "module_to_client_delivery_failed",
8554 "client egress refused a module frame",
8555 ),
8556 crate::forwarding::UndeliveredFrame {
8557 module_id: None,
8558 sink: &dying_ctx.egress,
8559 },
8560 )
8561 .unwrap());
8562 assert!(!dying_ctx.egress.is_closed());
8563
8564 let ack = handler
8566 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
8567 .await;
8568 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
8569 if module_loop_error.is_some() {
8570 handler.cleanup_connection(module_connection).unwrap();
8574 }
8575 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8579 .await
8580 .ok()
8581 .flatten();
8582
8583 assert!(
8585 registry
8586 .get_module_by_connection(module_connection)
8587 .unwrap()
8588 .is_some(),
8589 "one client's closing connection ended the shared module connection: \
8590 {module_loop_error:?}"
8591 );
8592 let cotenant = ConnectionId::new(32);
8594 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
8595 let (cotenant_task, cotenant_bind) = relay_route_open(
8596 &handler,
8597 cotenant,
8598 &cotenant_ctx.egress,
8599 &mut module_rx,
8600 102,
8601 "aft",
8602 "closing-cotenant",
8603 )
8604 .await;
8605 handler
8606 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
8607 .await
8608 .unwrap();
8609 assert!(cotenant_task.await.unwrap().is_empty());
8610 let (cotenant_channel, cotenant_epoch) =
8611 published_route(&cotenant_rx.recv().await.unwrap());
8612 assert!(matches!(
8613 forwarding
8614 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
8615 .unwrap(),
8616 DataRoute::Client(DataRouteState::Bound(_))
8617 ));
8618
8619 let goodbye = post_ack_module_frame
8622 .expect("module receives a GOODBYE for the abandoned route channel");
8623 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8624 assert_eq!(goodbye.header.channel, abandoned_channel);
8625 assert_eq!(goodbye.header.epoch, abandoned_epoch);
8626
8627 assert!(dying_rx.try_recv().is_err());
8631 let second_response = second_task.await.unwrap();
8632 assert_eq!(second_response.len(), 1);
8633 assert_eq!(
8634 parse_error(&second_response[0])["code"],
8635 "target_unavailable"
8636 );
8637 }
8638
8639 async fn route_open_held_after_bind_send() -> (
8643 ControlHandler,
8644 Arc<tokio::sync::Barrier>,
8645 RouteCtx,
8646 mpsc::Receiver<crate::router::OutboundFrame>,
8647 ) {
8648 let hold = Arc::new(tokio::sync::Barrier::new(2));
8649 let handler = ControlHandler::with_forwarding(
8650 Arc::new(Registry::default()),
8651 Arc::new(ForwardingTable::default()),
8652 )
8653 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
8654 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
8655 hello_via_sink(
8656 &handler,
8657 &module_ctx,
8658 &mut module_rx,
8659 hello_frame("aft", PROTOCOL_VERSION, 7),
8660 )
8661 .await;
8662 (handler, hold, module_ctx, module_rx)
8663 }
8664
8665 #[tokio::test]
8672 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
8673 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8674 let client = ConnectionId::new(41);
8675 let (client_ctx, mut client_rx) = route_ctx(client);
8676
8677 let (open_task, bind) = relay_route_open(
8678 &handler,
8679 client,
8680 &client_ctx.egress,
8681 &mut module_rx,
8682 200,
8683 "aft",
8684 "acked-before-mark",
8685 )
8686 .await;
8687 let (module_channel, module_epoch) = route_bind_channel(&bind);
8688 hold.wait().await;
8690
8691 assert!(handler
8693 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
8694 .await
8695 .unwrap()
8696 .is_empty());
8697 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
8698
8699 hold.wait().await;
8700 assert!(
8701 open_task.await.unwrap().is_empty(),
8702 "route.open answers accepted"
8703 );
8704
8705 match module_rx.try_recv() {
8708 Err(mpsc::error::TryRecvError::Empty) => {}
8709 Ok(outbound) => panic!(
8710 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
8711 outbound.frame.header.ty,
8712 outbound.frame.header.channel,
8713 outbound.frame.header.epoch
8714 ),
8715 Err(err) => panic!("module egress closed: {err}"),
8716 }
8717
8718 let router = Router::with_control_handler(Arc::new(handler.clone()));
8721 let request = Frame::build(
8722 FrameType::Request,
8723 Flags::new(true, Priority::Interactive, false),
8724 client_channel,
8725 client_epoch,
8726 201,
8727 b"first request".to_vec(),
8728 )
8729 .unwrap();
8730 router
8731 .route_for_connection(&client_ctx, request)
8732 .await
8733 .unwrap();
8734 let delivered = module_rx.try_recv().unwrap().frame;
8735 assert_eq!(delivered.header.ty, FrameType::Request);
8736 assert_eq!(
8737 (delivered.header.channel, delivered.header.epoch),
8738 (module_channel, module_epoch)
8739 );
8740 assert_eq!(delivered.body, b"first request");
8741 }
8742
8743 #[tokio::test]
8749 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
8750 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8751 let client = ConnectionId::new(42);
8752 let (client_ctx, _client_rx) = route_ctx(client);
8753
8754 let (open_task, bind) = relay_route_open(
8755 &handler,
8756 client,
8757 &client_ctx.egress,
8758 &mut module_rx,
8759 300,
8760 "aft",
8761 "released-before-mark",
8762 )
8763 .await;
8764 let (module_channel, module_epoch) = route_bind_channel(&bind);
8765 hold.wait().await;
8766
8767 handler.cleanup_connection(client).unwrap();
8768 assert!(
8769 module_rx.try_recv().is_err(),
8770 "client teardown must not GOODBYE a bind route.open is still queueing"
8771 );
8772
8773 hold.wait().await;
8774 let response = open_task.await.unwrap();
8775 assert_eq!(response.len(), 1);
8776 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8777
8778 let goodbye = module_rx
8779 .try_recv()
8780 .expect("module is told to drop the binding the daemon abandoned")
8781 .frame;
8782 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8783 assert_eq!(
8784 (goodbye.header.channel, goodbye.header.epoch),
8785 (module_channel, module_epoch)
8786 );
8787 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
8788 }
8789
8790 #[tokio::test]
8795 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
8796 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
8797 let client = ConnectionId::new(43);
8798 let (client_ctx, client_rx) = route_ctx(client);
8799
8800 let (open_task, bind) = relay_route_open(
8801 &handler,
8802 client,
8803 &client_ctx.egress,
8804 &mut module_rx,
8805 400,
8806 "aft",
8807 "gone-client-before-mark",
8808 )
8809 .await;
8810 let (module_channel, module_epoch) = route_bind_channel(&bind);
8811 hold.wait().await;
8812
8813 drop(client_rx);
8814 assert!(client_ctx.egress.is_closed());
8815 handler
8816 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
8817 .await
8818 .unwrap();
8819
8820 hold.wait().await;
8821 let response = open_task.await.unwrap();
8822 assert_eq!(response.len(), 1);
8823 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
8824
8825 let goodbye = module_rx
8826 .try_recv()
8827 .expect("module is told to drop the binding it acked")
8828 .frame;
8829 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
8830 assert_eq!(
8831 (goodbye.header.channel, goodbye.header.epoch),
8832 (module_channel, module_epoch)
8833 );
8834 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
8835 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
8836 }
8837
8838 #[test]
8845 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
8846 let handler = ControlHandler::default();
8847 let module_connection = ConnectionId::new(30);
8848 let client_connection = ConnectionId::new(31);
8849
8850 handler
8851 .refuse_to_end_module_connection_for_a_client(
8852 module_connection,
8853 77,
8854 ForwardingError::ConnectionClosing {
8855 connection_id: client_connection,
8856 },
8857 )
8858 .expect("a closing client must never end the module connection");
8859
8860 assert!(matches!(
8861 handler.refuse_to_end_module_connection_for_a_client(
8862 module_connection,
8863 78,
8864 ForwardingError::ConnectionClosing {
8865 connection_id: module_connection,
8866 },
8867 ),
8868 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
8869 connection_id
8870 })) if connection_id == module_connection
8871 ));
8872 assert!(matches!(
8873 handler.refuse_to_end_module_connection_for_a_client(
8874 module_connection,
8875 79,
8876 ForwardingError::Poisoned,
8877 ),
8878 Err(RouterError::Forwarding(ForwardingError::Poisoned))
8879 ));
8880 assert!(matches!(
8881 handler.refuse_to_end_module_connection_for_a_client(
8882 module_connection,
8883 80,
8884 ForwardingError::StaleModuleEndpoint,
8885 ),
8886 Err(RouterError::Forwarding(
8887 ForwardingError::StaleModuleEndpoint
8888 ))
8889 ));
8890 }
8891
8892 #[tokio::test]
8905 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
8906 let registry = Arc::new(Registry::default());
8907 let forwarding = Arc::new(ForwardingTable::default());
8908 let supervisor = SupervisorHandle::new();
8909 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
8910 let handler =
8911 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
8912 .with_supervisor(supervisor);
8913
8914 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
8915 hello_via_sink(
8916 &handler,
8917 &target_ctx,
8918 &mut target_rx,
8919 hello_frame("target", PROTOCOL_VERSION, 1),
8920 )
8921 .await;
8922
8923 let wrong_nonce = handler
8927 .handle_control_frame(
8928 &route_ctx(ConnectionId::new(91)).0,
8929 route_open_frame_with_admission_facts(
8930 20,
8931 "target",
8932 unique_project_root("admission-facts"),
8933 Some(subc_control::ConsumerIdentity {
8934 module_id: "fed".to_string(),
8935 launch_nonce: "not-the-real-nonce".to_string(),
8936 }),
8937 None,
8938 ),
8939 )
8940 .await
8941 .unwrap();
8942 assert_eq!(
8943 parse_error(&wrong_nonce[0])["code"],
8944 "bad_consumer_identity",
8945 "a mismatched launch nonce must be refused, not stamped Reserved"
8946 );
8947
8948 let never_spawned = handler
8952 .handle_control_frame(
8953 &route_ctx(ConnectionId::new(92)).0,
8954 route_open_frame_with_admission_facts(
8955 21,
8956 "target",
8957 unique_project_root("admission-facts"),
8958 Some(subc_control::ConsumerIdentity {
8959 module_id: "never-spawned".to_string(),
8960 launch_nonce: "any-nonce".to_string(),
8961 }),
8962 None,
8963 ),
8964 )
8965 .await
8966 .unwrap();
8967 assert_eq!(
8968 parse_error(&never_spawned[0])["code"],
8969 "bad_consumer_identity",
8970 "an unspawned module_id must be refused rather than accepted for lack of a record"
8971 );
8972 }
8973
8974 #[tokio::test]
8997 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
8998 let registry = Arc::new(Registry::default());
8999 let forwarding = Arc::new(ForwardingTable::default());
9000 let supervisor = SupervisorHandle::new();
9001 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9002 let handler =
9003 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9004 .with_supervisor(supervisor);
9005
9006 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
9007 hello_via_sink(
9008 &handler,
9009 &target_ctx,
9010 &mut target_rx,
9011 hello_frame("target", PROTOCOL_VERSION, 1),
9012 )
9013 .await;
9014
9015 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
9016 let route_handler = handler.clone();
9017 let route_task = tokio::spawn(async move {
9018 route_handler
9019 .handle_control_frame(
9020 &client_ctx,
9021 route_open_frame_with_admission_facts(
9022 30,
9023 "target",
9024 unique_project_root("admission-facts"),
9025 Some(subc_control::ConsumerIdentity {
9026 module_id: "fed".to_string(),
9027 launch_nonce: "fed-nonce".to_string(),
9028 }),
9029 None,
9030 ),
9031 )
9032 .await
9033 .unwrap()
9034 });
9035
9036 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9043 .await
9044 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
9045 .expect("module control channel closed before route.bind");
9046 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9047 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9048 panic!("expected route.bind")
9049 };
9050 assert_eq!(
9051 principal,
9052 Some(Principal::Reserved {
9053 module_id: "fed".to_string()
9054 }),
9055 "a correctly attested consumer must be stamped Reserved for its own id"
9056 );
9057
9058 handler
9059 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9060 .await
9061 .unwrap();
9062 assert!(route_task.await.unwrap().is_empty());
9063 assert!(
9064 matches!(
9065 serde_json::from_slice::<ClientControlResponse>(
9066 &client_rx.recv().await.unwrap().body
9067 )
9068 .unwrap(),
9069 ClientControlResponse::RouteOpen { .. }
9070 ),
9071 "the route must actually open, not merely avoid an error"
9072 );
9073 }
9074
9075 #[tokio::test(start_paused = true)]
9076 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
9077 let registry = Arc::new(Registry::default());
9078 let forwarding = Arc::new(ForwardingTable::default());
9079 let supervisor = SupervisorHandle::new();
9080 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9081 let handler =
9082 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9083 .with_supervisor(supervisor);
9084
9085 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
9086 hello_via_sink(
9087 &handler,
9088 &target_ctx,
9089 &mut target_rx,
9090 hello_frame("target", PROTOCOL_VERSION, 1),
9091 )
9092 .await;
9093
9094 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
9095 let direct_handler = handler.clone();
9096 let direct_open = tokio::spawn(async move {
9097 direct_handler
9098 .handle_control_frame(
9099 &direct_ctx,
9100 route_open_frame(2, "target", unique_project_root("route-census-direct")),
9101 )
9102 .await
9103 .unwrap()
9104 });
9105 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9106 .await
9107 .expect("no direct route.bind within 5s")
9108 .expect("target control channel closed before direct route.bind");
9109 handler
9110 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
9111 .await
9112 .unwrap();
9113 assert!(direct_open.await.unwrap().is_empty());
9114 let _ = direct_rx.recv().await.unwrap();
9115
9116 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
9117 let reserved_handler = handler.clone();
9118 let reserved_open = tokio::spawn(async move {
9119 reserved_handler
9120 .handle_control_frame(
9121 &reserved_ctx,
9122 route_open_frame_with_admission_facts(
9123 3,
9124 "target",
9125 unique_project_root("admission-facts"),
9126 Some(ConsumerIdentity {
9127 module_id: "fed".to_string(),
9128 launch_nonce: "fed-nonce".to_string(),
9129 }),
9130 None,
9131 ),
9132 )
9133 .await
9134 .unwrap()
9135 });
9136 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9137 .await
9138 .expect("no reserved route.bind within 5s")
9139 .expect("target control channel closed before reserved route.bind");
9140 handler
9141 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
9142 .await
9143 .unwrap();
9144 assert!(reserved_open.await.unwrap().is_empty());
9145 let _ = reserved_rx.recv().await.unwrap();
9146
9147 forwarding
9148 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
9149 .unwrap();
9150 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
9151 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
9152 module_id: Some("target".to_string()),
9153 })
9154 .unwrap();
9155 let census_frame =
9156 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
9157 let response = handler
9158 .handle_control_frame(&census_ctx, census_frame)
9159 .await
9160 .unwrap()
9161 .pop()
9162 .unwrap();
9163 let actual: Value = serde_json::from_slice(&response.body).unwrap();
9164 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
9165 assert!(matches!(
9166 decoded,
9167 ClientControlResponse::SupervisorRoutes { .. }
9168 ));
9169 let routes = actual["modules"][0]["routes"].as_array().unwrap();
9170 assert_eq!(routes.len(), 2);
9171 assert!(routes.iter().all(|route| route["draining"] == true));
9172 assert!(
9175 routes.iter().all(|route| route["drain_reason"] == "reload"),
9176 "draining routes must name the drain's reason: {routes:?}"
9177 );
9178 assert!(routes.iter().any(|route| {
9179 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
9180 }));
9181 assert!(routes.iter().any(|route| {
9182 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
9183 }));
9184
9185 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9186 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
9187 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9188 std::fs::write(
9189 &golden_path,
9190 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
9191 )
9192 .unwrap();
9193 }
9194 let expected: Value =
9195 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
9196 assert_eq!(actual, expected);
9197 }
9198
9199 async fn query_live_roots(
9200 handler: &ControlHandler,
9201 module_ctx: &RouteCtx,
9202 ) -> ModuleControlResponseToModule {
9203 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
9204 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
9205 let response = handler
9206 .handle_control_frame(module_ctx, frame)
9207 .await
9208 .unwrap()
9209 .pop()
9210 .unwrap();
9211 serde_json::from_slice(&response.body).unwrap()
9212 }
9213
9214 #[tokio::test(start_paused = true)]
9215 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
9216 let registry = Arc::new(Registry::default());
9217 let forwarding = Arc::new(ForwardingTable::default());
9218 let handler = ControlHandler::with_forwarding(registry, forwarding);
9219 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
9220 hello_via_sink(
9221 &handler,
9222 &target_ctx,
9223 &mut target_rx,
9224 hello_frame("target", PROTOCOL_VERSION, 1),
9225 )
9226 .await;
9227 let root = unique_project_root("live-roots-known");
9228 let path = ProjectRootId::from_path_allowing_missing(root.path())
9229 .unwrap()
9230 .as_path()
9231 .to_path_buf();
9232 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
9233 let open_handler = handler.clone();
9234 let opened = tokio::spawn(async move {
9235 open_handler
9236 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
9237 .await
9238 .unwrap()
9239 });
9240 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9241 .await
9242 .unwrap()
9243 .unwrap();
9244 handler
9245 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
9246 .await
9247 .unwrap();
9248 assert!(opened.await.unwrap().is_empty());
9249 let _ = client_rx.recv().await.unwrap();
9250
9251 let root = unique_project_root("live-roots-pending");
9252 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
9253 .unwrap()
9254 .as_path()
9255 .to_path_buf();
9256 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
9257 let open_handler = handler.clone();
9258 let pending = tokio::spawn(async move {
9259 open_handler
9260 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
9261 .await
9262 .unwrap()
9263 });
9264 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9265 .await
9266 .unwrap()
9267 .unwrap();
9268 let actual = query_live_roots(&handler, &target_ctx).await;
9269 let ModuleControlResponseToModule::LiveRoots {
9270 roots,
9271 unknown_root_bindings,
9272 total_bindings,
9273 } = actual
9274 else {
9275 panic!("expected live roots")
9276 };
9277 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
9278 assert_eq!(unknown_root_bindings, 0);
9279 assert_eq!(
9280 roots.len(),
9281 2,
9282 "root-known arm must retain each canonical root"
9283 );
9284 assert_eq!(
9285 total_bindings,
9286 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9287 );
9288 let counts = roots
9289 .iter()
9290 .map(|root| (root.project_root.clone(), root.bound, root.pending))
9291 .collect::<Vec<_>>();
9292 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
9293 expected.sort_by(|a, b| a.0.cmp(&b.0));
9294 assert_eq!(
9295 counts, expected,
9296 "roots must sort by path and count pending separately"
9297 );
9298 handler
9299 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
9300 .await
9301 .unwrap();
9302 assert!(pending.await.unwrap().is_empty());
9303 }
9304
9305 #[tokio::test(start_paused = true)]
9306 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
9307 let forwarding = Arc::new(ForwardingTable::default());
9308 let handler =
9309 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9310 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
9311 hello_via_sink(
9312 &handler,
9313 &target_ctx,
9314 &mut target_rx,
9315 hello_frame("target", PROTOCOL_VERSION, 1),
9316 )
9317 .await;
9318 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
9319 let pending = forwarding
9320 .begin_route_bind_relay_for_test(
9321 client_ctx.connection_id,
9322 client_ctx.egress.clone(),
9323 2,
9324 "target",
9325 )
9326 .unwrap();
9327 forwarding
9328 .complete_pending_relay(
9329 target_ctx.connection_id,
9330 pending.corr,
9331 RouteBindRelayOutcome::Accepted,
9332 )
9333 .unwrap();
9334 let actual = query_live_roots(&handler, &target_ctx).await;
9335 let ModuleControlResponseToModule::LiveRoots {
9336 roots,
9337 unknown_root_bindings,
9338 total_bindings,
9339 } = actual
9340 else {
9341 panic!("expected live roots")
9342 };
9343 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
9344 assert_eq!(
9345 unknown_root_bindings, 1,
9346 "unknown-root arm must not read as no bindings"
9347 );
9348 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
9349 assert_eq!(
9350 total_bindings,
9351 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9352 );
9353 }
9354
9355 #[tokio::test(start_paused = true)]
9363 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
9364 let forwarding = Arc::new(ForwardingTable::default());
9365 let handler =
9366 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9367 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
9368 let replies = handler
9369 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
9370 .await
9371 .unwrap();
9372 let queued_by_hello = module_rx.len();
9373
9374 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
9375 let open_handler = handler.clone();
9376 let open = tokio::spawn(async move {
9377 open_handler
9378 .handle_control_frame(
9379 &client_ctx,
9380 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
9381 )
9382 .await
9383 .unwrap()
9384 });
9385 let mut spins = 0;
9387 while module_rx.len() == queued_by_hello {
9388 spins += 1;
9389 assert!(spins < 10_000, "route.open never queued a route.bind");
9390 tokio::task::yield_now().await;
9391 }
9392
9393 for reply in replies {
9395 module_ctx.egress.send(reply).await.unwrap();
9396 }
9397
9398 let first = module_rx.recv().await.unwrap().frame;
9399 assert_eq!(
9400 first.header.ty,
9401 FrameType::HelloAck,
9402 "the first frame a registering module reads must be its HELLO_ACK"
9403 );
9404 assert_eq!(first.header.corr, 7);
9405 let second = module_rx.recv().await.unwrap().frame;
9406 assert_eq!(second.header.ty, FrameType::Request);
9407 assert!(
9408 matches!(
9409 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
9410 ModuleControlRequest::RouteBind { .. }
9411 ),
9412 "the route.bind follows the ack"
9413 );
9414 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
9415
9416 handler
9417 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
9418 .await
9419 .unwrap();
9420 assert!(open.await.unwrap().is_empty());
9421 let _ = client_rx.recv().await.unwrap();
9422 }
9423
9424 #[tokio::test(start_paused = true)]
9425 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
9426 let handler = ControlHandler::with_forwarding(
9427 Arc::new(Registry::default()),
9428 Arc::new(ForwardingTable::default()),
9429 );
9430 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
9431 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
9432 hello_via_sink(
9433 &handler,
9434 &first_ctx,
9435 &mut first_rx,
9436 hello_frame("first", PROTOCOL_VERSION, 1),
9437 )
9438 .await;
9439 hello_via_sink(
9440 &handler,
9441 &second_ctx,
9442 &mut second_rx,
9443 hello_frame("second", PROTOCOL_VERSION, 2),
9444 )
9445 .await;
9446 let root = unique_project_root("second-only");
9447 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
9448 let cloned = handler.clone();
9449 let open = tokio::spawn(async move {
9450 cloned
9451 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
9452 .await
9453 .unwrap()
9454 });
9455 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
9456 .await
9457 .unwrap()
9458 .unwrap();
9459 let first = query_live_roots(&handler, &first_ctx).await;
9460 let second = query_live_roots(&handler, &second_ctx).await;
9461 assert!(
9462 matches!(
9463 first,
9464 ModuleControlResponseToModule::LiveRoots {
9465 total_bindings: 0,
9466 ..
9467 }
9468 ),
9469 "cross-module scope must not expose another module's roots"
9470 );
9471 assert!(
9472 matches!(
9473 second,
9474 ModuleControlResponseToModule::LiveRoots {
9475 total_bindings: 1,
9476 ..
9477 }
9478 ),
9479 "second module must see its pending route"
9480 );
9481 handler
9482 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
9483 .await
9484 .unwrap();
9485 assert!(open.await.unwrap().is_empty());
9486 }
9487
9488 #[tokio::test(start_paused = true)]
9489 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
9490 let handler = ControlHandler::with_forwarding(
9491 Arc::new(Registry::default()),
9492 Arc::new(ForwardingTable::default()),
9493 );
9494 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
9495 hello_via_sink(
9496 &handler,
9497 &target_ctx,
9498 &mut target_rx,
9499 hello_frame("target", PROTOCOL_VERSION, 1),
9500 )
9501 .await;
9502 let actual = query_live_roots(&handler, &target_ctx).await;
9503 let ModuleControlResponseToModule::LiveRoots {
9504 roots,
9505 unknown_root_bindings,
9506 total_bindings,
9507 } = actual
9508 else {
9509 panic!("expected live roots")
9510 };
9511 assert!(roots.is_empty());
9512 assert_eq!(unknown_root_bindings, 0);
9513 assert_eq!(total_bindings, 0);
9514 assert_eq!(
9515 total_bindings,
9516 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9517 );
9518 }
9519
9520 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
9527 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9528 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
9529 let text = std::fs::read_to_string(&path)
9530 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
9531 let vectors: Vec<(String, Value)> = text
9532 .lines()
9533 .filter(|line| !line.trim().is_empty())
9534 .map(|line| {
9535 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
9536 let id = entry["corpus_id"]
9537 .as_str()
9538 .expect("every vector carries a corpus_id")
9539 .to_string();
9540 (id, entry["package"].clone())
9541 })
9542 .collect();
9543 assert_eq!(
9547 vectors.len(),
9548 3,
9549 "vendored fed corpus changed size; re-sync from subc-federation"
9550 );
9551
9552 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
9567 let richest = vectors
9568 .iter()
9569 .filter_map(|(_, package)| package.as_object())
9570 .map(|object| {
9571 object
9572 .keys()
9573 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
9574 .count()
9575 })
9576 .max()
9577 .unwrap_or(0);
9578 assert!(
9579 richest >= 2,
9580 "vendored corpus no longer carries a package with unmodelled fields, \
9581 so the relay test can no longer distinguish a verbatim relay from a lossy one"
9582 );
9583
9584 vectors
9585 }
9586
9587 #[tokio::test]
9595 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
9596 for (corpus_id, package) in fed_admission_facts_vectors() {
9597 let registry = Arc::new(Registry::default());
9598 let forwarding = Arc::new(ForwardingTable::default());
9599 let supervisor = SupervisorHandle::new();
9600 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9601 let handler =
9602 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9603 .with_supervisor(supervisor)
9604 .with_admission_facts_config(
9605 Some("fed".to_string()),
9606 Some(vec!["target".to_string()]),
9607 );
9608
9609 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9610 hello_via_sink(
9611 &handler,
9612 &target_ctx,
9613 &mut target_rx,
9614 hello_frame("target", PROTOCOL_VERSION, 1),
9615 )
9616 .await;
9617
9618 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
9619 let route_handler = handler.clone();
9620 let expected = package.clone();
9621 let route_task = tokio::spawn(async move {
9622 route_handler
9623 .handle_control_frame(
9624 &client_ctx,
9625 route_open_frame_with_admission_facts(
9626 20,
9627 "target",
9628 unique_project_root("admission-facts"),
9629 Some(subc_control::ConsumerIdentity {
9630 module_id: "fed".to_string(),
9631 launch_nonce: "fed-nonce".to_string(),
9632 }),
9633 Some(package),
9634 ),
9635 )
9636 .await
9637 .unwrap()
9638 });
9639
9640 let bind_frame = target_rx.recv().await.unwrap();
9641 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9642 let ModuleControlRequest::RouteBind {
9643 admission_facts, ..
9644 } = bind
9645 else {
9646 panic!("{corpus_id}: expected route.bind")
9647 };
9648 assert_eq!(
9649 admission_facts,
9650 Some(expected),
9651 "{corpus_id}: relay must not add, drop or reshape any field"
9652 );
9653
9654 handler
9655 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9656 .await
9657 .unwrap();
9658 route_task.await.unwrap();
9659 }
9660 }
9661
9662 #[tokio::test]
9663 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
9664 let registry = Arc::new(Registry::default());
9665 let forwarding = Arc::new(ForwardingTable::default());
9666 let supervisor = SupervisorHandle::new();
9667 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9668 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
9669 let handler =
9670 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9671 .with_supervisor(supervisor)
9672 .with_admission_facts_config(
9673 Some("fed".to_string()),
9674 Some(vec!["target".to_string()]),
9675 );
9676
9677 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
9678 hello_via_sink(
9679 &handler,
9680 &target_ctx,
9681 &mut target_rx,
9682 hello_frame("target", PROTOCOL_VERSION, 1),
9683 )
9684 .await;
9685 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
9686 hello_via_sink(
9687 &handler,
9688 &other_ctx,
9689 &mut other_rx,
9690 hello_frame("other", PROTOCOL_VERSION, 2),
9691 )
9692 .await;
9693
9694 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
9695 let expected_facts = facts.clone();
9696 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
9697 let route_handler = handler.clone();
9698 let route_task = tokio::spawn(async move {
9699 route_handler
9700 .handle_control_frame(
9701 &client_ctx,
9702 route_open_frame_with_admission_facts(
9703 10,
9704 "target",
9705 unique_project_root("admission-facts"),
9706 Some(subc_control::ConsumerIdentity {
9707 module_id: "fed".to_string(),
9708 launch_nonce: "fed-nonce".to_string(),
9709 }),
9710 Some(facts.clone()),
9711 ),
9712 )
9713 .await
9714 .unwrap()
9715 });
9716 let bind_frame = target_rx.recv().await.unwrap();
9717 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9718 let ModuleControlRequest::RouteBind {
9719 admission_facts, ..
9720 } = bind
9721 else {
9722 panic!("expected route.bind")
9723 };
9724 assert_eq!(admission_facts, Some(expected_facts));
9725 handler
9726 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9727 .await
9728 .unwrap();
9729 assert!(route_task.await.unwrap().is_empty());
9730 assert!(matches!(
9731 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
9732 .unwrap(),
9733 ClientControlResponse::RouteOpen { .. }
9734 ));
9735
9736 let direct = handler
9737 .handle_control_frame(
9738 &route_ctx(ConnectionId::new(73)).0,
9739 route_open_frame_with_admission_facts(
9740 11,
9741 "target",
9742 unique_project_root("admission-facts"),
9743 None,
9744 Some(json!({"x": 1})),
9745 ),
9746 )
9747 .await
9748 .unwrap();
9749 assert_eq!(
9750 parse_error(&direct[0])["code"],
9751 "admission_facts_not_permitted"
9752 );
9753
9754 let different_reserved = handler
9755 .handle_control_frame(
9756 &route_ctx(ConnectionId::new(77)).0,
9757 route_open_frame_with_admission_facts(
9758 15,
9759 "target",
9760 unique_project_root("admission-facts"),
9761 Some(subc_control::ConsumerIdentity {
9762 module_id: "other".to_string(),
9763 launch_nonce: "other-nonce".to_string(),
9764 }),
9765 Some(json!({"x": 1})),
9766 ),
9767 )
9768 .await
9769 .unwrap();
9770 assert_eq!(
9771 parse_error(&different_reserved[0])["code"],
9772 "admission_facts_not_permitted"
9773 );
9774
9775 let other_target = handler
9776 .handle_control_frame(
9777 &route_ctx(ConnectionId::new(74)).0,
9778 route_open_frame_with_admission_facts(
9779 12,
9780 "other",
9781 unique_project_root("admission-facts"),
9782 Some(subc_control::ConsumerIdentity {
9783 module_id: "fed".to_string(),
9784 launch_nonce: "fed-nonce".to_string(),
9785 }),
9786 Some(json!({"x": 1})),
9787 ),
9788 )
9789 .await
9790 .unwrap();
9791 assert_eq!(
9792 parse_error(&other_target[0])["code"],
9793 "admission_facts_target_not_allowed"
9794 );
9795
9796 let nonexistent = handler
9797 .handle_control_frame(
9798 &route_ctx(ConnectionId::new(75)).0,
9799 route_open_frame_with_admission_facts(
9800 13,
9801 "missing",
9802 unique_project_root("admission-facts"),
9803 None,
9804 Some(json!({"x": 1})),
9805 ),
9806 )
9807 .await
9808 .unwrap();
9809 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
9810
9811 let described = handler
9812 .handle_control_frame(
9813 &route_ctx(ConnectionId::new(76)).0,
9814 Frame::build(
9815 FrameType::Request,
9816 control_flags(),
9817 0,
9818 0,
9819 14,
9820 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9821 )
9822 .unwrap(),
9823 )
9824 .await
9825 .unwrap();
9826 let ClientControlResponse::ServerDescribe { capabilities, .. } =
9827 serde_json::from_slice(&described[0].body).unwrap()
9828 else {
9829 panic!("expected server.describe response")
9830 };
9831 assert!(capabilities
9832 .iter()
9833 .any(|cap| cap == "admission_facts_relay_v1"));
9834 }
9835
9836 #[tokio::test]
9837 async fn admission_facts_without_configured_carrier_are_rejected() {
9838 let registry = Arc::new(Registry::default());
9839 let forwarding = Arc::new(ForwardingTable::default());
9840 let handler = ControlHandler::with_forwarding(registry, forwarding);
9841 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
9842 hello_via_sink(
9843 &handler,
9844 &target_ctx,
9845 &mut target_rx,
9846 hello_frame("target", PROTOCOL_VERSION, 1),
9847 )
9848 .await;
9849
9850 let responses = handler
9851 .handle_control_frame(
9852 &route_ctx(ConnectionId::new(79)).0,
9853 route_open_frame_with_admission_facts(
9854 16,
9855 "target",
9856 unique_project_root("admission-facts"),
9857 None,
9858 Some(json!({"x": 1})),
9859 ),
9860 )
9861 .await
9862 .unwrap();
9863 assert_eq!(
9864 parse_error(&responses[0])["code"],
9865 "admission_facts_not_permitted"
9866 );
9867 }
9868
9869 #[tokio::test]
9870 async fn route_open_relays_consumer_capabilities_verbatim() {
9871 let registry = Arc::new(Registry::default());
9872 let forwarding = Arc::new(ForwardingTable::default());
9873 let handler =
9874 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9875 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
9876 hello_via_sink(
9877 &handler,
9878 &module_ctx,
9879 &mut module_rx,
9880 hello_frame("aft", PROTOCOL_VERSION, 7),
9881 )
9882 .await;
9883
9884 let expected = vec!["elicitation".to_string(), "roots".to_string()];
9885 let expected_for_request = expected.clone();
9886 let project_root = unique_project_root("consumer-capabilities-present");
9887 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
9888 let route_handler = handler.clone();
9889 let route_task = tokio::spawn(async move {
9890 route_handler
9891 .handle_control_frame(
9892 &client_ctx,
9893 route_open_frame_with_consumer_capabilities(
9894 401,
9895 "aft",
9896 project_root,
9897 Some(expected_for_request),
9898 ),
9899 )
9900 .await
9901 .unwrap()
9902 });
9903 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9904 .await
9905 .unwrap()
9906 .unwrap();
9907 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9908 let ModuleControlRequest::RouteBind {
9909 consumer_capabilities,
9910 ..
9911 } = bind
9912 else {
9913 panic!("expected route.bind request, got {bind:?}");
9914 };
9915 assert_eq!(consumer_capabilities, Some(expected.clone()));
9916
9917 handler
9918 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9919 .await
9920 .unwrap();
9921 let route_response = route_task.await.unwrap();
9922 assert!(route_response.is_empty());
9923 let published = client_rx.recv().await.unwrap();
9924 assert!(matches!(
9925 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9926 ClientControlResponse::RouteOpen { .. }
9927 ));
9928 }
9929
9930 #[tokio::test]
9931 async fn route_open_without_consumer_capabilities_relays_none() {
9932 let registry = Arc::new(Registry::default());
9933 let forwarding = Arc::new(ForwardingTable::default());
9934 let handler =
9935 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9936 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
9937 hello_via_sink(
9938 &handler,
9939 &module_ctx,
9940 &mut module_rx,
9941 hello_frame("aft", PROTOCOL_VERSION, 7),
9942 )
9943 .await;
9944
9945 let project_root = unique_project_root("consumer-capabilities-absent");
9946 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
9947 let route_handler = handler.clone();
9948 let route_task = tokio::spawn(async move {
9949 route_handler
9950 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
9951 .await
9952 .unwrap()
9953 });
9954 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9955 .await
9956 .unwrap()
9957 .unwrap();
9958 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9959 let ModuleControlRequest::RouteBind {
9960 consumer_capabilities,
9961 ..
9962 } = bind
9963 else {
9964 panic!("expected route.bind request, got {bind:?}");
9965 };
9966 assert_eq!(consumer_capabilities, None);
9967
9968 handler
9969 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9970 .await
9971 .unwrap();
9972 let route_response = route_task.await.unwrap();
9973 assert!(route_response.is_empty());
9974 let published = client_rx.recv().await.unwrap();
9975 assert!(matches!(
9976 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9977 ClientControlResponse::RouteOpen { .. }
9978 ));
9979 }
9980
9981 async fn bind_role_versions_for(
9985 sent: Option<BTreeMap<String, String>>,
9986 connection: u64,
9987 ) -> Option<BTreeMap<String, String>> {
9988 let registry = Arc::new(Registry::default());
9989 let forwarding = Arc::new(ForwardingTable::default());
9990 let handler =
9991 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9992 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
9993 hello_via_sink(
9994 &handler,
9995 &module_ctx,
9996 &mut module_rx,
9997 hello_frame("aft", PROTOCOL_VERSION, 7),
9998 )
9999 .await;
10000 let project_root = unique_project_root("role-versions");
10001 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
10002 let route_handler = handler.clone();
10003 let route_task = tokio::spawn(async move {
10004 route_handler
10005 .handle_control_frame(
10006 &client_ctx,
10007 route_open_frame_with_role_versions(403, "aft", project_root, sent),
10008 )
10009 .await
10010 .unwrap()
10011 });
10012 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10013 .await
10014 .expect("a well-formed route.open reaches the module as a bind")
10015 .unwrap();
10016 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10017 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
10018 panic!("expected route.bind request, got {bind:?}");
10019 };
10020 handler
10021 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10022 .await
10023 .unwrap();
10024 assert!(route_task.await.unwrap().is_empty());
10025 let published = client_rx.recv().await.unwrap();
10026 assert!(matches!(
10027 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10028 ClientControlResponse::RouteOpen { .. }
10029 ));
10030 role_versions
10031 }
10032
10033 #[tokio::test]
10034 async fn route_open_relays_role_versions_verbatim() {
10035 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
10036 assert_eq!(
10037 bind_role_versions_for(Some(sent.clone()), 141).await,
10038 Some(sent)
10039 );
10040 }
10041
10042 #[tokio::test]
10045 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
10046 assert_eq!(bind_role_versions_for(None, 143).await, None);
10047 assert_eq!(
10048 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
10049 None
10050 );
10051 }
10052
10053 #[tokio::test]
10054 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
10055 let registry = Arc::new(Registry::default());
10056 let forwarding = Arc::new(ForwardingTable::default());
10057 let handler =
10058 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10059 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
10060 hello_via_sink(
10061 &handler,
10062 &module_ctx,
10063 &mut module_rx,
10064 hello_frame("aft", PROTOCOL_VERSION, 7),
10065 )
10066 .await;
10067
10068 let nine: BTreeMap<String, String> = (0..9)
10069 .map(|index| (format!("role-{index}"), "v1".to_string()))
10070 .collect();
10071 for (label, malformed) in [
10072 (
10073 "invalid role name",
10074 role_versions(&[("Tool_Provider", "v1")]),
10075 ),
10076 ("invalid version", role_versions(&[("tool-provider", "v0")])),
10077 ("nine entries", nine),
10078 ] {
10079 let responses = tokio::time::timeout(
10082 Duration::from_secs(1),
10083 handler.handle_control_frame(
10084 &route_ctx(ConnectionId::new(148)).0,
10085 route_open_frame_with_role_versions(
10086 404,
10087 "aft",
10088 unique_project_root("role-versions-malformed"),
10089 Some(malformed),
10090 ),
10091 ),
10092 )
10093 .await
10094 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
10095 .unwrap();
10096 assert_eq!(responses.len(), 1, "{label}");
10097 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
10098 let error = parse_error(&responses[0]);
10099 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
10100 assert_eq!(
10101 error["detail"]["field"], "role_versions",
10102 "{label}: {error}"
10103 );
10104 assert!(
10105 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
10106 "{label}: a malformed declaration is terminal"
10107 );
10108 assert!(
10109 module_rx.try_recv().is_err(),
10110 "{label}: the module must never see a bind"
10111 );
10112 }
10113 }
10114
10115 #[tokio::test]
10119 async fn route_role_versions_capability_is_advertised() {
10120 let handler = ControlHandler::new(Arc::new(Registry::default()));
10121 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
10122 let ack = hello_via_sink(
10123 &handler,
10124 &ctx,
10125 &mut rx,
10126 hello_frame("m", PROTOCOL_VERSION, 1),
10127 )
10128 .await;
10129 let ack = parse_ack(&ack);
10130 assert!(
10131 ack.subc_capabilities
10132 .iter()
10133 .any(|c| c == "route-role-versions/v1"),
10134 "{:?}",
10135 ack.subc_capabilities
10136 );
10137
10138 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
10139 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
10140 let reply = handler
10141 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
10142 .await
10143 .unwrap()
10144 .pop()
10145 .unwrap();
10146 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10147 serde_json::from_slice(&reply.body).unwrap()
10148 else {
10149 panic!("not a server.describe reply");
10150 };
10151 assert!(
10152 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
10153 "{capabilities:?}"
10154 );
10155 }
10156
10157 #[tokio::test]
10161 async fn catalog_events_capability_is_advertised() {
10162 let handler = ControlHandler::new(Arc::new(Registry::default()));
10163 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
10164 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
10165 let reply = handler
10166 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
10167 .await
10168 .unwrap()
10169 .pop()
10170 .unwrap();
10171 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10172 serde_json::from_slice(&reply.body).unwrap()
10173 else {
10174 panic!("not a server.describe reply");
10175 };
10176 assert!(
10177 capabilities
10178 .iter()
10179 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
10180 "{capabilities:?}"
10181 );
10182 }
10183
10184 #[tokio::test]
10185 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
10186 let registry = Arc::new(Registry::default());
10187 let forwarding = Arc::new(ForwardingTable::default());
10188 let handler =
10189 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10190 .with_health_probe_timeout(Duration::from_secs(5));
10191 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
10192 hello_via_sink(
10193 &handler,
10194 &module_ctx,
10195 &mut module_rx,
10196 non_routable_hello_frame_with_control_ops(
10197 "mcp",
10198 300,
10199 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10200 ),
10201 )
10202 .await;
10203 assert!(registry
10204 .get_module("mcp")
10205 .unwrap()
10206 .unwrap()
10207 .manifest
10208 .provides
10209 .is_empty());
10210
10211 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
10212 let route_response = handler
10213 .handle_control_frame(
10214 &route_client_ctx,
10215 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
10216 )
10217 .await
10218 .unwrap();
10219 assert_eq!(route_response[0].header.ty, FrameType::Error);
10220 assert_eq!(
10221 parse_error(&route_response[0])["code"],
10222 "target_unavailable"
10223 );
10224 assert!(parse_error(&route_response[0])["message"]
10225 .as_str()
10226 .unwrap()
10227 .contains("does not provide the requested target"));
10228 assert!(module_rx.try_recv().is_err());
10229
10230 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
10231 let health_handler = handler.clone();
10232 let health_task = tokio::spawn(async move {
10233 health_handler
10234 .handle_control_frame(
10235 &health_client_ctx,
10236 supervisor_health_probe_frame(302, "mcp"),
10237 )
10238 .await
10239 .unwrap()
10240 });
10241 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10242 .await
10243 .unwrap()
10244 .unwrap();
10245 assert_eq!(
10246 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10247 ModuleControlRequest::HealthCheck {}
10248 );
10249 handler
10250 .handle_control_frame(
10251 &module_ctx,
10252 health_response(health_frame.header.corr, HealthStatus::Ok),
10253 )
10254 .await
10255 .unwrap();
10256 let health_response = health_task.await.unwrap();
10257 assert_eq!(health_response[0].header.ty, FrameType::Response);
10258 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10259 ClientControlResponse::SupervisorHealthProbe {
10260 module_id, status, ..
10261 } => {
10262 assert_eq!(module_id, "mcp");
10263 assert_eq!(status, HealthStatus::Ok);
10264 }
10265 other => panic!("unexpected health response: {other:?}"),
10266 }
10267
10268 forwarding
10273 .cleanup_connection(module_ctx.connection_id)
10274 .unwrap();
10275 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
10276 let cleanup_response = tokio::time::timeout(
10277 Duration::from_millis(200),
10278 handler.handle_control_frame(
10279 &cleanup_probe_ctx,
10280 supervisor_health_probe_frame(303, "mcp"),
10281 ),
10282 )
10283 .await
10284 .expect("probe should fail immediately when the control lane is gone")
10285 .unwrap();
10286 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
10287 assert_eq!(
10288 parse_error(&cleanup_response[0])["code"],
10289 "target_unavailable"
10290 );
10291 assert!(parse_error(&cleanup_response[0])["message"]
10292 .as_str()
10293 .unwrap()
10294 .contains("no module connection"));
10295
10296 handler
10297 .cleanup_connection(module_ctx.connection_id)
10298 .unwrap();
10299 }
10300
10301 #[tokio::test]
10302 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
10303 let registry = Arc::new(Registry::default());
10304 let supervisor_handle = SupervisorHandle::new();
10305 let supervisor =
10306 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10307 .with_handle(supervisor_handle.clone())
10308 .with_connection_file_path(
10309 std::env::temp_dir()
10310 .join(format!("subc-route-open-warming-{}", std::process::id())),
10311 );
10312 let module = supervisor
10313 .supervise_configured(
10314 ModuleSpec {
10315 module_id: "warming".to_string(),
10316 program: fake_aft_stub_path(),
10317 args: Vec::new(),
10318 env: Vec::new(),
10319 reserved: false,
10320 reserved_prefixes: Vec::new(),
10321 protocol: ModuleProtocol::Subc,
10322 overlap: Default::default(),
10323 },
10324 true,
10325 )
10326 .unwrap();
10327 assert_eq!(module.state().unwrap(), ModuleState::Running);
10328
10329 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10330 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
10331 let response = handler
10332 .handle_control_frame(
10333 &ctx,
10334 route_open_frame(304, "warming", unique_project_root("warming")),
10335 )
10336 .await
10337 .unwrap();
10338 module.stop().await.unwrap();
10339
10340 assert_eq!(response[0].header.ty, FrameType::Error);
10341 let error = parse_error(&response[0]);
10342 assert_eq!(error["code"], "module_warming");
10343 assert!(error["message"]
10344 .as_str()
10345 .unwrap()
10346 .contains("state=running, enabled=true, live=false"));
10347 }
10348
10349 #[test]
10350 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
10351 let handler = ControlHandler::new(Arc::new(Registry::default()));
10352 let capture = EventCapture::default();
10353 let _subscriber =
10354 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10355 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
10356 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
10357 let pending = (0..limit).collect::<Vec<_>>();
10358 let response = handler
10359 .route_open_capacity_refusal(
10360 &ctx,
10361 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
10362 "busy",
10363 pending.len(),
10364 limit,
10365 )
10366 .unwrap();
10367 assert_eq!(parse_error(&response)["code"], "target_unavailable");
10368 let event = capture
10369 .events()
10370 .into_iter()
10371 .find(|event| {
10372 event.target == "control"
10373 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
10374 })
10375 .expect("connection admission refusal event");
10376 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10377 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10378 }
10379
10380 #[test]
10381 fn route_open_target_cap_logs_admission_reason_and_capacity() {
10382 let handler = ControlHandler::new(Arc::new(Registry::default()));
10383 let capture = EventCapture::default();
10384 let _subscriber =
10385 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10386 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
10387 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
10388 let guards = (0..limit)
10389 .map(|_| {
10390 handler
10391 .route_bind_concurrency
10392 .try_admit("busy", limit)
10393 .unwrap()
10394 })
10395 .collect::<Vec<_>>();
10396 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
10397 Err(in_flight) => in_flight,
10398 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
10399 };
10400 let response = handler
10401 .route_open_target_capacity_refusal(
10402 &ctx,
10403 &route_open_frame(397, "busy", unique_project_root("target-cap")),
10404 "busy",
10405 in_flight,
10406 )
10407 .unwrap();
10408 assert_eq!(parse_error(&response)["code"], "target_unavailable");
10409 let event = capture
10410 .events()
10411 .into_iter()
10412 .find(|event| {
10413 event.target == "control"
10414 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
10415 })
10416 .expect("target admission refusal event");
10417 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10418 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10419 drop(guards);
10420 }
10421
10422 #[tokio::test(flavor = "current_thread")]
10423 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
10424 let handler = ControlHandler::new(Arc::new(Registry::default()));
10425 let capture = EventCapture::default();
10426 let _subscriber =
10427 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10428 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
10429 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10430
10431 let _ = handler
10432 .handle_client_control_request(
10433 &ctx,
10434 frame,
10435 ClientControlRequest::SupervisorSetEnabled {
10436 module_id: "broca".to_string(),
10437 enabled: false,
10438 },
10439 )
10440 .await
10441 .unwrap();
10442
10443 let events = capture
10444 .events()
10445 .into_iter()
10446 .filter(|event| {
10447 event.target == "control"
10448 && event.fields.get("message").map(String::as_str)
10449 == Some("supervisor request received")
10450 })
10451 .collect::<Vec<_>>();
10452 assert_eq!(events.len(), 1, "one supervisor request log line");
10453 let event = &events[0];
10454 assert_eq!(event.level, tracing::Level::INFO);
10455 assert_eq!(
10456 event.fields.get("op"),
10457 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
10458 );
10459 assert_eq!(
10460 event.fields.get("module_id"),
10461 Some(&"\"broca\"".to_string())
10462 );
10463 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
10464 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
10465 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
10466 }
10467
10468 #[tokio::test(flavor = "current_thread")]
10469 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
10470 let registry = Arc::new(Registry::default());
10471 let module_connection = ConnectionId::new(303);
10472 registry
10473 .register_with_control_ops(
10474 manifest("aft", PROTOCOL_VERSION),
10475 PROTOCOL_VERSION,
10476 module_connection,
10477 module_baseline_control_ops(),
10478 )
10479 .unwrap();
10480 let handler = ControlHandler::new(registry);
10481 let capture = EventCapture::default();
10482 let _subscriber =
10483 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10484 let (ctx, _rx) = route_ctx(module_connection);
10485 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10486
10487 let _ = handler
10488 .handle_client_control_request(
10489 &ctx,
10490 frame,
10491 ClientControlRequest::SupervisorSetEnabled {
10492 module_id: "broca".to_string(),
10493 enabled: false,
10494 },
10495 )
10496 .await
10497 .unwrap();
10498
10499 let callers = capture
10500 .events()
10501 .into_iter()
10502 .filter(|event| {
10503 event.fields.get("message").map(String::as_str)
10504 == Some("supervisor request received")
10505 })
10506 .map(|event| event.fields.get("caller").cloned())
10507 .collect::<Vec<_>>();
10508 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
10509 }
10510
10511 #[tokio::test(flavor = "current_thread")]
10512 async fn supervisor_list_does_not_log_request_received() {
10513 let handler = ControlHandler::new(Arc::new(Registry::default()));
10514 let capture = EventCapture::default();
10515 let _subscriber =
10516 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10517 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
10518 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
10519
10520 handler
10521 .handle_client_control_request(&ctx, frame, ClientControlRequest::SupervisorList {})
10522 .await
10523 .unwrap();
10524
10525 assert!(capture.events().into_iter().all(|event| {
10526 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10527 }));
10528 }
10529
10530 #[tokio::test(flavor = "current_thread")]
10531 async fn supervisor_rescan_preview_does_not_log_request_received() {
10532 let handler = ControlHandler::new(Arc::new(Registry::default()));
10533 let capture = EventCapture::default();
10534 let _subscriber =
10535 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10536 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
10537 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
10538
10539 handler
10540 .handle_client_control_request(
10541 &ctx,
10542 frame,
10543 ClientControlRequest::SupervisorRescan { preview: true },
10544 )
10545 .await
10546 .unwrap();
10547
10548 assert!(capture.events().into_iter().all(|event| {
10549 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10550 }));
10551 }
10552
10553 #[tokio::test]
10558 async fn route_open_refusal_names_the_check_that_refused() {
10559 let handler = ControlHandler::new(Arc::new(Registry::default()));
10560 let capture = EventCapture::default();
10561 let _subscriber =
10562 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10563 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10564 let response = handler
10565 .handle_control_frame(
10566 &ctx,
10567 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
10568 )
10569 .await
10570 .unwrap();
10571
10572 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10573 let event = capture
10574 .events()
10575 .into_iter()
10576 .find(|event| {
10577 event.target == "control"
10578 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10579 })
10580 .expect("route.open refusal event");
10581 assert_eq!(
10582 event.fields.get("reason"),
10583 Some(&"\"not_registered\"".to_string())
10584 );
10585 }
10586
10587 #[tokio::test]
10588 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
10589 let registry = Arc::new(Registry::default());
10590 let supervisor_handle = SupervisorHandle::new();
10591 let supervisor =
10592 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10593 .with_handle(supervisor_handle.clone())
10594 .with_connection_file_path(std::env::temp_dir().join(format!(
10595 "subc-route-open-refusal-info-{}",
10596 std::process::id()
10597 )));
10598 let module = supervisor
10599 .supervise_configured(
10600 ModuleSpec {
10601 module_id: "warming".to_string(),
10602 program: fake_aft_stub_path(),
10603 args: Vec::new(),
10604 env: Vec::new(),
10605 reserved: false,
10606 reserved_prefixes: Vec::new(),
10607 protocol: ModuleProtocol::Subc,
10608 overlap: Default::default(),
10609 },
10610 true,
10611 )
10612 .unwrap();
10613 assert_eq!(module.state().unwrap(), ModuleState::Running);
10614
10615 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10616 assert!(handler
10617 .counters()
10618 .snapshot()
10619 .get("route_open_refused_by_code")
10620 .is_none());
10621 let capture = EventCapture::default();
10622 let _subscriber =
10623 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10624 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
10625 let response = handler
10626 .handle_control_frame(
10627 &ctx,
10628 route_open_frame(394, "warming", unique_project_root("refusal-info")),
10629 )
10630 .await
10631 .unwrap();
10632 module.stop().await.unwrap();
10633
10634 assert_eq!(parse_error(&response[0])["code"], "module_warming");
10635 let event = capture
10636 .events()
10637 .into_iter()
10638 .find(|event| {
10639 event.target == "control"
10640 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
10641 })
10642 .expect("route.open refusal event");
10643 assert_eq!(
10644 event.fields.get("module_id"),
10645 Some(&"\"warming\"".to_string())
10646 );
10647 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
10648 assert_eq!(
10649 event.fields.get("reason"),
10650 Some(&"\"supervised_not_registered\"".to_string())
10651 );
10652 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
10653 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
10654 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
10655 assert_eq!(
10656 handler.counters().snapshot()["route_open_refused_by_code"],
10657 json!({ "module_warming": 1 })
10658 );
10659 }
10660
10661 const OUTAGE_START: &str = "route.open refusing module: not serving";
10662 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
10663
10664 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
10665 capture
10666 .events()
10667 .into_iter()
10668 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
10669 .collect()
10670 }
10671
10672 fn supervise_stub(
10673 registry: &Arc<Registry>,
10674 module_id: &str,
10675 enabled: bool,
10676 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
10677 let supervisor_handle = SupervisorHandle::new();
10678 let supervisor =
10679 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
10680 .with_handle(supervisor_handle.clone())
10681 .with_connection_file_path(std::env::temp_dir().join(format!(
10682 "subc-route-outage-{module_id}-{}",
10683 std::process::id()
10684 )));
10685 let module = supervisor
10686 .supervise_configured(
10687 ModuleSpec {
10688 module_id: module_id.to_string(),
10689 program: fake_aft_stub_path(),
10690 args: Vec::new(),
10691 env: Vec::new(),
10692 reserved: false,
10693 reserved_prefixes: Vec::new(),
10694 protocol: ModuleProtocol::Subc,
10695 overlap: Default::default(),
10696 },
10697 enabled,
10698 )
10699 .unwrap();
10700 (supervisor_handle, module)
10701 }
10702
10703 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
10704 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
10705 module_id: module_id.to_string(),
10706 drain_timeout_ms: Some(50),
10707 })
10708 .unwrap();
10709 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
10710 }
10711
10712 #[test]
10716 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
10717 let registry = Arc::new(Registry::default());
10718 let forwarding = Arc::new(ForwardingTable::default());
10719 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10720 let second = ControlHandler::with_forwarding(registry, forwarding);
10721 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
10722 }
10723
10724 #[tokio::test(flavor = "current_thread")]
10728 async fn route_open_unknown_module_refusals_add_no_outage_state() {
10729 let handler = ControlHandler::new(Arc::new(Registry::default()));
10730 let capture = EventCapture::default();
10731 let _subscriber =
10732 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10733 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
10734 for corr in 0..8 {
10735 let response = handler
10736 .handle_control_frame(
10737 &ctx,
10738 route_open_frame(
10739 380 + corr,
10740 &format!("nobody-{corr}"),
10741 unique_project_root("outage-unknown"),
10742 ),
10743 )
10744 .await
10745 .unwrap();
10746 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10747 }
10748
10749 assert_eq!(handler.route_outages.tracked_module_count(), 0);
10750 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
10751 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
10752 }
10753
10754 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10759 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
10760 let registry = Arc::new(Registry::default());
10761 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
10762 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10763 let capture = EventCapture::default();
10764 let _subscriber =
10765 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10766 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
10767 handler.route_outages.record_accepted("outage-restart");
10770
10771 let response = handler
10772 .handle_control_frame(
10773 &ctx,
10774 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
10775 )
10776 .await
10777 .unwrap();
10778 assert_eq!(response[0].header.ty, FrameType::Error);
10779 let starts = outage_lines(&capture, OUTAGE_START);
10780 assert_eq!(starts.len(), 1, "{starts:?}");
10781 assert_eq!(starts[0].level, tracing::Level::WARN);
10782 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
10783 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
10784 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
10785 handler.route_outages.record_accepted("outage-restart");
10786 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
10787
10788 let restart = handler
10789 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
10790 .await
10791 .unwrap();
10792 assert_eq!(
10793 restart[0].header.ty,
10794 FrameType::Response,
10795 "{:?}",
10796 parse_error(&restart[0])
10797 );
10798 handler
10799 .handle_control_frame(
10800 &ctx,
10801 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
10802 )
10803 .await
10804 .unwrap();
10805 module.stop().await.unwrap();
10806
10807 let starts = outage_lines(&capture, OUTAGE_START);
10808 assert_eq!(starts.len(), 2, "{starts:?}");
10809 assert_eq!(starts[1].level, tracing::Level::INFO);
10810 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
10811 }
10812
10813 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
10817 async fn failed_operator_restart_leaves_no_operator_mark() {
10818 let registry = Arc::new(Registry::default());
10819 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
10820 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10821 let capture = EventCapture::default();
10822 let _subscriber =
10823 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10824 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
10825 handler.route_outages.record_accepted("outage-disabled");
10826
10827 let restart = handler
10828 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
10829 .await
10830 .unwrap();
10831 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
10832 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
10833
10834 handler
10835 .handle_control_frame(
10836 &ctx,
10837 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
10838 )
10839 .await
10840 .unwrap();
10841 let starts = outage_lines(&capture, OUTAGE_START);
10842 assert_eq!(starts.len(), 1, "{starts:?}");
10843 assert_eq!(starts[0].level, tracing::Level::WARN);
10844 }
10845
10846 #[tokio::test(flavor = "current_thread")]
10847 async fn route_open_unknown_module_escapes_target_module_id() {
10848 let handler = ControlHandler::new(Arc::new(Registry::default()));
10849 let capture = EventCapture::default();
10850 let _subscriber =
10851 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10852 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
10853 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10854 let response = handler
10855 .handle_control_frame(
10856 &ctx,
10857 route_open_frame(
10858 395,
10859 hostile_module_id,
10860 unique_project_root("hostile-target-module-id"),
10861 ),
10862 )
10863 .await
10864 .unwrap();
10865
10866 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10867 let event = capture
10868 .events()
10869 .into_iter()
10870 .find(|event| {
10871 event.target == "control"
10872 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10873 })
10874 .expect("route.open unknown-module refusal event");
10875 let logged = event.fields.get("module_id").expect("module_id field");
10876 assert!(!logged.bytes().any(|byte| byte < 0x20));
10877 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10878 }
10879
10880 #[tokio::test(flavor = "current_thread")]
10881 async fn route_open_module_rejection_uses_daemon_counter_key() {
10882 let registry = Arc::new(Registry::default());
10883 let forwarding = Arc::new(ForwardingTable::default());
10884 let handler =
10885 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10886 let module_connection = ConnectionId::new(95);
10887 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10888 hello_via_sink(
10889 &handler,
10890 &module_ctx,
10891 &mut module_rx,
10892 hello_frame("aft", PROTOCOL_VERSION, 395),
10893 )
10894 .await;
10895
10896 let client_connection = ConnectionId::new(96);
10897 let (client_ctx, _client_rx) = route_ctx(client_connection);
10898 let capture = EventCapture::default();
10899 let _subscriber =
10900 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10901 let (route_task, bind) = relay_route_open(
10902 &handler,
10903 client_connection,
10904 &client_ctx.egress,
10905 &mut module_rx,
10906 396,
10907 "aft",
10908 "hostile-module-code",
10909 )
10910 .await;
10911 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
10912 let rejection = Frame::build(
10913 FrameType::Error,
10914 control_flags(),
10915 0,
10916 0,
10917 bind.header.corr,
10918 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
10919 )
10920 .unwrap();
10921 handler
10922 .handle_control_frame(&module_ctx, rejection)
10923 .await
10924 .unwrap();
10925
10926 let response = route_task.await.unwrap();
10927 assert_eq!(parse_error(&response[0])["code"], hostile_code);
10928 let counters = handler.counters().snapshot();
10929 assert_eq!(
10930 counters["route_open_refused_by_code"],
10931 json!({ "module_rejected": 1 })
10932 );
10933 assert!(counters["route_open_refused_by_code"]
10934 .get(hostile_code)
10935 .is_none());
10936
10937 let event = capture
10938 .events()
10939 .into_iter()
10940 .find(|event| {
10941 event.target == "control"
10942 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
10943 })
10944 .expect("route.open module-rejection refusal event");
10945 let logged = event.fields.get("module_code").expect("module_code field");
10946 assert!(!logged.bytes().any(|byte| byte < 0x20));
10947 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
10948 }
10949
10950 #[tokio::test]
10951 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
10952 let registry = Arc::new(Registry::default());
10953 let supervisor_handle = SupervisorHandle::new();
10954 let missing_program = std::env::temp_dir().join(format!(
10955 "subc-route-open-missing-program-{}",
10956 std::process::id()
10957 ));
10958 let supervisor =
10959 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10960 .with_handle(supervisor_handle.clone());
10961 let module = supervisor
10962 .supervise_configured(
10963 ModuleSpec {
10964 module_id: "failed".to_string(),
10965 program: missing_program,
10966 args: Vec::new(),
10967 env: Vec::new(),
10968 reserved: false,
10969 reserved_prefixes: Vec::new(),
10970 protocol: ModuleProtocol::Subc,
10971 overlap: Default::default(),
10972 },
10973 true,
10974 )
10975 .unwrap();
10976 assert_eq!(module.state().unwrap(), ModuleState::Failed);
10977
10978 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10979 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
10980 let response = handler
10981 .handle_control_frame(
10982 &ctx,
10983 route_open_frame(305, "failed", unique_project_root("failed")),
10984 )
10985 .await
10986 .unwrap();
10987
10988 assert_eq!(response[0].header.ty, FrameType::Error);
10989 let error = parse_error(&response[0]);
10990 assert_eq!(error["code"], "target_unavailable");
10991 assert!(error["message"]
10992 .as_str()
10993 .unwrap()
10994 .contains("state=failed, enabled=true, live=false"));
10995 }
10996
10997 #[tokio::test]
10998 async fn route_open_role_mismatch_remains_target_unavailable() {
10999 let registry = Arc::new(Registry::default());
11000 let handler = ControlHandler::new(Arc::clone(®istry));
11001 handler
11002 .handle_control(
11003 ConnectionId::new(41),
11004 non_routable_hello_frame_with_control_ops("health-only", 306, None),
11005 )
11006 .unwrap();
11007
11008 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
11009 let response = handler
11010 .handle_control_frame(
11011 &ctx,
11012 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
11013 )
11014 .await
11015 .unwrap();
11016
11017 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11018 assert!(parse_error(&response[0])["message"]
11019 .as_str()
11020 .unwrap()
11021 .contains("does not provide the requested target"));
11022 }
11023
11024 #[tokio::test]
11025 async fn route_open_inactive_registration_remains_target_unavailable() {
11026 let registry = Arc::new(Registry::default());
11027 let handler = ControlHandler::new(Arc::clone(®istry));
11028 handler
11029 .handle_control(
11030 ConnectionId::new(43),
11031 hello_frame("inactive", PROTOCOL_VERSION, 308),
11032 )
11033 .unwrap();
11034 assert!(registry
11035 .set_module_state_for_test("inactive", ChannelState::Closed)
11036 .unwrap());
11037
11038 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
11039 let response = handler
11040 .handle_control_frame(
11041 &ctx,
11042 route_open_frame(309, "inactive", unique_project_root("inactive")),
11043 )
11044 .await
11045 .unwrap();
11046
11047 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11048 assert!(parse_error(&response[0])["message"]
11049 .as_str()
11050 .unwrap()
11051 .contains("is not active"));
11052 }
11053
11054 #[tokio::test]
11055 async fn late_health_reply_is_recorded_through_the_module_response_path() {
11056 let registry = Arc::new(Registry::default());
11057 let forwarding = Arc::new(ForwardingTable::default());
11058 let supervisor_handle = SupervisorHandle::new();
11059 let supervisor =
11060 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
11061 .with_forwarding(Arc::clone(&forwarding))
11062 .with_handle(supervisor_handle.clone());
11063 let module = supervisor
11064 .supervise_configured(
11065 crate::ModuleSpec {
11066 module_id: "late-health-response".to_string(),
11067 program: PathBuf::from("disabled-module"),
11068 args: Vec::new(),
11069 env: Vec::new(),
11070 reserved: false,
11071 reserved_prefixes: Vec::new(),
11072 protocol: ModuleProtocol::Subc,
11073 overlap: Default::default(),
11074 },
11075 false,
11076 )
11077 .unwrap();
11078 let handler =
11079 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11080 .with_supervisor(supervisor_handle);
11081 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
11082 handler
11083 .handle_control_frame(
11084 &module_ctx,
11085 hello_frame_with_control_ops(
11086 "late-health-response",
11087 PROTOCOL_VERSION,
11088 7,
11089 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11090 ),
11091 )
11092 .await
11093 .unwrap();
11094 let probe_started_at = Instant::now() - Duration::from_millis(80);
11095 let pending = forwarding
11096 .begin_health_probe_rpc_for(
11097 "late-health-response",
11098 MODULE_CONTROL_OP_HEALTH_CHECK,
11099 probe_started_at,
11100 Instant::now() - Duration::from_millis(1),
11101 )
11102 .unwrap();
11103 assert!(forwarding
11104 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
11105 .unwrap());
11106
11107 let responses = handler
11108 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
11109 .await
11110 .unwrap();
11111
11112 assert!(responses.is_empty());
11113 let health = module.status().unwrap().health;
11114 assert_eq!(health.late_answer_count, 1);
11115 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
11116 }
11117
11118 #[tokio::test]
11119 async fn health_probe_timeout_and_module_death_are_typed() {
11120 let registry = Arc::new(Registry::default());
11121 let forwarding = Arc::new(ForwardingTable::default());
11122 let handler =
11123 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11124 .with_health_probe_timeout(Duration::from_millis(50));
11125 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11126 hello_via_sink(
11127 &handler,
11128 &module_ctx,
11129 &mut module_rx,
11130 hello_frame_with_control_ops(
11131 "aft",
11132 PROTOCOL_VERSION,
11133 7,
11134 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11135 ),
11136 )
11137 .await;
11138
11139 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11140 let responses = handler
11141 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
11142 .await
11143 .unwrap();
11144 assert_eq!(responses[0].header.ty, FrameType::Error);
11145 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
11146 let _ = module_rx.try_recv();
11147
11148 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
11149 let health_handler = handler.clone();
11150 let death_task = tokio::spawn(async move {
11151 health_handler
11152 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
11153 .await
11154 .unwrap()
11155 });
11156 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11157 .await
11158 .unwrap()
11159 .unwrap();
11160 handler
11161 .cleanup_connection(module_ctx.connection_id)
11162 .unwrap();
11163 let responses = death_task.await.unwrap();
11164 assert_eq!(responses[0].header.ty, FrameType::Error);
11165 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
11166 }
11167
11168 #[test]
11169 fn hello_requires_exact_protocol_version() {
11170 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
11171 let registry = Arc::new(Registry::default());
11172 let handler = ControlHandler::new(Arc::clone(®istry));
11173 let responses = handler
11174 .handle_control(
11175 ConnectionId::new(connection),
11176 hello_frame("aft", offered, 9),
11177 )
11178 .unwrap();
11179
11180 assert_eq!(responses.len(), 1);
11181 assert_eq!(responses[0].header.ty, FrameType::Error);
11182 let error = parse_error(&responses[0]);
11183 assert_eq!(error["code"], "version_unsupported");
11184 assert!(registry.get_module("aft").unwrap().is_none());
11185 assert_eq!(registry.active_registration_count().unwrap(), 0);
11186 }
11187 }
11188
11189 #[test]
11190 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
11191 let registry = Arc::new(Registry::default());
11192 let forwarding = Arc::new(ForwardingTable::default());
11193 let handler =
11194 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11195 let module_connection = ConnectionId::new(301);
11196 let registration = registry
11197 .register_with_control_ops(
11198 manifest("aft-push", PROTOCOL_VERSION),
11199 PROTOCOL_VERSION,
11200 module_connection,
11201 module_baseline_control_ops(),
11202 )
11203 .unwrap();
11204 let (module_tx, _module_rx) = mpsc::channel(8);
11205 let endpoint = forwarding
11206 .register_module_connection(
11207 module_connection,
11208 "aft-push".to_string(),
11209 PROTOCOL_VERSION,
11210 manifest_concurrency(®istration.manifest),
11211 FrameSink::new(module_tx),
11212 )
11213 .unwrap();
11214
11215 let unknown = Frame::build(
11217 FrameType::Push,
11218 control_flags(),
11219 0,
11220 0,
11221 5,
11222 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
11223 )
11224 .unwrap();
11225 let out = handler.handle_status_update(endpoint, unknown).unwrap();
11226 assert!(
11227 out.is_empty(),
11228 "unknown push op must be ignored, got {out:?}"
11229 );
11230
11231 let malformed = Frame::build(
11233 FrameType::Push,
11234 control_flags(),
11235 0,
11236 0,
11237 6,
11238 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
11239 )
11240 .unwrap();
11241 let out = handler.handle_status_update(endpoint, malformed).unwrap();
11242 assert_eq!(out.len(), 1);
11243 assert_eq!(out[0].header.ty, FrameType::Error);
11244 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
11245 }
11246
11247 #[test]
11248 fn hello_rejected_when_connection_already_owns_client_routes() {
11249 let registry = Arc::new(Registry::default());
11250 let forwarding = Arc::new(ForwardingTable::default());
11251 let handler =
11252 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11253 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
11255 let client_connection = ConnectionId::new(202);
11256
11257 let responses = handler
11260 .handle_control(
11261 client_connection,
11262 hello_frame("aft-second", PROTOCOL_VERSION, 9),
11263 )
11264 .unwrap();
11265 assert_eq!(responses[0].header.ty, FrameType::Error);
11266 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
11267 assert!(registry.get_module("aft-second").unwrap().is_none());
11268 }
11269
11270 #[tokio::test]
11271 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
11272 let registry = Arc::new(Registry::default());
11273 let forwarding = Arc::new(ForwardingTable::default());
11274 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
11275 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
11276 hello_via_sink(
11277 &handler,
11278 &module_ctx,
11279 &mut module_rx,
11280 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
11281 )
11282 .await;
11283 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
11284 let pending = forwarding
11285 .begin_route_bind_relay_for_test(
11286 client_ctx.connection_id,
11287 client_ctx.egress.clone(),
11288 2,
11289 "alpha",
11290 )
11291 .unwrap();
11292 forwarding
11293 .complete_pending_relay(
11294 module_ctx.connection_id,
11295 pending.corr,
11296 RouteBindRelayOutcome::Accepted,
11297 )
11298 .unwrap();
11299 client_rx.try_recv().unwrap();
11300 for module_id in ["beta", "alpha"] {
11301 let replies = handler
11302 .handle_control_frame(
11303 &module_ctx,
11304 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
11305 )
11306 .await
11307 .unwrap();
11308 assert_eq!(replies.len(), 1, "second HELLO must be refused");
11309 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
11310 }
11311 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
11312 assert!(registry.get_module("beta").unwrap().is_none());
11313 assert!(matches!(
11314 forwarding
11315 .lookup_data_route(
11316 client_ctx.connection_id,
11317 pending.client_channel,
11318 pending.client_epoch,
11319 )
11320 .unwrap(),
11321 DataRoute::Client(DataRouteState::Bound(_))
11322 ));
11323 assert!(handler
11324 .hello_launch_nonces
11325 .lock()
11326 .unwrap()
11327 .presented(module_ctx.connection_id, Some("alpha-nonce")));
11328 assert!(module_rx.try_recv().is_err());
11329 }
11330
11331 #[test]
11332 fn reserved_module_hello_requires_matching_launch_nonce() {
11333 let registry = Arc::new(Registry::default());
11334 let supervisor = SupervisorHandle::new();
11335 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
11338 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11339
11340 let no_nonce = handler
11342 .handle_control(
11343 ConnectionId::new(1),
11344 hello_frame("vault", PROTOCOL_VERSION, 1),
11345 )
11346 .unwrap();
11347 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
11348 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
11349 assert!(registry.get_module("vault").unwrap().is_none());
11350
11351 let wrong = handler
11353 .handle_control(
11354 ConnectionId::new(2),
11355 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
11356 )
11357 .unwrap();
11358 assert_eq!(wrong[0].header.ty, FrameType::Error);
11359 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
11360 assert!(registry.get_module("vault").unwrap().is_none());
11361
11362 let ok = handler
11364 .handle_control(
11365 ConnectionId::new(3),
11366 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
11367 )
11368 .unwrap();
11369 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
11370 assert!(registry.get_module("vault").unwrap().is_some());
11371 }
11372
11373 #[test]
11374 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
11375 let registry = Arc::new(Registry::default());
11376 let supervisor = SupervisorHandle::new();
11377 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11378 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11379 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11380
11381 let squat = handler
11382 .handle_control(
11383 ConnectionId::new(1),
11384 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
11385 )
11386 .unwrap();
11387 assert_eq!(squat[0].header.ty, FrameType::Error);
11388 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
11389 assert!(parse_error(&squat[0])["message"]
11390 .as_str()
11391 .unwrap()
11392 .contains("fed:"));
11393
11394 let accepted_peer = handler
11395 .handle_control(
11396 ConnectionId::new(2),
11397 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
11398 )
11399 .unwrap();
11400 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
11401
11402 let accepted_short = handler
11403 .handle_control(
11404 ConnectionId::new(3),
11405 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
11406 )
11407 .unwrap();
11408 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
11409
11410 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
11411 let response = handler
11412 .handle_control(
11413 ConnectionId::new(conn),
11414 hello_frame(module_id, PROTOCOL_VERSION, conn),
11415 )
11416 .unwrap();
11417 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
11418 }
11419 }
11420
11421 #[test]
11422 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
11423 let registry = Arc::new(Registry::default());
11424 let supervisor = SupervisorHandle::new();
11425 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11426 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11427 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
11428 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11429
11430 let owner_nonce = handler
11431 .handle_control(
11432 ConnectionId::new(1),
11433 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
11434 )
11435 .unwrap();
11436 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
11437 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
11438 assert!(registry.get_module("fed:special").unwrap().is_none());
11439
11440 let exact_nonce = handler
11441 .handle_control(
11442 ConnectionId::new(2),
11443 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
11444 )
11445 .unwrap();
11446 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
11447 assert!(registry.get_module("fed:special").unwrap().is_some());
11448 }
11449
11450 #[test]
11451 fn non_reserved_module_ignores_launch_nonce() {
11452 let registry = Arc::new(Registry::default());
11453 let handler = ControlHandler::new(Arc::clone(®istry));
11456 let no_nonce = handler
11457 .handle_control(
11458 ConnectionId::new(1),
11459 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
11460 )
11461 .unwrap();
11462 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
11463 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
11464
11465 let echoed_nonce = handler
11466 .handle_control(
11467 ConnectionId::new(2),
11468 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
11469 )
11470 .unwrap();
11471 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
11472 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
11473 }
11474
11475 #[test]
11476 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
11477 let handler = ControlHandler::default();
11478 let conn = ConnectionId::new(1);
11479 let malformed = Frame::build(
11480 FrameType::Hello,
11481 control_flags(),
11482 0,
11483 0,
11484 3,
11485 b"{not json".to_vec(),
11486 )
11487 .unwrap();
11488
11489 let error = handler.handle_control(conn, malformed).unwrap();
11490 assert_eq!(error[0].header.ty, FrameType::Error);
11491 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
11492
11493 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11494 let pong = handler.handle_control(conn, ping).unwrap();
11495 assert_eq!(pong[0].header.ty, FrameType::Pong);
11496 assert_eq!(pong[0].header.corr, 4);
11497 }
11498
11499 #[test]
11500 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
11501 let registry = Arc::new(Registry::default());
11502 let handler = ControlHandler::new(Arc::clone(®istry));
11503
11504 handler
11505 .handle_control(
11506 ConnectionId::new(1),
11507 hello_frame("aft", PROTOCOL_VERSION, 1),
11508 )
11509 .unwrap();
11510 let duplicate = handler
11511 .handle_control(
11512 ConnectionId::new(2),
11513 hello_frame("aft", PROTOCOL_VERSION, 2),
11514 )
11515 .unwrap();
11516
11517 assert_eq!(duplicate[0].header.ty, FrameType::Error);
11518 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11519 let registration = registry.get_module("aft").unwrap().unwrap();
11520 assert_eq!(registration.connection_id, ConnectionId::new(1));
11521 }
11522
11523 #[test]
11524 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
11525 let registry = Arc::new(Registry::default());
11526 let forwarding = Arc::new(ForwardingTable::default());
11527 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
11528 let handler =
11529 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11530 .with_process_liveness(process_liveness);
11531 let (ctx, route_channel, route_epoch) =
11532 bind_liveness_route(®istry, &forwarding, "aft-dead");
11533 let responses = handler
11534 .handle_route_poll(
11535 &ctx,
11536 route_poll_frame(41, PollKind::Liveness, route_channel),
11537 route_channel,
11538 route_epoch,
11539 PollKind::Liveness,
11540 )
11541 .unwrap();
11542
11543 assert_eq!(responses.len(), 1);
11544 assert_eq!(responses[0].header.ty, FrameType::Response);
11545 assert_route_poll_liveness(&responses[0], false);
11546 }
11547
11548 #[test]
11549 fn liveness_poll_without_process_source_uses_bound_route() {
11550 let registry = Arc::new(Registry::default());
11551 let forwarding = Arc::new(ForwardingTable::default());
11552 let handler =
11553 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11554 let (ctx, route_channel, route_epoch) =
11555 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
11556 let responses = handler
11557 .handle_route_poll(
11558 &ctx,
11559 route_poll_frame(42, PollKind::Liveness, route_channel),
11560 route_channel,
11561 route_epoch,
11562 PollKind::Liveness,
11563 )
11564 .unwrap();
11565
11566 assert_route_poll_liveness(&responses[0], true);
11567 }
11568
11569 #[test]
11570 fn liveness_poll_untracked_process_source_uses_bound_route() {
11571 let registry = Arc::new(Registry::default());
11572 let forwarding = Arc::new(ForwardingTable::default());
11573 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
11574 let handler =
11575 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11576 .with_process_liveness(process_liveness);
11577 let (ctx, route_channel, route_epoch) =
11578 bind_liveness_route(®istry, &forwarding, "aft-untracked");
11579 let responses = handler
11580 .handle_route_poll(
11581 &ctx,
11582 route_poll_frame(43, PollKind::Liveness, route_channel),
11583 route_channel,
11584 route_epoch,
11585 PollKind::Liveness,
11586 )
11587 .unwrap();
11588
11589 assert_route_poll_liveness(&responses[0], true);
11590 }
11591
11592 #[tokio::test]
11593 async fn unknown_op_returns_unknown_control_op() {
11594 let handler = ControlHandler::default();
11595 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
11596 let request = Frame::build(
11597 FrameType::Request,
11598 control_flags(),
11599 0,
11600 0,
11601 55,
11602 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
11603 )
11604 .unwrap();
11605
11606 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11607
11608 assert_eq!(response.len(), 1);
11609 assert_eq!(response[0].header.ty, FrameType::Error);
11610 assert_eq!(response[0].header.corr, 55);
11611 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
11612 }
11613
11614 #[tokio::test]
11615 async fn supervisor_provenance_rejects_unknown_exact_module() {
11616 let handler = ControlHandler::default();
11617 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
11618 let request = Frame::build(
11619 FrameType::Request,
11620 control_flags(),
11621 0,
11622 0,
11623 57,
11624 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
11625 )
11626 .unwrap();
11627
11628 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11629
11630 assert_eq!(response.len(), 1);
11631 assert_eq!(response[0].header.ty, FrameType::Error);
11632 assert_eq!(response[0].header.corr, 57);
11633 let error = parse_error(&response[0]);
11634 assert_eq!(error["code"], "unknown_module");
11635 assert_eq!(error["message"], "module_id 'missing' is not supervised");
11636 }
11637
11638 #[test]
11639 fn provenance_probe_override_keeps_handler_tests_deterministic() {
11640 let expected = subc_control::RunningImageAgreement::Unavailable {
11641 reason: subc_control::RunningImageUnavailableReason::HashFailed,
11642 };
11643 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
11644 assert_eq!(handler.provenance_probe_override, Some(expected));
11645 }
11646
11647 #[test]
11648 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
11649 let verdict = reload_verdict(
11650 std::path::Path::new("/bin/new"),
11651 Some(std::path::Path::new("/bin/old")),
11652 subc_control::RunningImageAgreement::Unavailable {
11653 reason: subc_control::RunningImageUnavailableReason::HashFailed,
11654 },
11655 );
11656 assert!(matches!(
11657 verdict.path,
11658 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
11659 if configured == std::path::Path::new("/bin/new")
11660 && spawned_from == std::path::Path::new("/bin/old")
11661 ));
11662 }
11663
11664 #[cfg(windows)]
11665 #[test]
11666 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
11667 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
11668 let configured = dir.join("module.exe");
11669 std::fs::write(&configured, b"executable path fixture").unwrap();
11670 let spawned = std::fs::canonicalize(&configured).unwrap();
11671 assert_ne!(
11672 configured, spawned,
11673 "fixture must exercise different path spellings"
11674 );
11675 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
11676 let lower_case = configured.to_string_lossy().to_lowercase();
11677 for spelling in [
11678 &configured,
11679 Path::new(&forward_slashes),
11680 Path::new(&lower_case),
11681 ] {
11682 let verdict = reload_verdict(
11683 spelling,
11684 Some(&spawned),
11685 subc_control::RunningImageAgreement::Unavailable {
11686 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11687 },
11688 );
11689 assert_eq!(
11690 verdict.path,
11691 ReloadPathAgreement::Match,
11692 "configured {spelling:?}, spawned {spawned:?}"
11693 );
11694 }
11695 }
11696
11697 #[cfg(windows)]
11698 #[test]
11699 fn windows_reload_path_keeps_different_existing_program_pending() {
11700 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
11701 let old = dir.join("old.exe");
11702 let new = dir.join("new.exe");
11703 std::fs::write(&old, b"same executable content").unwrap();
11704 std::fs::copy(&old, &new).unwrap();
11705 let spawned = std::fs::canonicalize(&old).unwrap();
11706 let image = subc_control::RunningImageAgreement::Unavailable {
11707 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11708 };
11709 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
11710 assert!(
11711 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
11712 if configured == new && spawned_from == spawned)
11713 );
11714 let missing = dir.join("missing.exe");
11715 assert!(matches!(
11716 reload_verdict(&missing, Some(&spawned), image).path,
11717 ReloadPathAgreement::Mismatch { .. }
11718 ));
11719 }
11720
11721 #[test]
11722 fn reload_verdict_detects_replaced_image_at_same_path() {
11723 let image = subc_control::RunningImageAgreement::Mismatch {
11724 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
11725 digest: "old".into(),
11726 },
11727 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
11728 digest: "new".into(),
11729 },
11730 };
11731 let verdict = reload_verdict(
11732 std::path::Path::new("/bin/same"),
11733 Some(std::path::Path::new("/bin/same")),
11734 image.clone(),
11735 );
11736 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11737 assert_eq!(verdict.image, image);
11738 }
11739
11740 #[test]
11741 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
11742 let image = subc_control::RunningImageAgreement::Unavailable {
11743 reason: subc_control::RunningImageUnavailableReason::NotRunning,
11744 };
11745 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
11746 assert_eq!(
11747 verdict.path,
11748 subc_control::ReloadPathAgreement::Unavailable {
11749 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
11750 }
11751 );
11752 assert_eq!(verdict.image, image);
11753
11754 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
11755 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
11756 };
11757 let verdict = reload_verdict(
11758 std::path::Path::new("/bin/same"),
11759 Some(std::path::Path::new("/bin/same")),
11760 unconfirmed.clone(),
11761 );
11762 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11763 assert_eq!(verdict.image, unconfirmed);
11764 }
11765
11766 #[test]
11767 fn reload_verdict_preserves_each_image_unavailability_reason() {
11768 use subc_control::RunningImageUnavailableReason as Reason;
11769
11770 for reason in [
11771 Reason::NotRunning,
11772 Reason::UnsupportedPlatform,
11773 Reason::RunningExecutableUnreadable,
11774 Reason::SpawnedPathUnreadable,
11775 Reason::HashFailed,
11776 Reason::ProcessIdentityUnconfirmed,
11777 Reason::Unknown("future_probe_reason".to_string()),
11778 ] {
11779 let image = subc_control::RunningImageAgreement::Unavailable {
11780 reason: reason.clone(),
11781 };
11782 let verdict = reload_verdict(
11783 std::path::Path::new("/bin/same"),
11784 Some(std::path::Path::new("/bin/same")),
11785 image.clone(),
11786 );
11787 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
11788 assert_eq!(verdict.image, image, "{reason:?}");
11789 }
11790 }
11791
11792 #[tokio::test]
11793 async fn malformed_control_bodies_return_invalid_control_body() {
11794 let handler = ControlHandler::default();
11795 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
11796
11797 for (corr, body) in [
11798 (56, br#"{"route_channel":1}"#.as_slice()),
11799 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
11800 (
11801 58,
11802 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
11803 ),
11804 ] {
11805 let request = Frame::build(
11806 FrameType::Request,
11807 control_flags(),
11808 0,
11809 0,
11810 corr,
11811 body.to_vec(),
11812 )
11813 .unwrap();
11814 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
11815
11816 assert_eq!(response.len(), 1);
11817 assert_eq!(response[0].header.ty, FrameType::Error);
11818 assert_eq!(response[0].header.corr, corr);
11819 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
11820 }
11821 }
11822
11823 #[tokio::test]
11824 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
11825 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11826 let registry = Arc::new(Registry::default());
11827 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11828 let router = Router::with_control_handler(Arc::clone(&control));
11829 let connection = router.begin_connection();
11830 let (ctx, mut rx) = route_ctx(connection.id());
11831
11832 router
11833 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
11834 .await
11835 .unwrap();
11836 let response = rx.recv().await.unwrap();
11837 let ack = parse_ack(&response);
11838 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11839 let channel = 1;
11840
11841 let goodbye =
11842 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
11843 router.route_for_connection(&ctx, goodbye).await.unwrap();
11844 assert!(rx.try_recv().is_err());
11845 assert!(registry.get_module("aft").unwrap().is_none());
11846
11847 router
11848 .route_for_connection(&ctx, channel_request(channel, 13))
11849 .await
11850 .unwrap();
11851 let error_frame = rx.recv().await.unwrap();
11852 assert_eq!(error_frame.header.ty, FrameType::Error);
11853 assert_eq!(error_frame.header.channel, channel);
11854 let captured = crate::router::test_log::captured_logs(&logs);
11855 assert_eq!(
11856 captured
11857 .lines()
11858 .filter(|line| {
11859 line.contains("module_id=aft")
11860 && line.contains("reason=explicit_goodbye")
11861 && line.contains("module registration ended")
11862 })
11863 .count(),
11864 1,
11865 "unexpected GOODBYE registry log: {captured}"
11866 );
11867 }
11868
11869 #[tokio::test]
11870 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
11871 let registry = Arc::new(Registry::default());
11872 let handler = ControlHandler::new(registry).with_capability_config(
11873 [("prov".to_string(), true), ("cons".to_string(), true)],
11874 BTreeMap::new(),
11875 );
11876 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
11877 register_capability_manifest(
11878 &handler,
11879 &provider_ctx,
11880 &mut provider_rx,
11881 capability_manifest("prov", &["thing/v1"], &[]),
11882 1,
11883 )
11884 .await;
11885 let mut consumer = capability_manifest("cons", &[], &[]);
11886 consumer.capabilities.as_mut().unwrap().requires.push(
11887 subc_protocol::manifest::CapabilityRequirement {
11888 capability: "thing/v1".to_string(),
11889 need: subc_protocol::manifest::CapabilityNeed::Required,
11890 },
11891 );
11892 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
11893 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
11894 assert_eq!(
11895 handler.capability_evaluator.verdict("cons", "thing/v1"),
11896 Some(CapabilityVerdict::Provided)
11897 );
11898 let (mut client_rx, _) = open_route_for_capability_test(
11899 &handler,
11900 &provider_ctx,
11901 &mut provider_rx,
11902 703,
11903 3,
11904 "prov",
11905 None,
11906 )
11907 .await;
11908 let goodbye =
11909 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11910 handler
11911 .handle_control_frame(&provider_ctx, goodbye)
11912 .await
11913 .unwrap();
11914 assert_eq!(
11915 handler.capability_evaluator.verdict("cons", "thing/v1"),
11916 Some(CapabilityVerdict::NeverProvided)
11917 );
11918 let closed = client_rx
11919 .try_recv()
11920 .expect("GOODBYE pushes route.closed before route GOODBYE");
11921 assert!(
11922 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
11923 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
11924 );
11925 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
11926 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11927 }
11928
11929 #[tokio::test]
11930 async fn dropping_router_connection_releases_registration() {
11931 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11932 let registry = Arc::new(Registry::default());
11933 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
11934 let router = Router::with_control_handler(Arc::clone(&control));
11935 let connection = router.begin_connection();
11936 let connection_id = connection.id();
11937 let (ctx, mut rx) = route_ctx(connection_id);
11938
11939 router
11940 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
11941 .await
11942 .unwrap();
11943 let response = rx.recv().await.unwrap();
11944 let ack = parse_ack(&response);
11945 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
11946 assert!(registry.get_module("aft").unwrap().is_some());
11947
11948 drop(connection);
11949
11950 assert!(registry.get_module("aft").unwrap().is_none());
11951 assert_eq!(registry.active_registration_count().unwrap(), 0);
11952
11953 control
11954 .cleanup_connection(ConnectionId::new(u64::MAX))
11955 .unwrap();
11956 let captured = crate::router::test_log::captured_logs(&logs);
11957 println!("captured registry lifecycle logs:\n{captured}");
11958 let events: Vec<_> = captured
11959 .lines()
11960 .filter(|line| {
11961 line.contains("module registered module_id=aft ")
11962 || line.contains("module registration ended")
11963 })
11964 .collect();
11965 assert_eq!(
11966 events.len(),
11967 2,
11968 "unexpected registry lifecycle logs: {captured}"
11969 );
11970 assert!(events[0].contains("module registered module_id=aft "));
11971 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
11972 assert!(events[1].contains(&format!(
11973 "module_id=aft connection_id={}",
11974 connection_id.get()
11975 )));
11976 assert!(events[1].contains("reason=connection_closed"));
11977 assert!(events[1].contains("module registration ended"));
11978 assert_eq!(
11979 captured
11980 .lines()
11981 .filter(|line| line.contains("module registered module_id=aft "))
11982 .count(),
11983 1,
11984 "legacy registration admission line must appear once: {captured}"
11985 );
11986 }
11987
11988 #[tokio::test]
11989 async fn hello_registration_keeps_legacy_module_registered_line_once() {
11990 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
11991 let registry = Arc::new(Registry::default());
11992 let control = ControlHandler::new(Arc::clone(®istry));
11993 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
11994 hello_via_sink(
11995 &control,
11996 &ctx,
11997 &mut rx,
11998 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
11999 )
12000 .await;
12001
12002 let captured = crate::router::test_log::captured_logs(&logs);
12003 let admissions: Vec<_> = captured
12004 .lines()
12005 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
12006 .collect();
12007 assert_eq!(
12008 admissions.len(),
12009 1,
12010 "legacy admission line must remain exactly once: {captured}"
12011 );
12012 assert!(admissions[0].contains("routable_provider=true"));
12013 assert!(admissions[0].contains("connection_id=777"));
12014 }
12015
12016 fn capability_manifest(
12017 module_id: &str,
12018 provides: &[&str],
12019 must_never_reach: &[&str],
12020 ) -> ModuleManifest {
12021 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
12022 manifest.capabilities = Some(CapabilityDeclarations {
12023 provides: provides
12024 .iter()
12025 .map(|capability| (*capability).to_string())
12026 .collect(),
12027 requires: Vec::new(),
12028 must_never_reach: must_never_reach
12029 .iter()
12030 .map(|capability| (*capability).to_string())
12031 .collect(),
12032 });
12033 manifest
12034 }
12035
12036 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
12037 Frame::build(
12038 FrameType::Hello,
12039 control_flags(),
12040 0,
12041 0,
12042 corr,
12043 serde_json::to_vec(&ModuleHelloBody {
12044 protocol_ver: manifest.protocol_ver,
12045 manifest,
12046 control_ops: None,
12047 launch_nonce: None,
12048 })
12049 .expect("capability test HELLO serializes"),
12050 )
12051 .expect("capability test HELLO frame builds")
12052 }
12053
12054 fn catalog_update_with_capabilities_frame(
12055 corr: u64,
12056 capabilities: CapabilityDeclarations,
12057 ) -> Frame {
12058 Frame::build(
12059 FrameType::Request,
12060 control_flags(),
12061 0,
12062 0,
12063 corr,
12064 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
12065 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
12066 capabilities: Some(capabilities),
12067 ready: None,
12068 })
12069 .expect("capability catalog.update serializes"),
12070 )
12071 .expect("capability catalog.update frame builds")
12072 }
12073
12074 async fn register_capability_manifest(
12075 handler: &ControlHandler,
12076 ctx: &RouteCtx,
12077 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
12078 manifest: ModuleManifest,
12079 corr: u64,
12080 ) {
12081 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
12082 }
12083
12084 async fn open_route_for_capability_test(
12085 handler: &ControlHandler,
12086 target_ctx: &RouteCtx,
12087 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
12088 client_connection_id: u64,
12089 corr: u64,
12090 target_module_id: &str,
12091 consumer_identity: Option<ConsumerIdentity>,
12092 ) -> (
12093 mpsc::Receiver<crate::router::OutboundFrame>,
12094 ModuleControlRequest,
12095 ) {
12096 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
12097 let route_handler = handler.clone();
12098 let target_module_id = target_module_id.to_string();
12099 let route_task = tokio::spawn(async move {
12100 route_handler
12101 .handle_control_frame(
12102 &client_ctx,
12103 route_open_frame_with_admission_facts(
12104 corr,
12105 &target_module_id,
12106 unique_project_root("admission-facts"),
12107 consumer_identity,
12108 None,
12109 ),
12110 )
12111 .await
12112 .expect("capability test route.open succeeds")
12113 });
12114 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
12115 .await
12116 .expect("capability test route.open must reach route.bind")
12117 .expect("target control receiver stays open");
12118 let bind_request: ModuleControlRequest =
12119 serde_json::from_slice(&bind.body).expect("route.bind decodes");
12120 handler
12121 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
12122 .await
12123 .expect("capability test route.bind ACK succeeds");
12124 assert!(route_task.await.expect("route.open task joins").is_empty());
12125 let opened = client_rx
12126 .recv()
12127 .await
12128 .expect("successful route.open publishes a response");
12129 assert!(matches!(
12130 serde_json::from_slice::<ClientControlResponse>(&opened.body),
12131 Ok(ClientControlResponse::RouteOpen { .. })
12132 ));
12133 (client_rx, bind_request)
12134 }
12135
12136 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
12137 assert_eq!(frame.header.ty, FrameType::Push);
12138 assert_eq!(frame.header.channel, 0);
12139 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
12140 .expect("route.closed control push decodes");
12141 let ClientControlPush::RouteClosed { channels, .. } = &push else {
12142 panic!("expected route.closed");
12143 };
12144 assert_eq!(channels.len(), 1, "exactly one violating route closed");
12145 let channels = channels.clone();
12146 assert_eq!(
12147 push,
12148 ClientControlPush::RouteClosed {
12149 module_id: target_module_id.to_string(),
12150 channels,
12151 reason: RouteCloseReason::CapabilityDenied,
12152 drained: false,
12153 abandoned: 0,
12154 excluded_subscriptions: 0,
12155 terminal: Some(false),
12156 }
12157 );
12158 }
12159
12160 #[tokio::test]
12161 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
12162 let registry = Arc::new(Registry::default());
12163 let forwarding = Arc::new(ForwardingTable::default());
12164 let supervisor = SupervisorHandle::new();
12165 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12166 let handler =
12167 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12168 .with_supervisor(supervisor);
12169 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
12170 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
12171 register_capability_manifest(
12172 &handler,
12173 &target_ctx,
12174 &mut target_rx,
12175 capability_manifest("target", &["credentials-provider/v1"], &[]),
12176 1,
12177 )
12178 .await;
12179 register_capability_manifest(
12180 &handler,
12181 &opener_ctx,
12182 &mut opener_rx,
12183 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12184 2,
12185 )
12186 .await;
12187
12188 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
12189 let replies = handler
12190 .handle_control_frame(
12191 &client_ctx,
12192 route_open_frame_with_admission_facts(
12193 3,
12194 "target",
12195 unique_project_root("admission-facts"),
12196 Some(ConsumerIdentity {
12197 module_id: "opener".to_string(),
12198 launch_nonce: "opener-nonce".to_string(),
12199 }),
12200 None,
12201 ),
12202 )
12203 .await
12204 .expect("denied route.open returns a typed frame");
12205 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
12206 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12207 assert!(
12208 target_rx.try_recv().is_err(),
12209 "forbidden route.open must not relay route.bind"
12210 );
12211 }
12212
12213 #[tokio::test]
12214 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
12215 let registry = Arc::new(Registry::default());
12216 let forwarding = Arc::new(ForwardingTable::default());
12217 let supervisor = SupervisorHandle::new();
12218 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12219 let handler =
12220 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12221 .with_supervisor(supervisor);
12222 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
12223 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
12224 register_capability_manifest(
12225 &handler,
12226 &target_ctx,
12227 &mut target_rx,
12228 capability_manifest("target", &["credentials-provider/v1"], &[]),
12229 1,
12230 )
12231 .await;
12232 register_capability_manifest(
12233 &handler,
12234 &old_opener_ctx,
12235 &mut old_opener_rx,
12236 capability_manifest("opener", &[], &[]),
12237 2,
12238 )
12239 .await;
12240 let (mut client_rx, _) = open_route_for_capability_test(
12241 &handler,
12242 &target_ctx,
12243 &mut target_rx,
12244 712,
12245 3,
12246 "target",
12247 Some(ConsumerIdentity {
12248 module_id: "opener".to_string(),
12249 launch_nonce: "opener-nonce".to_string(),
12250 }),
12251 )
12252 .await;
12253 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12254
12255 handler
12256 .cleanup_connection(old_opener_ctx.connection_id)
12257 .expect("old opener registration cleans up");
12258 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
12259 register_capability_manifest(
12260 &handler,
12261 &new_opener_ctx,
12262 &mut new_opener_rx,
12263 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12264 4,
12265 )
12266 .await;
12267
12268 assert_capability_denied_push(
12269 client_rx
12270 .try_recv()
12271 .expect("HELLO deny addition must emit route.closed")
12272 .frame,
12273 "target",
12274 );
12275 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12276 assert!(matches!(
12277 target_rx.try_recv(),
12278 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
12279 ));
12280 }
12281
12282 #[tokio::test]
12283 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
12284 let registry = Arc::new(Registry::default());
12285 let forwarding = Arc::new(ForwardingTable::default());
12286 let supervisor = SupervisorHandle::new();
12287 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12288 let handler =
12289 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12290 .with_supervisor(supervisor);
12291 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
12292 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
12293 register_capability_manifest(
12294 &handler,
12295 &target_ctx,
12296 &mut target_rx,
12297 capability_manifest("target", &[], &[]),
12298 1,
12299 )
12300 .await;
12301 register_capability_manifest(
12302 &handler,
12303 &opener_ctx,
12304 &mut opener_rx,
12305 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12306 2,
12307 )
12308 .await;
12309 let (mut client_rx, _) = open_route_for_capability_test(
12310 &handler,
12311 &target_ctx,
12312 &mut target_rx,
12313 722,
12314 3,
12315 "target",
12316 Some(ConsumerIdentity {
12317 module_id: "opener".to_string(),
12318 launch_nonce: "opener-nonce".to_string(),
12319 }),
12320 )
12321 .await;
12322 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12323
12324 let replies = handler
12325 .handle_control_frame(
12326 &target_ctx,
12327 catalog_update_with_capabilities_frame(
12328 4,
12329 CapabilityDeclarations {
12330 provides: vec!["credentials-provider/v1".to_string()],
12331 requires: Vec::new(),
12332 must_never_reach: Vec::new(),
12333 },
12334 ),
12335 )
12336 .await
12337 .expect("claim catalog.update succeeds");
12338 assert!(matches!(
12339 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
12340 Ok(ModuleControlResponseToModule::CatalogUpdate {})
12341 ));
12342 assert_capability_denied_push(
12343 client_rx
12344 .try_recv()
12345 .expect("claim addition must emit route.closed")
12346 .frame,
12347 "target",
12348 );
12349 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12350 assert!(matches!(
12351 target_rx.try_recv(),
12352 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
12353 ));
12354 }
12355
12356 #[tokio::test]
12357 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
12358 let registry = Arc::new(Registry::default());
12359 let forwarding = Arc::new(ForwardingTable::default());
12360 let supervisor = SupervisorHandle::new();
12361 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12362 let handler =
12363 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12364 .with_supervisor(supervisor);
12365 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
12366 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
12367 register_capability_manifest(
12368 &handler,
12369 &target_ctx,
12370 &mut target_rx,
12371 capability_manifest("target", &["credentials-provider/v1"], &[]),
12372 1,
12373 )
12374 .await;
12375 register_capability_manifest(
12376 &handler,
12377 &opener_ctx,
12378 &mut opener_rx,
12379 capability_manifest("opener", &[], &[]),
12380 2,
12381 )
12382 .await;
12383 let (mut client_rx, _) = open_route_for_capability_test(
12384 &handler,
12385 &target_ctx,
12386 &mut target_rx,
12387 732,
12388 3,
12389 "target",
12390 Some(ConsumerIdentity {
12391 module_id: "opener".to_string(),
12392 launch_nonce: "opener-nonce".to_string(),
12393 }),
12394 )
12395 .await;
12396 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12397
12398 handler
12399 .handle_control_frame(
12400 &target_ctx,
12401 catalog_update_with_capabilities_frame(
12402 4,
12403 CapabilityDeclarations {
12404 provides: Vec::new(),
12405 requires: Vec::new(),
12406 must_never_reach: Vec::new(),
12407 },
12408 ),
12409 )
12410 .await
12411 .expect("claim removal catalog.update succeeds");
12412 assert_eq!(
12413 forwarding.active_binding_count().unwrap(),
12414 1,
12415 "removing an attested target claim must leave the route census unchanged"
12416 );
12417 assert!(
12418 client_rx.try_recv().is_err(),
12419 "claim removal must not emit route.closed capability_denied"
12420 );
12421 assert!(
12422 target_rx.try_recv().is_err(),
12423 "claim removal must not send the target a route GOODBYE"
12424 );
12425 }
12426
12427 #[tokio::test]
12430 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
12431 let registry = Arc::new(Registry::default());
12432 let forwarding = Arc::new(ForwardingTable::default());
12433 let supervisor = SupervisorHandle::new();
12434 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12435 let handler =
12436 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12437 .with_supervisor(supervisor);
12438 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
12439 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
12440 register_capability_manifest(
12441 &handler,
12442 &target_ctx,
12443 &mut target_rx,
12444 capability_manifest("target", &["credentials-provider/v1"], &[]),
12445 1,
12446 )
12447 .await;
12448 register_capability_manifest(
12449 &handler,
12450 &opener_ctx,
12451 &mut opener_rx,
12452 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12453 2,
12454 )
12455 .await;
12456
12457 let (_client_rx, bind) = open_route_for_capability_test(
12458 &handler,
12459 &target_ctx,
12460 &mut target_rx,
12461 742,
12462 3,
12463 "target",
12464 None,
12465 )
12466 .await;
12467 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12468 panic!("direct scope-honesty route must bind");
12469 };
12470 assert_eq!(principal, Some(Principal::Direct));
12471 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12472 }
12473
12474 #[tokio::test]
12477 async fn must_never_reach_self_route_is_capability_forbidden() {
12478 let registry = Arc::new(Registry::default());
12479 let forwarding = Arc::new(ForwardingTable::default());
12480 let supervisor = SupervisorHandle::new();
12481 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
12482 let handler =
12483 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12484 .with_supervisor(supervisor);
12485 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
12486 register_capability_manifest(
12487 &handler,
12488 &self_ctx,
12489 &mut self_rx,
12490 capability_manifest(
12491 "self-provider",
12492 &["credentials-provider/v1"],
12493 &["credentials-provider/v1"],
12494 ),
12495 1,
12496 )
12497 .await;
12498
12499 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
12500 let replies = handler
12501 .handle_control_frame(
12502 &client_ctx,
12503 route_open_frame_with_admission_facts(
12504 2,
12505 "self-provider",
12506 unique_project_root("admission-facts"),
12507 Some(ConsumerIdentity {
12508 module_id: "self-provider".to_string(),
12509 launch_nonce: "self-nonce".to_string(),
12510 }),
12511 None,
12512 ),
12513 )
12514 .await
12515 .expect("self-route refusal returns a typed frame");
12516 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
12517 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12518 assert!(
12519 self_rx.try_recv().is_err(),
12520 "self denial must not relay route.bind"
12521 );
12522 }
12523
12524 #[test]
12525 fn unsupported_channel_zero_frame_returns_error() {
12526 let handler = ControlHandler::default();
12527 let request = Frame::build(
12528 FrameType::Request,
12529 control_flags(),
12530 0,
12531 0,
12532 21,
12533 b"opaque".to_vec(),
12534 )
12535 .unwrap();
12536
12537 let response = handler
12538 .handle_control(ConnectionId::new(1), request)
12539 .unwrap();
12540
12541 assert_eq!(response[0].header.ty, FrameType::Error);
12542 assert_eq!(
12543 parse_error(&response[0])["code"],
12544 "unsupported_control_frame"
12545 );
12546 }
12547
12548 mod swap {
12553 use super::*;
12554
12555 const INCUMBENT: ConnectionId = ConnectionId::new(30);
12556 const CANDIDATE: ConnectionId = ConnectionId::new(40);
12557
12558 struct Swap {
12559 registry: Arc<Registry>,
12560 forwarding: Arc<ForwardingTable>,
12561 handler: ControlHandler,
12562 incumbent_ctx: RouteCtx,
12563 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12564 candidate_ctx: RouteCtx,
12565 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12566 }
12567
12568 async fn swap_with_incumbent() -> Swap {
12569 let registry = Arc::new(Registry::default());
12570 let forwarding = Arc::new(ForwardingTable::default());
12571 let handler =
12572 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12573 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
12574 hello_via_sink(
12575 &handler,
12576 &incumbent_ctx,
12577 &mut incumbent_rx,
12578 hello_frame("aft", PROTOCOL_VERSION, 7),
12579 )
12580 .await;
12581 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
12582 Swap {
12583 registry,
12584 forwarding,
12585 handler,
12586 incumbent_ctx,
12587 incumbent_rx,
12588 candidate_ctx,
12589 candidate_rx,
12590 }
12591 }
12592
12593 fn register_candidate(swap: &Swap, ready: Option<bool>) {
12594 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
12595 candidate_manifest.ready = ready;
12596 let registration = swap
12597 .registry
12598 .register_candidate_with_control_ops(
12599 candidate_manifest,
12600 PROTOCOL_VERSION,
12601 CANDIDATE,
12602 module_baseline_control_ops(),
12603 )
12604 .unwrap();
12605 swap.forwarding
12606 .register_candidate_module_connection(
12607 CANDIDATE,
12608 "aft".to_string(),
12609 PROTOCOL_VERSION,
12610 manifest_concurrency(®istration.manifest),
12611 swap.candidate_ctx.egress.clone(),
12612 )
12613 .unwrap();
12614 }
12615
12616 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
12617 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
12618 swap.registry.promote_candidate("aft").unwrap().unwrap();
12619 cutover.incumbent.unwrap()
12620 }
12621
12622 fn keyed_total(counters: &Value, key: &str) -> u64 {
12623 counters[key]
12624 .as_object()
12625 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
12626 .unwrap_or(0)
12627 }
12628
12629 #[tokio::test]
12630 async fn replacement_logs_old_end_and_new_admission_once() {
12631 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12632 let mut swap = swap_with_incumbent().await;
12633 swap.handler
12634 .supervisor
12635 .open_swap("aft", "candidate-nonce".to_string());
12636 hello_via_sink(
12637 &swap.handler,
12638 &swap.candidate_ctx,
12639 &mut swap.candidate_rx,
12640 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
12641 )
12642 .await;
12643 cutover(&swap);
12644 swap.handler.cleanup_connection(INCUMBENT).unwrap();
12645
12646 let captured = crate::router::test_log::captured_logs(&logs);
12647 println!("captured replacement registry logs:\n{captured}");
12648 let new_admissions: Vec<_> = captured
12649 .lines()
12650 .filter(|line| {
12651 line.contains("connection_id=40")
12652 && line.contains("swap candidate registered; not routable until cutover")
12653 })
12654 .collect();
12655 assert_eq!(
12656 new_admissions.len(),
12657 1,
12658 "unexpected admission logs: {captured}"
12659 );
12660 assert!(new_admissions[0].contains("module_id=aft"));
12661 assert!(new_admissions[0].contains("connection_id=40"));
12662 assert!(
12663 new_admissions[0].contains("swap candidate registered; not routable until cutover")
12664 );
12665 assert!(new_admissions[0].contains("ready=true"));
12666
12667 let old_admissions: Vec<_> = captured
12668 .lines()
12669 .filter(|line| {
12670 line.contains("module registered module_id=aft ")
12671 && line.contains("connection_id=30")
12672 })
12673 .collect();
12674 assert_eq!(
12675 old_admissions.len(),
12676 1,
12677 "unexpected incumbent admission: {captured}"
12678 );
12679
12680 let old_ends: Vec<_> = captured
12681 .lines()
12682 .filter(|line| {
12683 line.contains("connection_id=30") && line.contains("module registration ended")
12684 })
12685 .collect();
12686 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
12687 assert!(old_ends[0].contains("module_id=aft"));
12688 assert!(old_ends[0].contains("connection_id=30"));
12689 assert!(old_ends[0].contains("reason=replaced"));
12690 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
12691 assert_eq!(
12692 captured
12693 .lines()
12694 .filter(|line| line.contains("module registration promoted"))
12695 .count(),
12696 1,
12697 "unexpected promotion logs: {captured}"
12698 );
12699 }
12700
12701 #[tokio::test]
12707 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
12708 let mut swap = swap_with_incumbent().await;
12709 let handler = swap.handler.clone();
12710
12711 let cotenant = ConnectionId::new(31);
12713 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
12714 let (cotenant_task, cotenant_bind) = relay_route_open(
12715 &handler,
12716 cotenant,
12717 &cotenant_ctx.egress,
12718 &mut swap.incumbent_rx,
12719 100,
12720 "aft",
12721 "swap-cotenant",
12722 )
12723 .await;
12724 handler
12725 .handle_control_frame(
12726 &swap.incumbent_ctx,
12727 route_bind_ack(cotenant_bind.header.corr),
12728 )
12729 .await
12730 .unwrap();
12731 assert!(cotenant_task.await.unwrap().is_empty());
12732 let (cotenant_channel, cotenant_epoch) =
12733 published_route(&cotenant_rx.recv().await.unwrap());
12734
12735 let caller = ConnectionId::new(32);
12737 let (caller_ctx, mut caller_rx) = route_ctx(caller);
12738 let (caller_task, caller_bind) = relay_route_open(
12739 &handler,
12740 caller,
12741 &caller_ctx.egress,
12742 &mut swap.incumbent_rx,
12743 101,
12744 "aft",
12745 "swap-caller",
12746 )
12747 .await;
12748 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
12749
12750 register_candidate(&swap, None);
12751 cutover(&swap);
12752
12753 let ack = handler
12755 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
12756 .await;
12757 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
12758 if module_loop_error.is_some() {
12759 handler.cleanup_connection(INCUMBENT).unwrap();
12762 }
12763
12764 assert!(
12766 cotenant_rx.try_recv().is_err(),
12767 "the co-tenant route on the incumbent was torn down by one late ack: \
12768 {module_loop_error:?}"
12769 );
12770 assert!(matches!(
12771 swap.forwarding
12772 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
12773 .unwrap(),
12774 DataRoute::Client(DataRouteState::Bound(_))
12775 ));
12776 assert_eq!(module_loop_error, None);
12777 assert!(swap
12778 .registry
12779 .get_module_by_connection(INCUMBENT)
12780 .unwrap()
12781 .is_some());
12782
12783 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
12785 .await
12786 .expect("the incumbent is told to drop the abandoned binding")
12787 .unwrap()
12788 .frame;
12789 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12790 assert_eq!(goodbye.header.channel, abandoned_channel);
12791 assert_eq!(goodbye.header.epoch, abandoned_epoch);
12792 assert!(swap.incumbent_rx.try_recv().is_err());
12793
12794 let response = caller_task.await.unwrap();
12796 assert_eq!(response.len(), 1);
12797 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
12798 assert!(caller_rx.try_recv().is_err());
12799
12800 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
12804 let counters = handler.counters().snapshot();
12805 assert_eq!(
12806 keyed_total(&counters, "route_open_accepted_by_principal"),
12807 1
12808 );
12809 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
12810 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
12811 }
12812
12813 #[tokio::test]
12817 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
12818 let mut swap = swap_with_incumbent().await;
12819 register_candidate(&swap, None);
12820 let incumbent = cutover(&swap);
12821 swap.forwarding
12822 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
12823 .unwrap()
12824 .expect("the incumbent is still registered");
12825
12826 let client = ConnectionId::new(33);
12827 let (client_ctx, mut client_rx) = route_ctx(client);
12828 let route_handler = swap.handler.clone();
12829 let open_ctx = RouteCtx {
12830 connection_id: client,
12831 egress: client_ctx.egress.clone(),
12832 };
12833 let mut route_task = tokio::spawn(async move {
12834 route_handler
12835 .handle_control_frame(
12836 &open_ctx,
12837 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
12838 )
12839 .await
12840 .unwrap()
12841 });
12842 let bind = tokio::select! {
12843 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
12844 response = &mut route_task => {
12845 let response = response.unwrap();
12846 panic!(
12847 "post-cutover route.open was refused instead of relayed to the candidate: {}",
12848 parse_error(&response[0])["code"]
12849 );
12850 }
12851 };
12852 swap.handler
12853 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
12854 .await
12855 .unwrap();
12856 assert!(route_task.await.unwrap().is_empty());
12857 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
12858 match swap
12859 .forwarding
12860 .lookup_data_route(client, channel, epoch)
12861 .unwrap()
12862 {
12863 DataRoute::Client(DataRouteState::Bound(route)) => {
12864 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
12865 }
12866 other => panic!("expected a bound route on the candidate, got {other:?}"),
12867 }
12868 assert!(swap.incumbent_rx.try_recv().is_err());
12869 }
12870
12871 #[tokio::test]
12876 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
12877 let swap = swap_with_incumbent().await;
12878 register_candidate(&swap, Some(false));
12879 let update = Frame::build(
12880 FrameType::Request,
12881 control_flags(),
12882 0,
12883 0,
12884 55,
12885 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
12886 provides: manifest("aft", PROTOCOL_VERSION).provides,
12887 capabilities: None,
12888 ready: Some(true),
12889 })
12890 .unwrap(),
12891 )
12892 .unwrap();
12893
12894 let replies = swap
12895 .handler
12896 .handle_control_frame(&swap.candidate_ctx, update)
12897 .await
12898 .unwrap();
12899
12900 assert_eq!(replies.len(), 1);
12901 assert_eq!(
12902 replies[0].header.ty,
12903 FrameType::Response,
12904 "candidate catalog.update was refused: {:?}",
12905 serde_json::from_slice::<Value>(&replies[0].body).ok()
12906 );
12907 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
12908 assert_eq!(
12909 swap.registry
12910 .get_module("aft")
12911 .unwrap()
12912 .unwrap()
12913 .connection_id,
12914 INCUMBENT
12915 );
12916 }
12917 }
12918
12919 mod swap_admission {
12923 use super::*;
12924
12925 const INCUMBENT_NONCE: &str = "incumbent-nonce";
12926 const CANDIDATE_NONCE: &str = "candidate-nonce";
12927
12928 fn handler_with_incumbent(
12929 module_id: &str,
12930 reserved: bool,
12931 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
12932 let registry = Arc::new(Registry::default());
12933 let supervisor = SupervisorHandle::new();
12934 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
12935 if reserved {
12936 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
12937 }
12938 let handler =
12939 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
12940 let incumbent = handler
12941 .handle_control(
12942 ConnectionId::new(1),
12943 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
12944 )
12945 .unwrap();
12946 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
12947 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
12948 (registry, supervisor, handler)
12949 }
12950
12951 #[test]
12957 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
12958 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
12959
12960 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
12961 let replies = handler
12962 .handle_control(
12963 ConnectionId::new(connection),
12964 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
12965 )
12966 .unwrap();
12967 assert_eq!(replies[0].header.ty, FrameType::Error);
12968 assert_eq!(
12969 parse_error(&replies[0])["code"],
12970 "swap_token_invalid",
12971 "nonce {nonce:?}"
12972 );
12973 }
12974 assert!(registry.get_candidate("aft").unwrap().is_none());
12975 assert_eq!(
12976 registry.get_module("aft").unwrap().unwrap().connection_id,
12977 ConnectionId::new(1)
12978 );
12979
12980 let admitted = handler
12983 .handle_control(
12984 ConnectionId::new(4),
12985 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
12986 )
12987 .unwrap();
12988 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
12989 assert_eq!(
12990 registry
12991 .get_candidate("aft")
12992 .unwrap()
12993 .unwrap()
12994 .connection_id,
12995 ConnectionId::new(4)
12996 );
12997 assert_eq!(
12998 registry.get_module("aft").unwrap().unwrap().connection_id,
12999 ConnectionId::new(1),
13000 "the candidate must not take the active slot"
13001 );
13002 let replayed = handler
13003 .handle_control(
13004 ConnectionId::new(5),
13005 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
13006 )
13007 .unwrap();
13008 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
13009
13010 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
13014 let squatter = handler
13015 .handle_control(
13016 ConnectionId::new(6),
13017 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
13018 )
13019 .unwrap();
13020 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
13021 assert!(
13022 registry.get_module("aft").unwrap().is_none(),
13023 "a squatter took the active slot of an id being swapped"
13024 );
13025 }
13026
13027 #[test]
13032 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
13033 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
13034
13035 let replies = handler
13036 .handle_control(
13037 ConnectionId::new(2),
13038 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
13039 )
13040 .unwrap();
13041
13042 assert_eq!(
13043 replies[0].header.ty,
13044 FrameType::HelloAck,
13045 "reserved candidate refused: {:?}",
13046 serde_json::from_slice::<Value>(&replies[0].body).ok()
13047 );
13048 assert_eq!(
13049 registry
13050 .get_candidate("vault")
13051 .unwrap()
13052 .unwrap()
13053 .connection_id,
13054 ConnectionId::new(2)
13055 );
13056 }
13057
13058 #[test]
13061 fn without_an_open_swap_the_ordinary_gates_decide() {
13062 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
13063 supervisor.close_swap("vault");
13064
13065 let candidate = handler
13066 .handle_control(
13067 ConnectionId::new(2),
13068 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
13069 )
13070 .unwrap();
13071 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
13072 let duplicate = handler
13073 .handle_control(
13074 ConnectionId::new(3),
13075 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
13076 )
13077 .unwrap();
13078 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13079 assert!(registry.get_candidate("vault").unwrap().is_none());
13080 }
13081 }
13082
13083 mod scopes {
13087 mod delta {
13088 include!("scope_handler_delta_tests.rs");
13089 }
13090
13091 use subc_protocol::scope::{
13092 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
13093 ScopeStatus,
13094 };
13095
13096 use super::*;
13097
13098 const OWNER: &str = "prefrontal-core";
13099
13100 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
13101 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
13102 }
13103
13104 async fn call(
13105 handler: &ControlHandler,
13106 ctx: &RouteCtx,
13107 request: &ModuleControlRequestFromModule,
13108 ) -> Frame {
13109 let body = serde_json::to_vec(request).unwrap();
13110 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
13111 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
13112 assert_eq!(replies.len(), 1, "{replies:?}");
13113 replies.pop().unwrap()
13114 }
13115
13116 async fn sync(
13117 handler: &ControlHandler,
13118 ctx: &RouteCtx,
13119 generation: u64,
13120 scopes: Vec<ScopeRecord>,
13121 ) -> Result<ModuleControlResponseToModule, String> {
13122 let reply = call(
13123 handler,
13124 ctx,
13125 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
13126 )
13127 .await;
13128 match reply.header.ty {
13129 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
13130 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
13131 }
13132 }
13133
13134 async fn describe(
13135 handler: &ControlHandler,
13136 ctx: &RouteCtx,
13137 owner: &str,
13138 scope_ref: &str,
13139 ) -> ModuleControlResponseToModule {
13140 let reply = call(
13141 handler,
13142 ctx,
13143 &ModuleControlRequestFromModule::ScopeDescribe {
13144 owner: Principal::Reserved {
13145 module_id: owner.to_string(),
13146 },
13147 scope_ref: scope_ref.to_string(),
13148 },
13149 )
13150 .await;
13151 assert_eq!(
13152 reply.header.ty,
13153 FrameType::Response,
13154 "{:?}",
13155 parse_error(&reply)
13156 );
13157 serde_json::from_slice(&reply.body).unwrap()
13158 }
13159
13160 async fn module(
13162 handler: &ControlHandler,
13163 connection: u64,
13164 module_id: &str,
13165 nonce: Option<&str>,
13166 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
13167 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
13168 hello_via_sink(
13169 handler,
13170 &ctx,
13171 &mut rx,
13172 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
13173 )
13174 .await;
13175 (ctx, rx)
13176 }
13177
13178 #[tokio::test]
13181 async fn a_client_connection_cannot_sync_or_describe() {
13182 let handler = ControlHandler::new(Arc::new(Registry::default()));
13183 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
13184 for request in [
13185 ModuleControlRequestFromModule::ScopeSync {
13186 generation: 1,
13187 scopes: vec![head("s", 1)],
13188 },
13189 ModuleControlRequestFromModule::ScopeDescribe {
13190 owner: Principal::Direct,
13191 scope_ref: "s".to_string(),
13192 },
13193 ] {
13194 let reply = call(&handler, &ctx, &request).await;
13195 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
13196 }
13197 assert!(
13198 !handler
13199 .scopes
13200 .read()
13201 .unwrap()
13202 .describe(
13203 &Principal::Reserved {
13204 module_id: OWNER.to_string()
13205 },
13206 "s"
13207 )
13208 .owner_synced
13209 );
13210 }
13211
13212 #[tokio::test]
13215 async fn a_module_without_a_supervised_launch_cannot_sync() {
13216 let handler = ControlHandler::new(Arc::new(Registry::default()));
13217 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
13218 assert_eq!(
13219 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
13220 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13221 );
13222 }
13223
13224 #[tokio::test]
13225 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
13226 let supervisor = SupervisorHandle::new();
13227 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13228 let handler = ControlHandler::new(Arc::new(Registry::default()))
13229 .with_supervisor(supervisor.clone());
13230 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
13231 sync(&handler, &incumbent, 1, vec![head("s", 1)])
13232 .await
13233 .expect("the current launch syncs");
13234
13235 supervisor.open_swap(OWNER, "n2".to_string());
13238 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
13239 assert_eq!(
13240 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
13241 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13242 );
13243 sync(&handler, &incumbent, 2, vec![head("s", 1)])
13244 .await
13245 .expect("the serving owner syncs during the swap");
13246
13247 supervisor.close_swap(OWNER);
13250 assert_eq!(
13251 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
13252 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13253 );
13254 sync(&handler, &incumbent, 3, vec![head("s", 1)])
13255 .await
13256 .expect("the serving owner syncs after the rollback");
13257 handler.cleanup_connection(candidate.connection_id).unwrap();
13258
13259 supervisor.open_swap(OWNER, "n3".to_string());
13264 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
13265 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
13266 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
13267 .await
13268 .expect("the promoted launch takes authority");
13269 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
13270 panic!("unexpected reply {reply:?}");
13271 };
13272 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
13273 assert_eq!(
13274 sync(&handler, &incumbent, 4, Vec::new()).await,
13275 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13276 );
13277 }
13278
13279 #[tokio::test]
13282 async fn closing_the_authority_connection_frees_sync_authority() {
13283 let supervisor = SupervisorHandle::new();
13284 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13285 let handler = ControlHandler::new(Arc::new(Registry::default()))
13286 .with_supervisor(supervisor.clone());
13287 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
13288 sync(&handler, &first, 10, vec![head("s", 1)])
13289 .await
13290 .unwrap();
13291 handler.cleanup_connection(first.connection_id).unwrap();
13292
13293 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
13294 sync(&handler, &second, 1, vec![head("s", 1)])
13295 .await
13296 .expect("the next connection takes the released authority");
13297 }
13298
13299 #[tokio::test]
13300 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
13301 let supervisor = SupervisorHandle::new();
13302 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13303 let handler =
13304 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
13305 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
13306 sync(&handler, &first, 10, vec![head("s", 1)])
13307 .await
13308 .unwrap();
13309 handler
13310 .handle_control_frame(
13311 &first,
13312 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
13313 )
13314 .await
13315 .unwrap();
13316 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
13317 sync(&handler, &second, 1, vec![head("s", 1)])
13318 .await
13319 .expect("GOODBYE releases authority even if the old socket remains open");
13320 }
13321
13322 #[tokio::test]
13323 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
13324 let registry = Arc::new(Registry::default());
13325 let supervisor_handle = SupervisorHandle::new();
13326 let supervisor =
13327 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
13328 .with_handle(supervisor_handle.clone())
13329 .with_daemon_incarnation("incarnation-7".to_string());
13330 supervisor
13333 .supervise_configured(
13334 ModuleSpec {
13335 module_id: OWNER.to_string(),
13336 program: PathBuf::from("/nonexistent/prefrontal-core"),
13337 args: Vec::new(),
13338 env: Vec::new(),
13339 reserved: false,
13340 reserved_prefixes: Vec::new(),
13341 protocol: ModuleProtocol::Subc,
13342 overlap: Default::default(),
13343 },
13344 false,
13345 )
13346 .unwrap();
13347 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
13348 let handler =
13349 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13350 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
13351
13352 let ModuleControlResponseToModule::ScopeDescribe {
13354 status,
13355 daemon_incarnation,
13356 owner_synced,
13357 owner_configured,
13358 scope,
13359 ..
13360 } = describe(&handler, &reader, OWNER, "s").await
13361 else {
13362 panic!("not a describe reply");
13363 };
13364 assert_eq!(status, ScopeStatus::NotLive);
13365 assert_eq!(daemon_incarnation, "incarnation-7");
13366 assert!(!owner_synced);
13367 assert!(owner_configured);
13368 assert!(scope.is_none());
13369
13370 let ModuleControlResponseToModule::ScopeDescribe {
13373 status,
13374 owner_configured,
13375 ..
13376 } = describe(&handler, &reader, "ghost", "s").await
13377 else {
13378 panic!("not a describe reply");
13379 };
13380 assert_eq!(status, ScopeStatus::NotLive);
13381 assert!(!owner_configured);
13382
13383 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
13385 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
13386 let ModuleControlResponseToModule::ScopeDescribe {
13387 status,
13388 scope_epoch,
13389 owner_synced,
13390 scope,
13391 ..
13392 } = describe(&handler, &reader, OWNER, "s").await
13393 else {
13394 panic!("not a describe reply");
13395 };
13396 assert_eq!(status, ScopeStatus::Live);
13397 assert_eq!(scope_epoch, Some(4));
13398 assert!(owner_synced);
13399 let stamp = scope.expect("a live scope carries its stamp");
13400 assert!(
13401 stamp.owner_authorized,
13402 "prefrontal-core is the default authority"
13403 );
13404 assert_eq!(stamp.kind, ScopeKind::Head);
13405 }
13406
13407 #[tokio::test]
13408 async fn scope_authority_owners_decides_owner_authorized() {
13409 let supervisor = SupervisorHandle::new();
13410 supervisor.set_spawn_nonce("broca", "b1".to_string());
13411 let handler = ControlHandler::new(Arc::new(Registry::default()))
13412 .with_supervisor(supervisor)
13413 .with_scope_authority_owners(vec!["broca".to_string()]);
13414 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
13415 let mut gated = head("s", 1);
13416 gated.attributes.agent_id = Some("agent".to_string());
13417 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
13418 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
13419 describe(&handler, &broca, "broca", "s").await
13420 else {
13421 panic!("not a describe reply");
13422 };
13423 assert!(scope.unwrap().owner_authorized);
13424 }
13425
13426 #[tokio::test]
13430 async fn scope_ops_and_the_scopes_capability_are_advertised() {
13431 let handler = ControlHandler::new(Arc::new(Registry::default()));
13432 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
13433 let ack = hello_via_sink(
13434 &handler,
13435 &ctx,
13436 &mut rx,
13437 hello_frame("m", PROTOCOL_VERSION, 1),
13438 )
13439 .await;
13440 let ack = parse_ack(&ack);
13441 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
13442 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
13443 }
13444 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
13445
13446 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
13447 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13448 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13449 let reply = handler
13450 .handle_control_frame(&client, frame)
13451 .await
13452 .unwrap()
13453 .pop()
13454 .unwrap();
13455 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13456 serde_json::from_slice(&reply.body).unwrap()
13457 else {
13458 panic!("not a server.describe reply");
13459 };
13460 assert!(
13461 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
13462 "{capabilities:?}"
13463 );
13464 }
13465
13466 const PLEXUS: &str = "plexus";
13469 const OTHER: &str = "other";
13470 const AFT: &str = "aft";
13471 const BROCA: &str = "broca";
13472 const MAGIC: &str = "magic-context";
13473
13474 fn nonce(module_id: &str) -> String {
13475 format!("nonce-{module_id}")
13476 }
13477
13478 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
13479 let (tx, rx) = mpsc::channel(64);
13480 (
13481 RouteCtx {
13482 connection_id: ConnectionId::new(connection),
13483 egress: FrameSink::new(tx),
13484 },
13485 rx,
13486 )
13487 }
13488
13489 struct Rig {
13493 handler: ControlHandler,
13494 forwarding: Arc<ForwardingTable>,
13495 owner: RouteCtx,
13496 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13497 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
13498 generation: u64,
13499 next_connection: u64,
13500 _supervisor: Supervisor,
13501 }
13502
13503 async fn rig() -> Rig {
13504 rig_with_flow_support(true).await
13505 }
13506
13507 async fn rig_with_flow_support(flow_support: bool) -> Rig {
13508 rig_with_scope_capabilities(flow_support, false).await
13509 }
13510
13511 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
13512 let registry = Arc::new(Registry::default());
13513 let forwarding = Arc::new(ForwardingTable::default());
13514 let supervisor_handle = SupervisorHandle::new();
13515 let supervisor =
13516 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
13517 .with_handle(supervisor_handle.clone());
13518 supervisor
13519 .supervise_configured(
13520 ModuleSpec {
13521 module_id: OWNER.to_string(),
13522 program: PathBuf::from("/nonexistent/prefrontal-core"),
13523 args: Vec::new(),
13524 env: Vec::new(),
13525 reserved: false,
13526 reserved_prefixes: Vec::new(),
13527 protocol: ModuleProtocol::Subc,
13528 overlap: Default::default(),
13529 },
13530 false,
13531 )
13532 .unwrap();
13533 for module_id in [OWNER, AFT, BROCA, MAGIC] {
13534 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
13535 }
13536 let handler =
13537 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13538 .with_supervisor(supervisor_handle);
13539 let (owner, mut owner_rx) = wide_ctx(1);
13540 hello_via_sink(
13541 &handler,
13542 &owner,
13543 &mut owner_rx,
13544 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
13545 )
13546 .await;
13547 let mut modules = BTreeMap::new();
13548 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
13549 let (ctx, mut rx) = wide_ctx(connection);
13550 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
13551 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
13552 body["manifest"]["provenance"] =
13556 serde_json::json!({"wire_crate_version": "0.29.0"});
13557 if flow_support || run_support {
13558 let mut provides = Vec::new();
13559 if flow_support {
13560 provides.push("flow-scopes/v1");
13561 }
13562 if run_support {
13563 provides.push("agent-run-scopes/v1");
13564 }
13565 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
13566 }
13567 let hello = Frame::build(
13568 FrameType::Hello,
13569 control_flags(),
13570 0,
13571 0,
13572 connection,
13573 serde_json::to_vec(&body).unwrap(),
13574 )
13575 .unwrap();
13576 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
13577 modules.insert(module_id.to_string(), (ctx, rx));
13578 }
13579 Rig {
13580 handler,
13581 forwarding,
13582 owner,
13583 _owner_rx: owner_rx,
13584 modules,
13585 generation: 0,
13586 next_connection: 100,
13587 _supervisor: supervisor,
13588 }
13589 }
13590
13591 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
13592 ScopeCarrier::new(Principal::Reserved {
13593 module_id: module_id.to_string(),
13594 })
13595 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
13596 }
13597
13598 fn session(scope_epoch: u64) -> ScopeRecord {
13601 let mut record = head("s", scope_epoch);
13602 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
13603 record.attributes.agent_id = Some("agent-1".to_string());
13604 record.attributes.delegates = true;
13605 record
13606 }
13607
13608 impl Rig {
13609 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
13610 self.generation += 1;
13611 sync(&self.handler, &self.owner, self.generation, scopes)
13612 .await
13613 .expect("the owner's sync is accepted");
13614 }
13615
13616 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
13617 ScopeSelector {
13618 owner: Principal::Reserved {
13619 module_id: OWNER.to_string(),
13620 },
13621 scope_ref: scope_ref.to_string(),
13622 scope_epoch,
13623 }
13624 }
13625
13626 fn open_frame(
13627 &mut self,
13628 opener: Option<&str>,
13629 target: &str,
13630 scope: Option<ScopeSelector>,
13631 ) -> (
13632 RouteCtx,
13633 mpsc::Receiver<crate::router::OutboundFrame>,
13634 Frame,
13635 ) {
13636 self.next_connection += 1;
13637 let (ctx, rx) = wide_ctx(self.next_connection);
13638 let root = unique_project_root("scoped-open");
13639 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
13640 target: RouteTarget::ToolProvider {
13641 module_id: target.to_string(),
13642 },
13643 identity: BindIdentity::new(
13644 root.path().to_path_buf(),
13645 "unit".to_string(),
13646 "session".to_string(),
13647 ),
13648 consumer_identity: opener.map(|module_id| ConsumerIdentity {
13649 module_id: module_id.to_string(),
13650 launch_nonce: nonce(module_id),
13651 }),
13652 consumer_capabilities: None,
13653 role_versions: None,
13654 admission_facts: None,
13655 scope,
13656 })
13657 .unwrap();
13658 let frame = Frame::build(
13659 FrameType::Request,
13660 control_flags(),
13661 0,
13662 0,
13663 self.next_connection,
13664 body,
13665 )
13666 .unwrap();
13667 (ctx, rx, frame)
13668 }
13669
13670 async fn refused(
13672 &mut self,
13673 opener: Option<&str>,
13674 target: &str,
13675 scope: Option<ScopeSelector>,
13676 ) -> String {
13677 self.refusal_body(opener, target, scope).await["code"]
13678 .as_str()
13679 .unwrap()
13680 .to_string()
13681 }
13682
13683 async fn refusal_body(
13684 &mut self,
13685 opener: Option<&str>,
13686 target: &str,
13687 scope: Option<ScopeSelector>,
13688 ) -> Value {
13689 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
13690 let replies = tokio::time::timeout(
13691 Duration::from_secs(2),
13692 self.handler.handle_control_frame(&ctx, frame),
13693 )
13694 .await
13695 .expect("the open must be refused before waiting for a bind ack")
13696 .unwrap();
13697 assert_eq!(replies.len(), 1, "{replies:?}");
13698 assert_eq!(replies[0].header.ty, FrameType::Error);
13699 let (_, module_rx) = self.modules.get_mut(target).unwrap();
13700 assert!(
13701 module_rx.try_recv().is_err(),
13702 "a refused open relays nothing"
13703 );
13704 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
13705 parse_error(&replies[0])
13706 }
13707
13708 async fn relayed(
13710 &mut self,
13711 opener: Option<&str>,
13712 target: &str,
13713 scope: Option<ScopeSelector>,
13714 ) -> Relayed {
13715 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
13716 let handler = self.handler.clone();
13717 let task_ctx = ctx.clone();
13718 let task = tokio::spawn(async move {
13719 handler
13720 .handle_control_frame(&task_ctx, frame)
13721 .await
13722 .unwrap()
13723 });
13724 let (_, module_rx) = self.modules.get_mut(target).unwrap();
13725 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
13726 .await
13727 .expect("the target receives the relayed route.bind")
13728 .unwrap()
13729 .frame;
13730 Relayed {
13731 target: target.to_string(),
13732 client: ctx,
13733 client_rx: rx,
13734 task,
13735 bind,
13736 }
13737 }
13738
13739 async fn ack(&self, relayed: &Relayed) {
13740 let (module, _) = &self.modules[&relayed.target];
13741 self.handler
13742 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
13743 .await
13744 .unwrap();
13745 }
13746
13747 async fn bound(
13749 &mut self,
13750 opener: Option<&str>,
13751 target: &str,
13752 scope: Option<ScopeSelector>,
13753 ) -> Bound {
13754 let relayed = self.relayed(opener, target, scope).await;
13755 self.ack(&relayed).await;
13756 let Relayed {
13757 target,
13758 client,
13759 mut client_rx,
13760 task,
13761 bind,
13762 } = relayed;
13763 assert!(
13764 task.await.unwrap().is_empty(),
13765 "the open is answered by commit"
13766 );
13767 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
13768 Bound {
13769 target,
13770 client,
13771 client_rx,
13772 channel,
13773 epoch,
13774 bind,
13775 }
13776 }
13777
13778 fn live(&self, route: &Bound) -> bool {
13779 matches!(
13780 self.forwarding
13781 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
13782 .unwrap(),
13783 DataRoute::Client(DataRouteState::Bound(_))
13784 )
13785 }
13786 }
13787
13788 struct Relayed {
13789 target: String,
13790 client: RouteCtx,
13791 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13792 task: tokio::task::JoinHandle<Vec<Frame>>,
13793 bind: Frame,
13794 }
13795
13796 struct Bound {
13797 target: String,
13798 client: RouteCtx,
13799 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13800 channel: u16,
13801 epoch: u32,
13802 bind: Frame,
13803 }
13804
13805 impl Bound {
13806 fn closed_reason(&mut self) -> RouteCloseReason {
13809 let mut reason = None;
13810 let mut goodbye = false;
13811 while let Ok(outbound) = self.client_rx.try_recv() {
13812 let frame = outbound.frame;
13813 match frame.header.ty {
13814 FrameType::Goodbye => {
13815 assert_eq!(
13816 (frame.header.channel, frame.header.epoch),
13817 (self.channel, self.epoch)
13818 );
13819 goodbye = true;
13820 }
13821 FrameType::Push => {
13822 let ClientControlPush::RouteClosed {
13823 reason: r,
13824 module_id,
13825 ..
13826 } = serde_json::from_slice(&frame.body).unwrap()
13827 else {
13828 panic!("unexpected push");
13829 };
13830 assert_eq!(module_id, self.target);
13831 reason = Some(r);
13832 }
13833 other => panic!("unexpected frame {other:?}"),
13834 }
13835 }
13836 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
13837 reason.expect("the client is told why the route closed")
13838 }
13839
13840 fn untouched(&mut self) -> bool {
13841 self.client_rx.try_recv().is_err()
13842 }
13843
13844 fn stamp(&self) -> Option<ScopeStamp> {
13845 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
13846 ModuleControlRequest::RouteBind { scope, .. } => scope,
13847 other => panic!("expected a route.bind, got {other:?}"),
13848 }
13849 }
13850 }
13851
13852 #[tokio::test]
13853 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
13854 ) {
13855 let mut rig = rig().await;
13856 let mut record = session(1);
13857 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
13858 record.child_owners = vec![Principal::Reserved {
13859 module_id: MAGIC.to_string(),
13860 }];
13861 rig.sync(vec![record]).await;
13862 let scope = || Some(rig_selector("s", Some(1)));
13863
13864 rig.bound(Some(OWNER), PLEXUS, scope()).await;
13867 rig.bound(Some(AFT), OTHER, scope()).await;
13868 rig.bound(Some(BROCA), PLEXUS, scope()).await;
13869
13870 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
13874 assert_eq!(
13875 rig.refused(opener, target, scope()).await,
13876 error_codes::SCOPE_NOT_CARRIER,
13877 "{opener:?} -> {target}"
13878 );
13879 }
13880 }
13881
13882 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
13883 ScopeSelector {
13884 owner: Principal::Reserved {
13885 module_id: OWNER.to_string(),
13886 },
13887 scope_ref: scope_ref.to_string(),
13888 scope_epoch,
13889 }
13890 }
13891
13892 #[tokio::test]
13893 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
13894 let mut rig = rig().await;
13895 rig.sync(vec![session(1)]).await;
13896 for opener in [OWNER, AFT] {
13897 assert_eq!(
13898 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
13899 .await,
13900 error_codes::SCOPE_EPOCH_REQUIRED,
13901 "{opener}"
13902 );
13903 }
13904 }
13905
13906 #[tokio::test]
13907 async fn admission_separates_not_synced_not_live_and_ended() {
13908 let mut rig = rig().await;
13909 let code = rig
13911 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13912 .await;
13913 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
13914 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
13915
13916 let ghost = ScopeSelector {
13918 owner: Principal::Reserved {
13919 module_id: "ghost".to_string(),
13920 },
13921 scope_ref: "s".to_string(),
13922 scope_epoch: Some(1),
13923 };
13924 assert_eq!(
13925 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
13926 error_codes::SCOPE_NOT_LIVE
13927 );
13928
13929 rig.sync(vec![session(2)]).await;
13930 assert_eq!(
13931 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
13932 .await,
13933 error_codes::SCOPE_NOT_LIVE
13934 );
13935 for epoch in [1, 3] {
13936 assert_eq!(
13937 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
13938 .await,
13939 error_codes::SCOPE_ENDED,
13940 "epoch {epoch}"
13941 );
13942 }
13943 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
13945 .await;
13946 }
13947
13948 #[tokio::test]
13949 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
13950 let mut rig = rig().await;
13951 rig.sync(vec![session(1)]).await;
13952 let route = rig
13953 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
13954 .await;
13955 let stamp = route.stamp().expect("a scoped bind carries the stamp");
13956 assert_eq!(stamp.scope_ref, "s");
13957 assert_eq!(stamp.scope_epoch, 1);
13958 assert_eq!(stamp.kind, ScopeKind::Head);
13959 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
13960 assert!(stamp.attributes.delegates);
13961 assert!(stamp.owner_authorized);
13962 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
13963 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
13964
13965 let (broca, mut broca_rx) = wide_ctx(50);
13968 hello_via_sink(
13969 &rig.handler,
13970 &broca,
13971 &mut broca_rx,
13972 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
13973 )
13974 .await;
13975 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
13976 .await
13977 .unwrap();
13978 let own = ScopeSelector {
13979 owner: Principal::Reserved {
13980 module_id: BROCA.to_string(),
13981 },
13982 scope_ref: "b".to_string(),
13983 scope_epoch: Some(1),
13984 };
13985 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
13986 assert!(!route.stamp().unwrap().owner_authorized);
13987 }
13988
13989 #[tokio::test]
13990 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
13991 let mut rig = rig().await;
13992 let mut record = head("s", 1);
13993 record.carriers = vec![carrier(AFT, None)];
13994 let flow_id = "Flow:run-7/step_2!~";
13995 record.attributes.flow_id = Some(flow_id.to_string());
13996 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
13997 .await
13998 .unwrap();
13999 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
14000 panic!("not a sync reply");
14001 };
14002 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
14003 let route = rig
14004 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14005 .await;
14006 assert!(rig.live(&route), "the stamped bind committed");
14007 let stamp = route.stamp().expect("a flow scope carries a stamp");
14008 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
14009 assert_eq!(stamp.attributes.agent_id, None);
14010 assert!(!stamp.attributes.delegates);
14011 assert!(stamp.owner_authorized);
14012 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14013 assert_eq!(unscoped.stamp(), None);
14014 }
14015
14016 #[tokio::test]
14017 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
14018 let mut rig = rig_with_flow_support(false).await;
14019 let mut record = session(1);
14020 record.attributes.flow_id = Some("flow:7".to_string());
14021 rig.sync(vec![record]).await;
14022 for opener in [OWNER, AFT] {
14023 let body = rig
14024 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
14025 .await;
14026 assert_eq!(body["code"], "target_flow_unsupported");
14027 let message = body["message"].as_str().unwrap();
14028 for required in [PLEXUS, "flow-scopes/v1"] {
14029 assert!(message.contains(required), "{message}");
14030 }
14031 }
14032 }
14033
14034 #[tokio::test]
14035 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
14036 let mut rig = rig_with_flow_support(true).await;
14037 let mut record = session(1);
14038 record.attributes.flow_id = Some("flow:7".to_string());
14039 rig.sync(vec![record]).await;
14040 for opener in [OWNER, AFT] {
14041 let route = rig
14042 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
14043 .await;
14044 assert!(rig.live(&route));
14045 assert_eq!(
14046 route.stamp().unwrap().attributes.flow_id.as_deref(),
14047 Some("flow:7")
14048 );
14049 }
14050 }
14051
14052 #[tokio::test]
14053 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
14054 use std::future::Future;
14055
14056 let mut rig = rig_with_flow_support(true).await;
14057 let mut record = session(1);
14058 record.attributes.flow_id = Some("flow:7".to_string());
14059 rig.sync(vec![record]).await;
14060 let (client, mut client_rx, frame) =
14061 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
14062 for _ in 0..64 {
14065 client.egress.try_send(route_bind_ack(1)).unwrap();
14066 }
14067 let handler = rig.handler.clone();
14068 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
14069 std::future::poll_fn(|cx| {
14070 assert!(open.as_mut().poll(cx).is_pending());
14071 std::task::Poll::Ready(())
14072 })
14073 .await;
14074 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14075
14076 let old_connection = rig.modules[PLEXUS].0.connection_id;
14077 rig.handler.cleanup_connection(old_connection).unwrap();
14078 let (replacement, mut replacement_rx) = wide_ctx(200);
14079 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
14080 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
14081 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
14082 let hello = Frame::build(
14083 FrameType::Hello,
14084 control_flags(),
14085 0,
14086 0,
14087 200,
14088 serde_json::to_vec(&body).unwrap(),
14089 )
14090 .unwrap();
14091 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
14092
14093 client_rx.try_recv().unwrap();
14094 let replies = tokio::time::timeout(Duration::from_secs(2), open)
14095 .await
14096 .expect("the replacement is refused without waiting for a bind ack")
14097 .unwrap();
14098 assert_eq!(replies.len(), 1);
14099 let body = parse_error(&replies[0]);
14100 assert_eq!(body["code"], "target_flow_unsupported");
14101 for required in [PLEXUS, "flow-scopes/v1"] {
14102 assert!(body["message"].as_str().unwrap().contains(required));
14103 }
14104 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
14105 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
14106 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14107 assert!(rig
14108 .handler
14109 .registry
14110 .get_module_by_connection(replacement.connection_id)
14111 .unwrap()
14112 .is_some());
14113 }
14114
14115 #[tokio::test]
14116 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
14117 {
14118 let mut rig = rig_with_flow_support(false).await;
14119 rig.sync(vec![session(1)]).await;
14120 let route = rig
14121 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14122 .await;
14123 assert!(rig.live(&route));
14124 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
14125 let mut record = session(1);
14126 record.attributes.flow_id = Some("flow:7".to_string());
14127 rig.sync(vec![record]).await;
14128 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
14129 assert!(rig.live(&unscoped));
14130 assert_eq!(unscoped.stamp(), None);
14131 }
14132
14133 #[tokio::test]
14134 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
14135 let mut rig = rig().await;
14136 let mut record = session(1);
14137 record.attributes.flow_id = Some("flow:7".to_string());
14138 rig.sync(vec![record.clone()]).await;
14139 let before = rig.forwarding.published_scope_tag(OWNER, "s");
14140 let mut owner_route = rig
14141 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
14142 .await;
14143 let mut carrier_route = rig
14144 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
14145 .await;
14146 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14147 rig.sync(vec![record.clone()]).await;
14148 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
14149 assert!(rig.live(&owner_route) && owner_route.untouched());
14150 assert!(rig.live(&carrier_route) && carrier_route.untouched());
14151
14152 record.attributes.flow_id = Some("flow:8".to_string());
14153 rig.sync(vec![record]).await;
14154 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
14155 let before = before.unwrap();
14156 assert_eq!(after.scope_epoch, before.scope_epoch);
14157 assert!(after.version > before.version);
14158 for route in [&mut owner_route, &mut carrier_route] {
14159 assert!(!rig.live(route));
14160 assert_eq!(
14161 route.closed_reason(),
14162 RouteCloseReason::ScopeDelegationChanged
14163 );
14164 }
14165 assert!(rig.live(&unscoped) && unscoped.untouched());
14166 for target in [PLEXUS, OTHER] {
14169 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
14170 let goodbye = module_rx
14171 .try_recv()
14172 .expect("the provider sees the drain")
14173 .frame;
14174 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14175 assert!(module_rx.try_recv().is_err());
14176 }
14177 let rebound = rig
14178 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14179 .await;
14180 assert_eq!(
14181 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
14182 Some("flow:8")
14183 );
14184 }
14185
14186 #[tokio::test]
14191 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
14192 let mut rig = rig().await;
14193 rig.sync(vec![session(1)]).await;
14194 let mut cotenant = rig
14195 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
14196 .await;
14197
14198 let mut changed = session(1);
14199 changed.child_owners.push(Principal::Reserved {
14200 module_id: MAGIC.to_string(),
14201 });
14202 let mut ended = None;
14203 for (code, next) in [
14204 (error_codes::SCOPE_CHANGED, vec![changed]),
14205 (error_codes::SCOPE_ENDED, Vec::new()),
14206 ] {
14207 let relayed = rig
14208 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14209 .await;
14210 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
14211 ended = Some(next.is_empty());
14212 rig.sync(next).await;
14213 rig.ack(&relayed).await;
14214 let replies = relayed.task.await.unwrap();
14215 assert_eq!(replies.len(), 1, "{replies:?}");
14216 assert_eq!(parse_error(&replies[0])["code"], code);
14217 assert_eq!(
14218 subc_protocol::error_codes::is_retryable_route_open(code),
14219 code == error_codes::SCOPE_CHANGED
14220 );
14221 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14222 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14224 let mut goodbyes = Vec::new();
14227 while let Ok(outbound) = plexus_rx.try_recv() {
14228 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
14229 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
14230 }
14231 assert!(
14232 goodbyes.contains(&(bind_channel, bind_epoch)),
14233 "{goodbyes:?}"
14234 );
14235 assert!(rig
14236 .handler
14237 .registry
14238 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
14239 .unwrap()
14240 .is_some());
14241 }
14242 assert_eq!(ended, Some(true));
14243 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
14246 }
14247
14248 #[tokio::test]
14251 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
14252 struct Case {
14253 name: &'static str,
14254 change: fn(&mut ScopeRecord),
14255 closed: [Option<RouteCloseReason>; 4],
14258 }
14259 use RouteCloseReason::*;
14260 let cases = [
14261 Case {
14262 name: "a carrier entry removed",
14263 change: |r| {
14264 r.carriers.retain(|c| {
14265 c.principal
14266 != Principal::Reserved {
14267 module_id: AFT.to_string(),
14268 }
14269 })
14270 },
14271 closed: [None, Some(ScopeCarrierRemoved), None, None],
14272 },
14273 Case {
14274 name: "a target removed from a carrier",
14275 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
14276 closed: [None, None, None, Some(ScopeCarrierRemoved)],
14277 },
14278 Case {
14279 name: "a bare carrier narrowed to targets",
14280 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
14281 closed: [None, Some(ScopeCarrierRemoved), None, None],
14282 },
14283 Case {
14284 name: "delegates turned off",
14285 change: |r| r.attributes.delegates = false,
14286 closed: [Some(ScopeDelegationChanged); 4],
14287 },
14288 Case {
14289 name: "agent_id changed",
14290 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
14291 closed: [Some(ScopeDelegationChanged); 4],
14292 },
14293 Case {
14294 name: "a carrier added, child owners changed, the record re-sent",
14295 change: |r| {
14296 r.carriers.push(carrier(MAGIC, None));
14297 r.child_owners.push(Principal::Reserved {
14298 module_id: MAGIC.to_string(),
14299 });
14300 },
14301 closed: [None; 4],
14302 },
14303 Case {
14304 name: "a target added",
14305 change: |r| {
14306 r.carriers[1]
14307 .targets
14308 .as_mut()
14309 .unwrap()
14310 .push("third".to_string())
14311 },
14312 closed: [None; 4],
14313 },
14314 Case {
14315 name: "delegates turned on",
14316 change: |r| r.attributes.delegates = true,
14317 closed: [None; 4],
14318 },
14319 ];
14320 for case in cases {
14321 let mut rig = rig().await;
14322 rig.sync(vec![session(1)]).await;
14323 let scope = || Some(rig_selector("s", Some(1)));
14324 let mut routes = [
14325 rig.bound(Some(OWNER), PLEXUS, scope()).await,
14326 rig.bound(Some(AFT), PLEXUS, scope()).await,
14327 rig.bound(Some(BROCA), PLEXUS, scope()).await,
14328 rig.bound(Some(BROCA), OTHER, scope()).await,
14329 ];
14330 let mut record = session(1);
14331 (case.change)(&mut record);
14332 rig.sync(vec![record]).await;
14333 for (index, expected) in case.closed.iter().enumerate() {
14334 let route = &mut routes[index];
14335 match expected {
14336 Some(reason) => {
14337 assert!(!rig.live(route), "{}: route {index} still live", case.name);
14338 assert_eq!(
14339 route.closed_reason(),
14340 *reason,
14341 "{}: route {index}",
14342 case.name
14343 );
14344 }
14345 None => {
14346 assert!(rig.live(route), "{}: route {index} closed", case.name);
14347 assert!(
14348 route.untouched(),
14349 "{}: route {index} was told something",
14350 case.name
14351 );
14352 }
14353 }
14354 }
14355 }
14356 }
14357
14358 #[tokio::test]
14359 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
14360 for next in [Vec::new(), vec![session(2)]] {
14362 let mut rig = rig().await;
14363 rig.sync(vec![session(1)]).await;
14364 let mut route = rig
14365 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14366 .await;
14367 rig.sync(next).await;
14368 assert!(!rig.live(&route));
14369 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
14370 }
14371
14372 let mut rig = rig().await;
14375 let mut child = session(1);
14376 child.scope_ref = "child".to_string();
14377 child.kind = ScopeKind::Worker;
14378 child.parent = Some(ScopeParent::new(
14379 Principal::Reserved {
14380 module_id: OWNER.to_string(),
14381 },
14382 "s".to_string(),
14383 1,
14384 ));
14385 rig.sync(vec![session(1), child.clone()]).await;
14386 let mut child_route = rig
14387 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
14388 .await;
14389 assert_eq!(
14390 child_route.stamp().unwrap().parent_state,
14391 Some(ParentState::Linked)
14392 );
14393 rig.sync(vec![child]).await;
14394 assert!(!rig.live(&child_route));
14395 assert_eq!(
14396 child_route.closed_reason(),
14397 RouteCloseReason::ScopeParentEnded
14398 );
14399 }
14400
14401 #[tokio::test]
14402 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
14403 ) {
14404 let mut rig = rig().await;
14405 rig.sync(vec![session(1)]).await;
14406 let mut route = rig
14407 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14408 .await;
14409 let before = rig.forwarding.published_scope_tag(OWNER, "s");
14410 rig.sync(vec![session(1)]).await;
14411 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
14412 assert!(rig.live(&route) && route.untouched());
14413
14414 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
14418 .forwarding
14419 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
14420 .unwrap()
14421 else {
14422 panic!("the route is bound");
14423 };
14424 binding.flow.acquire_tagged(9, false).await.unwrap();
14425 let mut widened = session(1);
14426 widened.carriers.push(carrier(MAGIC, None));
14427 rig.sync(vec![widened]).await;
14428 assert!(rig.live(&route) && route.untouched());
14429 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14430 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
14431 assert_eq!(binding.flow.in_flight(), 1);
14434 binding
14435 .flow
14436 .acquire_tagged(10, false)
14437 .await
14438 .expect("the flow is still open");
14439 }
14440
14441 #[tokio::test]
14444 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
14445 let mut rig = rig().await;
14446 rig.sync(vec![session(1)]).await;
14447 let mut on_incumbent = rig
14448 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14449 .await;
14450
14451 let (candidate, _candidate_rx) = wide_ctx(9);
14454 let registration = rig
14455 .handler
14456 .registry
14457 .register_candidate_with_control_ops(
14458 manifest(PLEXUS, PROTOCOL_VERSION),
14459 PROTOCOL_VERSION,
14460 candidate.connection_id,
14461 module_baseline_control_ops(),
14462 )
14463 .unwrap();
14464 rig.forwarding
14465 .register_candidate_module_connection(
14466 candidate.connection_id,
14467 PLEXUS.to_string(),
14468 PROTOCOL_VERSION,
14469 manifest_concurrency(®istration.manifest),
14470 candidate.egress.clone(),
14471 )
14472 .unwrap();
14473 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
14474 rig.handler
14475 .registry
14476 .promote_candidate(PLEXUS)
14477 .unwrap()
14478 .unwrap();
14479 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
14480
14481 rig.sync(Vec::new()).await;
14482 assert!(!rig.live(&on_incumbent));
14483 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
14484 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14485 let goodbye = incumbent_rx
14486 .try_recv()
14487 .expect("the superseded endpoint is told")
14488 .frame;
14489 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14490 }
14491 }
14492}
14493
14494#[cfg(test)]
14495mod concurrency_default_exposure_tests {
14496 use super::*;
14497
14498 fn hello_body(role_json: &str) -> Vec<u8> {
14499 format!(
14500 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
14501 )
14502 .into_bytes()
14503 }
14504
14505 fn manifest_from(body: &[u8]) -> ModuleManifest {
14506 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
14507 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
14508 .expect("manifest parses")
14509 }
14510
14511 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
14512
14513 #[test]
14514 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
14515 let body = hello_body(&format!(
14516 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
14517 ));
14518 let manifest = manifest_from(&body);
14519 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
14522 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
14523 }
14524
14525 #[test]
14526 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
14527 let body = hello_body(&format!(
14528 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
14529 ));
14530 let manifest = manifest_from(&body);
14531 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
14532 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
14533 }
14534
14535 #[test]
14536 fn non_management_roles_are_never_reported() {
14537 let body = hello_body(
14538 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
14539 );
14540 let manifest = manifest_from(&body);
14541 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
14542 }
14543}