1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
26 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
27 },
28 scope::{
29 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
30 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
31 SCOPE_SYNC_OP,
32 },
33 session::{
34 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
35 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
36 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
37 ROLE_VERSIONS_FIELD,
38 },
39 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
40 Priority, RouteTarget, PROTOCOL_VERSION,
41};
42use tokio::time::{timeout_at, Instant};
43use tracing::{debug, info, warn};
44
45use crate::{
46 caller_identity::CallerIdentity,
47 capability_requirements::{
48 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
49 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
50 RuntimeModule,
51 },
52 child_roster::DaemonShutdownFlag,
53 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
54 forwarding::{
55 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
56 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
57 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
58 },
59 observability::{
60 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
61 },
62 privileged_prompt::{prompt_text, PrivilegedOperation},
63 provenance::{
64 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
65 },
66 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
67 router::{RouteCtx, RouterError},
68 scopes::{HelloLaunchNonces, ScopeTable},
69 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
70 stderr_tail::{CaptureState, TailEntry},
71 supervise::{
72 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
73 SupervisorHandle, SwapHelloAdmission,
74 },
75 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
76};
77
78pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
84
85const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
86const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
87const CAP_PING_PONG: &str = "ping_pong_v1";
88const CAP_SESSION_ATTACH: &str = "session_attach_v1";
89const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
90
91const SUBC_CONTROL_OPS: &[&str] = &[
92 ops::ACTIVITY_RECORD,
93 ops::ACTIVITY_QUERY,
94 ops::ACTIVITY_SUBSCRIBE,
95 ops::SERVER_DESCRIBE,
96 ops::CATALOG_LIST,
97 ops::ROUTE_OPEN,
98 ops::ROUTE_POLL,
99 ops::ROUTE_CLOSING,
100 ops::ROUTE_CLOSED,
101 ops::SUPERVISOR_LIST,
102 ops::SUPERVISOR_RESTART,
103 ops::SUPERVISOR_SWAP,
104 ops::SUPERVISOR_RELOAD,
105 ops::SUPERVISOR_RESCAN,
106 ops::SUPERVISOR_RELEASE_RESERVED,
107 ops::SUPERVISOR_SET_ENABLED,
108 ops::SUPERVISOR_HEALTH_PROBE,
109 ops::SUPERVISOR_HEALTH,
110 ops::SUPERVISOR_STDERR_TAIL,
111 ops::SUPERVISOR_TERMINALS,
112 ops::SUPERVISOR_ROUTES,
113 ops::SUPERVISOR_PROVENANCE,
114 ops::SUPERVISOR_SPAWN_SNAPSHOT,
115 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
116];
117
118const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
125
126mod rescan_checked;
127use rescan_checked::{RescanMode, RescanPlan};
128
129const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
130 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
131 "supervisor.live_roots",
132 SCOPE_SYNC_OP,
133 SCOPE_APPLY_OP,
134 SCOPE_DESCRIBE_OP,
135 "operator.confirm",
136];
137
138const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
142
143const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
144
145pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
153
154pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
169
170pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
188
189const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
190const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
191
192fn reload_verdict(
193 configured: &Path,
194 spawned_from: Option<&Path>,
195 image: subc_control::RunningImageAgreement,
196) -> PendingReloadVerdict {
197 let path = match spawned_from {
198 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
199 ReloadPathAgreement::Match
200 }
201 Some(spawned_from) => ReloadPathAgreement::Mismatch {
202 configured: configured.to_path_buf(),
203 spawned_from: spawned_from.to_path_buf(),
204 },
205 None => ReloadPathAgreement::Unavailable {
206 reason: if matches!(
207 image,
208 subc_control::RunningImageAgreement::Unavailable {
209 reason: subc_control::RunningImageUnavailableReason::NotRunning
210 }
211 ) {
212 ReloadPathUnavailableReason::NotRunning
213 } else {
214 ReloadPathUnavailableReason::SpawnedPathUnavailable
215 },
216 },
217 };
218 PendingReloadVerdict {
219 path,
220 image,
221 spec: None,
222 }
223}
224
225fn launch_spec_agreement(
226 configured: &crate::supervise::ModuleSpec,
227 running: bool,
228 spawned: Option<&crate::supervise::LaunchSpecDigest>,
229) -> subc_control::LaunchSpecAgreement {
230 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
231 if !running {
232 return Agreement::Unavailable {
233 reason: Reason::NotRunning,
234 };
235 }
236 match spawned {
237 None => Agreement::Unavailable {
238 reason: Reason::SpawnedSpecUnavailable,
239 },
240 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
241 Agreement::Match
242 }
243 Some(_) => Agreement::Mismatch,
244 }
245}
246
247fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
248 if configured == spawned_from {
249 return true;
250 }
251 #[cfg(windows)]
252 {
253 match (
259 std::fs::canonicalize(configured),
260 std::fs::canonicalize(spawned_from),
261 ) {
262 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
263 _ => false,
264 }
265 }
266 #[cfg(not(windows))]
267 {
268 false
269 }
270}
271
272#[derive(Clone)]
273struct DaemonProvenanceFacts {
274 build: DaemonBuildProvenance,
275 pid: Option<u32>,
276 started_at_ms: Option<u64>,
277 start_clock: Option<crate::clock::StartClock>,
278 executable_path: Option<PathBuf>,
279 executable_identity: Option<SpawnedFileIdentity>,
280 process_start_time: Option<u64>,
281 probe: ExecutableIdentityProbe,
282}
283
284impl Default for DaemonProvenanceFacts {
285 fn default() -> Self {
286 Self {
287 build: DaemonBuildProvenance {
288 build_git_sha: None,
289 build_lock_digest: None,
290 },
291 pid: None,
292 started_at_ms: None,
293 start_clock: None,
294 executable_path: None,
295 executable_identity: None,
296 process_start_time: None,
297 probe: ExecutableIdentityProbe::default(),
298 }
299 }
300}
301
302#[derive(Debug, Clone)]
303struct SupervisorRescanContext {
304 supervisor: Supervisor,
305 config_path: PathBuf,
306 configured_port: Option<u16>,
307 storage_config: Option<crate::daemon_config::StorageConfig>,
308 admission_facts_carrier_module_id: Option<String>,
309 admission_facts_targets: Option<Vec<String>>,
310 scope_authority_owners: Vec<String>,
311 privileged_operator_prompt: PrivilegedOperatorPrompt,
312 generation: Arc<std::sync::atomic::AtomicU64>,
318}
319
320const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
341 "reloading",
342 "supervisor_not_live",
343 "registration_not_active",
344 "no_forwarding_connection",
345 "relay_send_failed",
346];
347
348#[derive(Clone)]
350pub struct ControlHandler {
351 registry: Arc<Registry>,
352 activity: Option<crate::activity::ActivityLedger>,
353 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
354 forwarding: Arc<ForwardingTable>,
355 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
356 supervisor: SupervisorHandle,
357 subc_capabilities: Arc<[String]>,
358 route_bind_relay_timeout: Duration,
362 route_bind_relay_timeouts: BTreeMap<String, Duration>,
366 route_bind_breakers: RouteBindBreakers,
369 route_bind_concurrency: RouteBindConcurrency,
372 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
377 route_bind_breaker_threshold: u32,
379 route_bind_breaker_cooldown: Duration,
381 health_probe_timeout: Duration,
382 storage_config: Option<crate::daemon_config::StorageConfig>,
385 machine_id: Option<crate::machine_id::MachineId>,
389 admission_facts_carrier_module_id: Option<String>,
390 admission_facts_targets: Option<Vec<String>>,
391 scopes: Arc<RwLock<ScopeTable>>,
395 scope_deadline_added: Arc<tokio::sync::Notify>,
399 scope_authority_owners: Vec<String>,
402 privileged_operator_prompt: PrivilegedOperatorPrompt,
404 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
407 #[cfg(test)]
408 caller_lookup: Option<TestCallerLookup>,
409 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
412 rescan: Option<SupervisorRescanContext>,
413 connected_clients: ConnectedClients,
414 counters: DaemonCounters,
415 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
416 daemon_shutdown: DaemonShutdownFlag,
421 daemon_provenance: DaemonProvenanceFacts,
422 #[cfg(test)]
423 control_dispatch_delay: Option<Duration>,
424 #[cfg(test)]
425 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
426 #[cfg(test)]
430 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
431 #[cfg(test)]
437 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
438}
439
440impl fmt::Debug for ControlHandler {
441 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
442 f.debug_struct("ControlHandler")
443 .field("registry", &self.registry)
444 .field("forwarding", &self.forwarding)
445 .field("process_liveness", &self.process_liveness.is_some())
446 .field("supervisor", &self.supervisor)
447 .field("subc_capabilities", &self.subc_capabilities)
448 .finish()
449 }
450}
451
452struct RouteOpenRequest {
453 target: RouteTarget,
454 identity: BindIdentity,
455 consumer_identity: Option<ConsumerIdentity>,
456 consumer_capabilities: Option<Vec<String>>,
457 role_versions: Option<BTreeMap<String, String>>,
458 admission_facts: Option<serde_json::Value>,
459 scope: Option<ScopeSelector>,
460}
461
462#[derive(Clone, Copy)]
463enum RouteOpenPhase {
464 Admission,
465 ProjectRoot,
466 EgressReservation,
467 BindSend,
468 BindAckWait,
469}
470
471struct RouteOpenTimings {
475 module_id: String,
476 principal: String,
477 admission: Duration,
478 project_root: Duration,
479 egress_reservation: Duration,
480 bind_send: Duration,
481 bind_ack_wait: Duration,
482 phase: Option<RouteOpenPhase>,
483 phase_started_at: Instant,
484}
485
486impl RouteOpenTimings {
487 fn new() -> Self {
488 Self {
489 module_id: String::new(),
490 principal: "unresolved".to_string(),
492 admission: Duration::ZERO,
493 project_root: Duration::ZERO,
494 egress_reservation: Duration::ZERO,
495 bind_send: Duration::ZERO,
496 bind_ack_wait: Duration::ZERO,
497 phase: Some(RouteOpenPhase::Admission),
498 phase_started_at: Instant::now(),
499 }
500 }
501
502 fn finish(&mut self) {
503 let elapsed = self.phase_started_at.elapsed();
504 match self.phase.take() {
505 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
506 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
507 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
508 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
509 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
510 None => {}
511 }
512 }
513
514 fn enter(&mut self, phase: RouteOpenPhase) {
515 self.finish();
516 self.phase = Some(phase);
517 self.phase_started_at = Instant::now();
518 }
519}
520
521struct RouteBindReservationGuard {
522 forwarding: Arc<ForwardingTable>,
523 endpoint: ModuleEndpointId,
524 relay_corr: u64,
525 armed: bool,
526}
527
528struct ModuleControlRpcGuard {
529 forwarding: Arc<ForwardingTable>,
530 endpoint: ModuleEndpointId,
531 corr: u64,
532 armed: bool,
533}
534
535impl ModuleControlRpcGuard {
536 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
537 Self {
538 forwarding,
539 endpoint,
540 corr,
541 armed: true,
542 }
543 }
544
545 fn disarm(&mut self) {
546 self.armed = false;
547 }
548}
549
550impl Drop for ModuleControlRpcGuard {
551 fn drop(&mut self) {
552 if self.armed {
553 let _ = self
554 .forwarding
555 .cancel_module_control_rpc(self.endpoint, self.corr);
556 }
557 }
558}
559
560impl RouteBindReservationGuard {
561 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
562 Self {
563 forwarding,
564 endpoint,
565 relay_corr,
566 armed: true,
567 }
568 }
569
570 fn release_and_disarm(&mut self) {
571 if !self.armed {
572 return;
573 }
574 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
575 self.endpoint,
576 self.relay_corr,
577 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
578 ) {
579 send_goodbye_target_best_effort(
580 &self.forwarding.counters(),
581 &target,
582 "canceled route.bind",
583 );
584 }
585 self.armed = false;
586 }
587
588 fn disarm(&mut self) {
589 self.armed = false;
590 }
591}
592
593impl Drop for RouteBindReservationGuard {
594 fn drop(&mut self) {
595 self.release_and_disarm();
596 }
597}
598
599#[derive(Debug, Clone, Default)]
629pub(crate) struct RouteBindBreakers {
630 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
631}
632
633#[derive(Debug, Clone, Default)]
634pub(crate) struct RouteBindConcurrency {
635 modules: Arc<Mutex<HashMap<String, usize>>>,
636}
637
638struct RouteBindConcurrencyGuard {
639 concurrency: RouteBindConcurrency,
640 module_id: String,
641}
642
643impl RouteBindConcurrency {
644 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
647 let mut modules = self
648 .modules
649 .lock()
650 .expect("route.bind concurrency mutex poisoned");
651 let in_flight = modules.entry(module_id.to_string()).or_default();
652 if *in_flight >= limit {
653 return Err(*in_flight);
654 }
655 *in_flight += 1;
656 Ok(RouteBindConcurrencyGuard {
657 concurrency: self.clone(),
658 module_id: module_id.to_string(),
659 })
660 }
661}
662
663impl Drop for RouteBindConcurrencyGuard {
664 fn drop(&mut self) {
665 let mut modules = self
666 .concurrency
667 .modules
668 .lock()
669 .expect("route.bind concurrency mutex poisoned");
670 let remove = {
671 let in_flight = modules
672 .get_mut(&self.module_id)
673 .expect("admitted route.bind has a concurrency entry");
674 *in_flight -= 1;
675 *in_flight == 0
676 };
677 if remove {
678 modules.remove(&self.module_id);
679 }
680 }
681}
682
683#[derive(Debug, Default)]
684struct ModuleBreakerState {
685 consecutive_timeouts: u32,
687 cooldown_until: Option<Instant>,
690 probe_in_flight: Option<Arc<()>>,
695}
696
697enum RouteBindAdmission<'a> {
699 Admitted {
700 guard: RouteBindBreakerGuard<'a>,
701 probe: bool,
704 },
705 Refused {
706 consecutive_timeouts: u32,
707 retry_in: Duration,
711 probe_in_flight: bool,
712 },
713}
714
715struct RouteBindBreakerGuard<'a> {
722 breakers: RouteBindBreakers,
723 module_id: &'a str,
724 probe_token: Option<Arc<()>>,
725 settled: bool,
726}
727
728impl RouteBindBreakerGuard<'_> {
729 fn record_accepted(&mut self) -> bool {
733 self.settled = true;
734 self.breakers.record_accepted(self.module_id)
735 }
736
737 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
740 self.settled = true;
741 self.breakers.record_timeout(
742 self.module_id,
743 self.probe_token.as_ref(),
744 threshold,
745 cooldown,
746 )
747 }
748
749 fn record_inconclusive(&mut self) {
758 self.settled = true;
759 self.breakers
760 .record_inconclusive(self.module_id, self.probe_token.as_ref());
761 }
762}
763
764impl Drop for RouteBindBreakerGuard<'_> {
765 fn drop(&mut self) {
766 if !self.settled {
767 self.breakers
768 .record_inconclusive(self.module_id, self.probe_token.as_ref());
769 }
770 }
771}
772
773struct BreakerOpened {
777 consecutive_timeouts: u32,
778 reopened_after_probe: bool,
781}
782
783impl RouteBindBreakers {
784 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
785 self.modules
786 .lock()
787 .expect("route.bind breaker mutex poisoned")
788 }
789
790 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
793 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
794 probe: probe_token.is_some(),
795 guard: RouteBindBreakerGuard {
796 breakers: self.clone(),
797 module_id,
798 probe_token,
799 settled: false,
800 },
801 };
802
803 let mut modules = self.lock();
804 let Some(state) = modules.get_mut(module_id) else {
805 return admitted(None);
806 };
807 let Some(cooldown_until) = state.cooldown_until else {
808 return admitted(None);
809 };
810 if state.probe_in_flight.is_some() {
811 return RouteBindAdmission::Refused {
812 consecutive_timeouts: state.consecutive_timeouts,
813 retry_in: Duration::ZERO,
814 probe_in_flight: true,
815 };
816 }
817 let now = Instant::now();
818 if now < cooldown_until {
819 return RouteBindAdmission::Refused {
820 consecutive_timeouts: state.consecutive_timeouts,
821 retry_in: cooldown_until - now,
822 probe_in_flight: false,
823 };
824 }
825 let token = Arc::new(());
826 state.probe_in_flight = Some(Arc::clone(&token));
827 admitted(Some(token))
828 }
829
830 fn record_accepted(&self, module_id: &str) -> bool {
831 self.lock()
832 .remove(module_id)
833 .is_some_and(|state| state.cooldown_until.is_some())
834 }
835
836 fn record_timeout(
837 &self,
838 module_id: &str,
839 probe_token: Option<&Arc<()>>,
840 threshold: u32,
841 cooldown: Duration,
842 ) -> Option<BreakerOpened> {
843 let mut modules = self.lock();
844 let state = modules.entry(module_id.to_string()).or_default();
845 let was_open = state.cooldown_until.is_some();
846 let was_probe = Self::owns_probe(state, probe_token);
847 if was_probe {
848 state.probe_in_flight = None;
849 }
850 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
851 if state.consecutive_timeouts < threshold {
852 return None;
853 }
854 state.cooldown_until = Some(Instant::now() + cooldown);
855 Some(BreakerOpened {
856 consecutive_timeouts: state.consecutive_timeouts,
857 reopened_after_probe: was_open && was_probe,
858 })
859 }
860
861 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
862 state
866 .probe_in_flight
867 .as_ref()
868 .zip(token)
869 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
870 }
871
872 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
873 if let Some(state) = self.lock().get_mut(module_id) {
874 if Self::owns_probe(state, probe_token) {
875 state.probe_in_flight = None;
876 }
877 }
878 }
879
880 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
895 self.lock()
896 .remove(module_id)
897 .map(|state| state.consecutive_timeouts)
898 .filter(|discarded| *discarded > 0)
899 }
900
901 fn open_snapshot(&self) -> Option<serde_json::Value> {
908 let now = Instant::now();
909 let modules = self.lock();
910 let open = modules
911 .iter()
912 .filter_map(|(module_id, state)| {
913 let cooldown_until = state.cooldown_until?;
914 Some((
915 module_id.clone(),
916 serde_json::json!({
917 "consecutive_timeouts": state.consecutive_timeouts,
918 "cooldown_remaining_ms":
919 cooldown_until.saturating_duration_since(now).as_millis() as u64,
920 "probe_in_flight": state.probe_in_flight.is_some(),
921 }),
922 ))
923 })
924 .collect::<serde_json::Map<String, serde_json::Value>>();
925 (!open.is_empty()).then_some(serde_json::Value::Object(open))
926 }
927}
928
929impl ControlHandler {
930 pub fn new(registry: Arc<Registry>) -> Self {
931 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
932 }
933
934 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
935 let counters = forwarding.counters();
936 let route_bind_breakers = forwarding.route_bind_breakers();
940 let route_bind_concurrency = forwarding.route_bind_concurrency();
941 let route_outages = forwarding.route_outages();
942 Self {
943 registry,
944 activity: None,
945 activity_streams: Arc::new(Mutex::new(HashMap::new())),
946 forwarding,
947 process_liveness: None,
948 supervisor: SupervisorHandle::new(),
949 subc_capabilities: Arc::from([
950 CAP_MANIFEST_REGISTRATION.to_string(),
951 CAP_CHANNEL_LIFECYCLE.to_string(),
952 CAP_PING_PONG.to_string(),
953 CAP_SESSION_ATTACH.to_string(),
954 CAP_ADMISSION_FACTS_RELAY.to_string(),
955 CAP_SCOPES_V1.to_string(),
956 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
957 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
958 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
959 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
960 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
961 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
962 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
963 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
964 ]),
965 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
966 route_bind_relay_timeouts: BTreeMap::new(),
967 route_bind_breakers,
968 route_bind_concurrency,
969 route_outages,
970 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
971 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
972 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
973 storage_config: None,
974 machine_id: None,
975 admission_facts_carrier_module_id: None,
976 admission_facts_targets: None,
977 scopes: Arc::new(RwLock::new(ScopeTable::new(
978 crate::daemon_config::default_scope_authority_owners(),
979 ))),
980 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
981 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
982 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
983 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
984 #[cfg(test)]
985 caller_lookup: None,
986 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
987 rescan: None,
988 connected_clients: ConnectedClients::new(),
989 counters,
990 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
991 daemon_shutdown: DaemonShutdownFlag::default(),
992 daemon_provenance: DaemonProvenanceFacts::default(),
993 #[cfg(test)]
994 control_dispatch_delay: None,
995 #[cfg(test)]
996 provenance_probe_override: None,
997 #[cfg(test)]
998 route_bind_relay_sent_hold: None,
999 #[cfg(test)]
1000 hello_registry_to_forwarding_hold: None,
1001 }
1002 }
1003
1004 pub fn with_storage_config(
1007 mut self,
1008 storage_config: Option<crate::daemon_config::StorageConfig>,
1009 ) -> Self {
1010 self.storage_config = storage_config;
1011 self
1012 }
1013
1014 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1017 self.activity = Some(crate::activity::ActivityLedger::open(
1018 run_dir.join("activity.jsonl"),
1019 incarnation,
1020 ));
1021 self.supervisor.set_activity(self.activity.clone());
1022 self
1023 }
1024
1025 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1028 self.machine_id = machine_id;
1029 self
1030 }
1031
1032 pub fn with_admission_facts_config(
1036 mut self,
1037 carrier_module_id: Option<String>,
1038 targets: Option<Vec<String>>,
1039 ) -> Self {
1040 self.admission_facts_carrier_module_id = carrier_module_id;
1041 self.admission_facts_targets = targets;
1042 self
1043 }
1044
1045 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1049 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1050 self.scope_authority_owners = owners;
1051 self
1052 }
1053
1054 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1056 self.privileged_operator_prompt = mode;
1057 self
1058 }
1059
1060 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1063 self.route_bind_relay_timeout = timeout;
1064 self
1065 }
1066
1067 pub fn with_route_bind_relay_timeouts(
1073 mut self,
1074 timeouts: impl IntoIterator<Item = (String, Duration)>,
1075 ) -> Self {
1076 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1077 self
1078 }
1079
1080 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1086 self.route_bind_relay_timeouts
1087 .get(module_id)
1088 .copied()
1089 .unwrap_or(self.route_bind_relay_timeout)
1090 }
1091
1092 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1100 self.route_bind_breaker_threshold = threshold.max(1);
1101 self.route_bind_breaker_cooldown = cooldown;
1102 self
1103 }
1104
1105 #[cfg(test)]
1106 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1107 self.health_probe_timeout = timeout;
1108 self
1109 }
1110
1111 #[cfg(test)]
1112 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1113 self.control_dispatch_delay = Some(delay);
1114 self
1115 }
1116
1117 #[cfg(test)]
1121 pub(crate) fn with_route_bind_relay_sent_hold(
1122 mut self,
1123 hold: Arc<tokio::sync::Barrier>,
1124 ) -> Self {
1125 self.route_bind_relay_sent_hold = Some(hold);
1126 self
1127 }
1128
1129 #[cfg(test)]
1134 pub(crate) fn with_hello_registry_to_forwarding_hold(
1135 mut self,
1136 hold: Arc<std::sync::Barrier>,
1137 ) -> Self {
1138 self.hello_registry_to_forwarding_hold = Some(hold);
1139 self
1140 }
1141
1142 #[cfg(test)]
1143 fn hold_between_registry_and_forwarding(&self) {
1144 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1145 hold.wait();
1146 hold.wait();
1147 }
1148 }
1149
1150 pub fn with_process_liveness(
1151 mut self,
1152 process_liveness: Arc<dyn ModuleProcessLiveness>,
1153 ) -> Self {
1154 self.process_liveness = Some(process_liveness);
1155 self
1156 }
1157
1158 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1159 self.supervisor = supervisor;
1160 self.supervisor.set_activity(self.activity.clone());
1161 self
1162 }
1163
1164 pub fn with_daemon_provenance(
1165 mut self,
1166 pid: u32,
1167 started_at_ms: u64,
1168 executable_path: Option<PathBuf>,
1169 build_git_sha: Option<String>,
1170 build_lock_digest: Option<String>,
1171 ) -> Self {
1172 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1173 let process_start_time = process_start_time(pid);
1174 self.daemon_provenance = DaemonProvenanceFacts {
1175 build: DaemonBuildProvenance {
1176 build_git_sha,
1177 build_lock_digest,
1178 },
1179 pid: Some(pid),
1180 started_at_ms: Some(started_at_ms),
1181 start_clock: None,
1182 executable_path,
1183 executable_identity,
1184 process_start_time,
1185 probe: ExecutableIdentityProbe::default(),
1186 };
1187 self
1188 }
1189
1190 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1191 self.daemon_provenance.start_clock = Some(clock);
1192 self
1193 }
1194
1195 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1197 self.scopes
1198 .write()
1199 .unwrap_or_else(|poisoned| poisoned.into_inner())
1200 .set_wall_clock(Arc::new(clock));
1201 self
1202 }
1203
1204 #[cfg(test)]
1205 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1206 self.provenance_probe_override = Some(result);
1207 self
1208 }
1209
1210 pub fn with_capability_config(
1214 self,
1215 modules: impl IntoIterator<Item = (String, bool)>,
1216 reserved_capabilities: BTreeMap<String, String>,
1217 ) -> Self {
1218 self.capability_evaluator
1219 .configure(modules, reserved_capabilities);
1220 self
1221 }
1222
1223 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1227 self.daemon_shutdown = flag;
1228 self
1229 }
1230
1231 pub fn with_supervisor_rescan(
1232 mut self,
1233 supervisor: Supervisor,
1234 config_path: impl Into<PathBuf>,
1235 configured_port: Option<u16>,
1236 ) -> Self {
1237 self.rescan = Some(SupervisorRescanContext {
1238 supervisor,
1239 config_path: config_path.into(),
1240 configured_port,
1241 storage_config: self.storage_config.clone(),
1242 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1243 admission_facts_targets: self.admission_facts_targets.clone(),
1244 scope_authority_owners: self.scope_authority_owners.clone(),
1245 privileged_operator_prompt: self.privileged_operator_prompt,
1246 generation: Arc::default(),
1247 });
1248 self
1249 }
1250
1251 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1252 self.connected_clients = connected_clients;
1253 self
1254 }
1255
1256 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1257 Arc::clone(&self.forwarding)
1258 }
1259
1260 pub(crate) fn counters(&self) -> DaemonCounters {
1261 self.counters.clone()
1262 }
1263
1264 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1267 tokio::spawn(async move {
1268 loop {
1269 self.capability_evaluator
1270 .wait_for_change_or_deadline()
1271 .await;
1272 self.refresh_capability_requirements();
1273 }
1274 });
1275 }
1276
1277 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1283 tokio::spawn(async move {
1284 loop {
1285 let has_deadlines = self
1286 .scopes
1287 .read()
1288 .unwrap_or_else(|poisoned| poisoned.into_inner())
1289 .has_deadlines();
1290 if !has_deadlines {
1291 self.scope_deadline_added.notified().await;
1292 continue;
1293 }
1294 tokio::time::sleep(Duration::from_secs(1)).await;
1295 if let Err(error) = self.sweep_expired_scopes() {
1296 warn!(%error, "scope expiry sweep failed");
1297 }
1298 }
1299 });
1300 }
1301
1302 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1303 let mut table = self
1304 .scopes
1305 .write()
1306 .unwrap_or_else(|poisoned| poisoned.into_inner());
1307 let swept = table.sweep_expired();
1308 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1309 drop(table);
1310 self.close_scope_drained_routes(drained);
1311 Ok(())
1312 }
1313
1314 fn publish_scope_changes(
1315 &self,
1316 changes: &[crate::scopes::ScopeTagChange],
1317 expired: &[crate::scopes::ScopeExpired],
1318 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1319 if changes.is_empty() {
1320 return Ok(Vec::new());
1321 }
1322 let drained = self
1323 .forwarding
1324 .publish_scope_changes(changes)
1325 .map_err(RouterError::Forwarding)?;
1326 let mut counts = HashMap::new();
1327 for route in &drained {
1328 *counts
1329 .entry((
1330 route.scope.owner.as_str(),
1331 route.scope.scope_ref.as_str(),
1332 route.scope.tag.scope_epoch,
1333 ))
1334 .or_insert(0usize) += 1;
1335 }
1336 for scope in expired {
1337 let routes_closed = counts
1338 .get(&(
1339 scope.owner.as_str(),
1340 scope.scope_ref.as_str(),
1341 scope.scope_epoch,
1342 ))
1343 .copied()
1344 .unwrap_or(0);
1345 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1346 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1347 routes_closed, "scope expired");
1348 }
1349 Ok(drained)
1350 }
1351
1352 fn runtime_capability_snapshot(
1353 &self,
1354 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1355 let runtime = self
1356 .supervisor
1357 .list()
1358 .into_iter()
1359 .map(|module| {
1360 let status = module.status().map_err(|err| {
1361 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1362 })?;
1363 Ok(RuntimeModule {
1364 module_id: status.module_id,
1365 state: status.state,
1366 enabled: status.enabled,
1367 })
1368 })
1369 .collect::<Result<Vec<_>, RouterError>>()?;
1370 let registrations = self.registry.list_registered_modules().map_err(|err| {
1376 RouterError::backend(
1377 0,
1378 0,
1379 format!("failed to list capability registrations: {err}"),
1380 )
1381 })?;
1382 let registrations = registrations
1383 .into_iter()
1384 .map(|registration| RegisteredModule {
1385 module_id: registration.manifest.module_id,
1386 module_version: registration.manifest.module_version,
1387 capabilities: registration.manifest.capabilities,
1388 })
1389 .collect();
1390 Ok((runtime, registrations))
1391 }
1392
1393 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1400 let cached_registration = RegisteredModule {
1401 module_id: registration.manifest.module_id.clone(),
1402 module_version: registration.manifest.module_version.clone(),
1403 capabilities: registration.manifest.capabilities.clone(),
1404 };
1405 if self.capability_evaluator.record_hello(&cached_registration) {
1406 warn!(
1407 module_id = %cached_registration.module_id,
1408 "capability claims drifted from the cached manifest"
1409 );
1410 }
1411 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1412 self.enforce_capability_denies();
1413 }
1414 self.refresh_capability_requirements();
1415 }
1416
1417 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1421 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1422 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1423 self.supervisor.set_swap_promotion_observer(observer);
1424 }
1425
1426 pub fn refresh_capability_requirements(&self) {
1427 match self.runtime_capability_snapshot() {
1428 Ok((runtime, registrations)) => {
1429 log_requirement_events(
1430 self.capability_evaluator
1431 .evaluate_now(&runtime, ®istrations),
1432 &self.daemon_shutdown,
1433 );
1434 }
1435 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1436 }
1437 }
1438
1439 fn enforce_capability_denies(&self) {
1443 let registrations = match self.registry.list_registered_modules() {
1444 Ok(snapshot) => snapshot,
1445 Err(err) => {
1446 warn!(error = %err, "failed to read registrations for capability deny census");
1447 return;
1448 }
1449 };
1450 let manifests = registrations
1451 .into_iter()
1452 .map(|registration| {
1453 (
1454 registration.manifest.module_id.clone(),
1455 registration.manifest,
1456 )
1457 })
1458 .collect::<BTreeMap<_, _>>();
1459 let census = match self.forwarding.route_census(None) {
1460 Ok(census) => census,
1461 Err(err) => {
1462 warn!(error = %err, "failed to read route census for capability deny enforcement");
1463 return;
1464 }
1465 };
1466
1467 for (target_module_id, routes) in census {
1468 let Some(target_manifest) = manifests.get(&target_module_id) else {
1469 continue;
1470 };
1471 let mut closed_routes = Vec::new();
1472 let mut module_goodbyes = Vec::new();
1473 for route in routes {
1474 let Principal::Reserved {
1475 module_id: opening_module_id,
1476 } = &route.principal
1477 else {
1478 continue;
1479 };
1480 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1481 continue;
1482 };
1483 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1484 continue;
1485 };
1486
1487 match self.forwarding.release_client_route(
1488 route.goodbye_target.connection_id,
1489 route.goodbye_target.channel,
1490 route.goodbye_target.epoch,
1491 ) {
1492 Ok(RouteRelease::Removed(module_goodbye)) => {
1493 warn!(
1494 opening_module_id,
1495 target_module_id,
1496 capability,
1497 "force-closing route because an attested capability deny edge now matches"
1498 );
1499 closed_routes.push(route);
1500 module_goodbyes.push(module_goodbye);
1501 }
1502 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1503 Err(err) => warn!(
1504 opening_module_id,
1505 target_module_id,
1506 capability,
1507 error = %err,
1508 "failed to force-close capability-denied route"
1509 ),
1510 }
1511 }
1512
1513 if closed_routes.is_empty() {
1514 continue;
1515 }
1516 send_route_control_pushes(
1517 &self.forwarding,
1518 closed_routes,
1519 ClientControlPush::RouteClosed {
1520 module_id: target_module_id,
1521 channels: Vec::new(),
1522 reason: RouteCloseReason::CapabilityDenied,
1523 drained: false,
1524 abandoned: 0,
1525 excluded_subscriptions: 0,
1526 terminal: Some(false),
1527 },
1528 );
1529 self.emit_route_goodbyes(module_goodbyes);
1530 }
1531 }
1532
1533 fn not_ready_reason(
1539 &self,
1540 registration: &crate::registry::ModuleRegistration,
1541 ) -> Option<NotReadyReason> {
1542 if !registration.ready {
1543 return Some(NotReadyReason {
1544 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1545 capability: None,
1546 });
1547 }
1548 self.first_unprovided_required_capability(registration)
1549 .map(|capability| NotReadyReason {
1550 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1551 capability: Some(capability),
1552 })
1553 }
1554
1555 fn first_unprovided_required_capability(
1573 &self,
1574 registration: &crate::registry::ModuleRegistration,
1575 ) -> Option<String> {
1576 let required = registration
1577 .manifest
1578 .capabilities
1579 .iter()
1580 .flat_map(|declarations| declarations.requires.iter())
1581 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1582 .map(|requirement| requirement.capability.as_str())
1583 .collect::<BTreeSet<_>>();
1584 if required.is_empty() {
1585 return None;
1586 }
1587 let module_id = registration.manifest.module_id.as_str();
1588 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1589 if required
1590 .iter()
1591 .any(|capability| verdict(capability).is_none())
1592 {
1593 self.refresh_capability_requirements();
1594 }
1595 required
1596 .into_iter()
1597 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1598 .map(str::to_string)
1599 }
1600
1601 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1602 self.capability_evaluator
1603 .statuses()
1604 .into_iter()
1605 .map(capability_requirement_status)
1606 .collect()
1607 }
1608
1609 fn deregister_connection(
1616 &self,
1617 connection_id: ConnectionId,
1618 reason: RegistrationEndReason,
1619 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1620 self.registry
1621 .deregister_connection_with_reason(connection_id, reason)
1622 }
1623
1624 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1625 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1626 return None;
1627 }
1628 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1629 parse_client_control_request(&frame.body)
1630 else {
1631 return None;
1632 };
1633 Some(target_module_id(&target).to_string())
1634 }
1635
1636 pub(crate) fn route_open_capacity_refusal(
1637 &self,
1638 ctx: &RouteCtx,
1639 frame: &Frame,
1640 target_module_id: &str,
1641 in_flight: usize,
1642 limit: usize,
1643 ) -> Result<Frame, RouterError> {
1644 self.route_open_admission_refusal_frame(
1645 ctx,
1646 frame,
1647 target_module_id,
1648 "open_admission_full",
1649 (in_flight, limit),
1650 format!(
1651 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1652 ),
1653 )
1654 }
1655
1656 fn route_open_target_capacity_refusal(
1657 &self,
1658 ctx: &RouteCtx,
1659 frame: &Frame,
1660 target_module_id: &str,
1661 in_flight: usize,
1662 ) -> Result<Frame, RouterError> {
1663 self.route_open_admission_refusal_frame(
1664 ctx,
1665 frame,
1666 target_module_id,
1667 "target_binds_full",
1668 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1669 format!(
1670 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1671 ),
1672 )
1673 }
1674
1675 fn route_open_admission_refusal_frame(
1683 &self,
1684 ctx: &RouteCtx,
1685 frame: &Frame,
1686 target_module_id: &str,
1687 reason: &'static str,
1688 (in_flight, limit): (usize, usize),
1689 message: impl Into<String>,
1690 ) -> Result<Frame, RouterError> {
1691 let code = error_codes::TARGET_UNAVAILABLE;
1692 self.counters.increment_route_open_refused(code);
1693 info!(
1694 target: "control",
1695 code,
1696 reason,
1697 module_id = ?target_module_id,
1698 connection_id = ctx.connection_id.get(),
1699 in_flight,
1700 limit,
1701 "route.open refused"
1702 );
1703 control_error_frame(frame, code, message.into())
1704 }
1705
1706 #[cfg(test)]
1712 pub fn handle_control(
1713 &self,
1714 connection_id: ConnectionId,
1715 frame: Frame,
1716 ) -> Result<Vec<Frame>, RouterError> {
1717 match frame.header.ty {
1718 FrameType::Ping => Ok(vec![pong(&frame)?]),
1719 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1720 FrameType::Goodbye => self.handle_goodbye(connection_id),
1721 ty => Ok(vec![control_error_frame(
1722 &frame,
1723 "unsupported_control_frame",
1724 format!("unsupported channel-0 frame {ty:?}"),
1725 )?]),
1726 }
1727 }
1728
1729 pub async fn handle_control_frame(
1730 &self,
1731 ctx: &RouteCtx,
1732 frame: Frame,
1733 ) -> Result<Vec<Frame>, RouterError> {
1734 self.handle_control_frame_timed(ctx, frame, None).await
1735 }
1736
1737 pub(crate) fn record_caller_endpoints(
1738 &self,
1739 connection: ConnectionId,
1740 endpoints: Option<(SocketAddr, SocketAddr)>,
1741 ) {
1742 let mut callers = self
1743 .caller_endpoints
1744 .lock()
1745 .unwrap_or_else(|p| p.into_inner());
1746 callers.remove(&connection);
1747 if let Some(endpoints) = endpoints {
1748 callers.insert(connection, endpoints);
1749 }
1750 }
1751
1752 #[cfg(test)]
1753 pub(crate) fn with_caller_lookup_for_test(
1754 mut self,
1755 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1756 ) -> Self {
1757 self.caller_lookup = Some(Arc::new(lookup));
1758 self
1759 }
1760
1761 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1762 let endpoints = self
1763 .caller_endpoints
1764 .lock()
1765 .unwrap_or_else(|p| p.into_inner())
1766 .get(&connection)
1767 .copied();
1768 #[cfg(test)]
1769 let lookup = self.caller_lookup.clone();
1770 tokio::task::spawn_blocking(move || {
1771 #[cfg(test)]
1772 if let Some(lookup) = lookup {
1773 return lookup(endpoints);
1774 }
1775 endpoints
1776 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1777 .unwrap_or_default()
1778 })
1779 .await
1780 .unwrap_or_default()
1781 }
1782
1783 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1784 frame.header.channel == 0
1785 && frame.header.ty == FrameType::Request
1786 && self
1787 .forwarding
1788 .module_endpoint_for_connection(ctx.connection_id)
1789 .is_ok_and(|endpoint| endpoint.is_none())
1790 && parse_client_control_request(&frame.body)
1791 .ok()
1792 .is_some_and(|request| privileged_operation(&request).is_some())
1793 }
1794
1795 pub(crate) async fn handle_control_frame_timed(
1796 &self,
1797 ctx: &RouteCtx,
1798 frame: Frame,
1799 dispatch_started_at: Option<StdInstant>,
1800 ) -> Result<Vec<Frame>, RouterError> {
1801 match frame.header.ty {
1802 FrameType::Ping => Ok(vec![pong(&frame)?]),
1803 FrameType::Hello => {
1804 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1805 }
1806 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1807 FrameType::Cancel => {
1808 if self
1811 .forwarding
1812 .operator_confirms()
1813 .cancel(ctx.connection_id, frame.header.corr)
1814 || self
1815 .supervisor
1816 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1817 || self.cancel_activity_subscription(ctx, &frame)?
1818 {
1819 Ok(Vec::new())
1820 } else {
1821 Ok(vec![control_error_frame(
1822 &frame,
1823 "unknown_subscription",
1824 "no subscription has this correlation id",
1825 )?])
1826 }
1827 }
1828 FrameType::Request => {
1829 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1832 if op
1833 .as_ref()
1834 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1835 {
1836 return self.handle_activity_record(ctx, frame).await;
1837 }
1838 if op.as_ref().is_some_and(|probe| {
1839 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1840 }) {
1841 return self.handle_activity_read(ctx, frame).await;
1842 }
1843 if op
1844 .as_ref()
1845 .is_some_and(|probe| probe.op == "operator.confirm")
1846 {
1847 return self.handle_operator_confirm(ctx, frame);
1848 }
1849 if op
1850 .as_ref()
1851 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1852 {
1853 return self.handle_scope_served_by(ctx.connection_id, frame);
1854 }
1855 if self
1856 .forwarding
1857 .module_endpoint_for_connection(ctx.connection_id)
1858 .map_err(RouterError::Forwarding)?
1859 .is_some()
1860 {
1861 if !is_known_module_request_op(&frame.body) {
1862 return Ok(vec![control_error_frame(
1863 &frame,
1864 "unsupported_control_frame",
1865 "module-originated channel-0 REQUEST is not supported",
1866 )?]);
1867 }
1868 let request = match parse_module_control_request_from_module(&frame.body) {
1869 Ok(request) => request,
1870 Err((err, ControlRequestBodyError::UnknownOp)) => {
1871 return Ok(vec![control_error_frame(
1872 &frame,
1873 "unsupported_control_frame",
1874 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1875 )?])
1876 }
1877 Err((err, ControlRequestBodyError::InvalidBody)) => {
1878 return Ok(vec![control_error_frame(
1879 &frame,
1880 "invalid_control_body",
1881 format!("malformed module control body: {err}"),
1882 )?])
1883 }
1884 };
1885 let op = module_control_request_op(&request);
1886 let corr = frame.header.corr;
1887 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1888 let result =
1889 self.handle_module_control_request(ctx.connection_id, frame, request);
1890 log_slow_control_dispatch(
1891 dispatch_started_at,
1892 op,
1893 ctx.connection_id,
1894 corr,
1895 None,
1896 );
1897 return result;
1898 }
1899
1900 if is_known_module_request_op(&frame.body) {
1901 return Ok(vec![control_error_frame(
1902 &frame,
1903 "not_registered",
1904 "catalog.update requires an active module registration owned by this connection",
1905 )?]);
1906 }
1907
1908 let request = match parse_client_control_request(&frame.body) {
1909 Ok(request) => request,
1910 Err((err, ControlRequestBodyError::UnknownOp)) => {
1911 return Ok(vec![control_error_frame(
1912 &frame,
1913 "unknown_control_op",
1914 format!("unknown client control op: {err}"),
1915 )?])
1916 }
1917 Err((err, ControlRequestBodyError::InvalidBody)) => {
1918 return Ok(vec![control_error_frame(
1919 &frame,
1920 "invalid_control_body",
1921 format!("malformed client control body: {err}"),
1922 )?])
1923 }
1924 };
1925 let op = client_control_request_op(&request);
1926 let corr = frame.header.corr;
1927 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1928 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1929 #[cfg(test)]
1930 if let Some(delay) = self.control_dispatch_delay {
1931 tokio::time::sleep(delay).await;
1932 }
1933 let result = self
1934 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1935 .await;
1936 if let Some(timings) = route_open_timings.as_mut() {
1937 timings.finish();
1938 }
1939 log_slow_control_dispatch(
1940 dispatch_started_at,
1941 op,
1942 ctx.connection_id,
1943 corr,
1944 route_open_timings.as_ref(),
1945 );
1946 result
1947 }
1948 FrameType::Push => {
1949 let Some(endpoint) = self
1950 .forwarding
1951 .module_endpoint_for_connection(ctx.connection_id)
1952 .map_err(RouterError::Forwarding)?
1953 else {
1954 return Ok(vec![control_error_frame(
1955 &frame,
1956 "unsupported_control_frame",
1957 "client-originated channel-0 PUSH is not supported",
1958 )?]);
1959 };
1960 self.handle_status_update(endpoint, frame)
1961 }
1962 FrameType::Response | FrameType::Error
1963 if self
1964 .forwarding
1965 .module_endpoint_for_connection(ctx.connection_id)
1966 .map_err(RouterError::Forwarding)?
1967 .is_some() =>
1968 {
1969 self.handle_module_relay_response(ctx.connection_id, frame)
1970 }
1971 ty => Ok(vec![control_error_frame(
1972 &frame,
1973 "unsupported_control_frame",
1974 format!("unsupported channel-0 frame {ty:?}"),
1975 )?]),
1976 }
1977 }
1978
1979 pub fn cleanup_connection(
1980 &self,
1981 connection_id: ConnectionId,
1982 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1983 self.caller_endpoints
1984 .lock()
1985 .unwrap_or_else(|p| p.into_inner())
1986 .remove(&connection_id);
1987 self.forwarding
1988 .operator_confirms()
1989 .connection_closed(connection_id);
1990 self.cleanup_connection_with_end_reason(
1991 connection_id,
1992 RegistrationEndReason::ConnectionClosed,
1993 )
1994 }
1995
1996 fn cleanup_connection_with_end_reason(
1997 &self,
1998 connection_id: ConnectionId,
1999 requested_reason: RegistrationEndReason,
2000 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2001 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2002 self.registry
2003 .get_module_by_connection(connection_id)?
2004 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2005 .and_then(|module| module.registration_end_reason().ok().flatten())
2006 .unwrap_or(requested_reason)
2007 } else {
2008 requested_reason
2009 };
2010 let crash_closed = self
2011 .registry
2012 .get_module_by_connection(connection_id)?
2013 .and_then(|registration| {
2014 self.forwarding
2015 .module_endpoint_for_connection(connection_id)
2016 .ok()
2017 .flatten()
2018 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2019 .map(|routes| (registration.manifest.module_id, routes))
2020 });
2021 let crash_closed = crash_closed.map(|(module_id, routes)| {
2022 let terminal = match self.supervisor.get(&module_id) {
2023 None => false,
2024 Some(module) => match module.will_recover_after_connection_loss() {
2025 Ok(will_recover) => !will_recover,
2026 Err(err) => {
2027 warn!(
2028 %module_id,
2029 error = %err,
2030 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2031 );
2032 false
2033 }
2034 },
2035 };
2036 let (reason, drained) = match self
2040 .forwarding
2041 .daemon_drain_result_for_connection(connection_id)
2042 {
2043 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2044 Ok(None) => (RouteCloseReason::Crash, false),
2045 Err(err) => {
2046 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2047 (RouteCloseReason::Crash, false)
2048 }
2049 };
2050 (module_id, routes, reason, drained, terminal)
2051 });
2052 let registrations = self.deregister_connection(connection_id, end_reason);
2053 let cleanup = if crash_closed.is_some() {
2054 self.forwarding.cleanup_connection_counted(connection_id)
2055 } else {
2056 self.forwarding
2062 .cleanup_connection(connection_id)
2063 .map(|released| crate::forwarding::ConnectionCleanup {
2064 released,
2065 abandoned_relays: 0,
2066 })
2067 };
2068 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2073 let abandoned = cleanup
2074 .as_ref()
2075 .map_or(0, |cleanup| cleanup.abandoned_relays);
2076 send_route_control_pushes(
2077 &self.forwarding,
2078 routes,
2079 ClientControlPush::RouteClosed {
2080 module_id,
2081 channels: Vec::new(),
2082 reason,
2083 drained,
2084 abandoned,
2085 excluded_subscriptions: 0,
2086 terminal: Some(terminal),
2087 },
2088 );
2089 }
2090 if let Ok(cleanup) = cleanup {
2091 self.emit_route_goodbyes(cleanup.released);
2092 }
2093 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2097 crate::supervise::notify_registration_release();
2098 self.capability_evaluator.wake_deadline_loop();
2099 self.refresh_capability_requirements();
2100 }
2101 self.supervisor.remove_spawn_subscribers(connection_id);
2102 self.activity_streams
2103 .lock()
2104 .unwrap_or_else(|p| p.into_inner())
2105 .retain(|(id, _), task| {
2106 if *id == connection_id {
2107 task.abort();
2108 false
2109 } else {
2110 true
2111 }
2112 });
2113 self.hello_launch_nonces
2116 .lock()
2117 .unwrap_or_else(|poisoned| poisoned.into_inner())
2118 .forget(connection_id);
2119 self.scopes
2120 .write()
2121 .unwrap_or_else(|poisoned| poisoned.into_inner())
2122 .release_connection(connection_id);
2123 registrations
2124 }
2125
2126 pub(crate) fn handle_route_goodbye(
2127 &self,
2128 connection_id: ConnectionId,
2129 route_channel: u16,
2130 route_epoch: u32,
2131 ) -> Result<bool, RouterError> {
2132 debug!(
2133 connection_id = connection_id.get(),
2134 route_channel, route_epoch, "handling route GOODBYE"
2135 );
2136 let RouteRelease::Removed(released_route) = self
2137 .forwarding
2138 .release_client_route(connection_id, route_channel, route_epoch)
2139 .map_err(RouterError::Forwarding)?
2140 else {
2141 return Ok(false);
2142 };
2143 self.emit_route_goodbyes(vec![released_route]);
2144 Ok(true)
2145 }
2146
2147 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2148 for released in released_routes {
2149 let frame = match Frame::build_with_version(
2150 released.negotiated_ver,
2151 FrameType::Goodbye,
2152 control_flags(),
2153 released.channel,
2154 released.epoch,
2155 0,
2156 Vec::new(),
2157 ) {
2158 Ok(frame) => frame,
2159 Err(err) => {
2160 warn!(
2161 route_channel = released.channel,
2162 error = %err,
2163 "failed to build route GOODBYE frame"
2164 );
2165 continue;
2166 }
2167 };
2168 if !released.close_on_delivery_failure() {
2169 crate::forwarding::send_module_route_goodbye(
2170 &self.counters,
2171 &released.sink,
2172 frame,
2173 released.module_id.as_deref(),
2174 "client route released",
2175 );
2176 continue;
2177 }
2178 if let Err(err) = released.sink.try_send(frame) {
2179 warn!(
2180 target_connection_id = released.connection_id.get(),
2181 route_channel = released.channel,
2182 error = %err,
2183 "route GOODBYE was not delivered to client; closing target connection"
2184 );
2185 if self
2186 .forwarding
2187 .escalate_client_delivery_failure(
2188 released.connection_id,
2189 released.channel,
2190 released.epoch,
2191 CloseReason::new(
2192 "route_goodbye_delivery_failed",
2193 format!(
2194 "failed to enqueue route GOODBYE for channel {}: {err}",
2195 released.channel
2196 ),
2197 ),
2198 crate::forwarding::UndeliveredFrame {
2199 module_id: released.module_id.as_deref(),
2200 sink: &released.sink,
2201 },
2202 )
2203 .unwrap_or(false)
2204 {
2205 self.counters.increment_goodbye_relay_client_failed();
2206 }
2207 }
2208 }
2209 }
2210
2211 fn send_abandoned_route_bind_goodbye(
2223 &self,
2224 module_sink: &crate::FrameSink,
2225 negotiated_ver: u8,
2226 module_channel: u16,
2227 module_epoch: u32,
2228 ) {
2229 let frame = match Frame::build_with_version(
2230 negotiated_ver,
2231 FrameType::Goodbye,
2232 control_flags(),
2233 module_channel,
2234 module_epoch,
2235 0,
2236 Vec::new(),
2237 ) {
2238 Ok(frame) => frame,
2239 Err(err) => {
2240 warn!(
2241 route_channel = module_channel,
2242 error = %err,
2243 "failed to build GOODBYE for abandoned route.bind"
2244 );
2245 return;
2246 }
2247 };
2248 crate::forwarding::send_module_route_goodbye(
2249 &self.counters,
2250 module_sink,
2251 frame,
2252 None,
2253 "abandoned route.bind",
2254 );
2255 }
2256
2257 fn handle_hello(
2258 &self,
2259 connection_id: ConnectionId,
2260 sink: Option<crate::FrameSink>,
2261 frame: Frame,
2262 ) -> Result<Vec<Frame>, RouterError> {
2263 debug!(
2264 connection_id = connection_id.get(),
2265 corr = frame.header.corr,
2266 "handling HELLO"
2267 );
2268 if self
2272 .registry
2273 .get_module_by_connection(connection_id)
2274 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2275 .is_some()
2276 {
2277 return Ok(vec![control_error_frame(
2278 &frame,
2279 "invalid_hello",
2280 "connection is already registered as a module",
2281 )?]);
2282 }
2283 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2284 Ok(value) => value,
2285 Err(err) => {
2286 return Ok(vec![control_error_frame(
2287 &frame,
2288 "invalid_hello",
2289 format!("malformed HELLO body: {err}"),
2290 )?])
2291 }
2292 };
2293 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2294 return Ok(vec![control_error_frame(
2295 &frame,
2296 "invalid_capability_grammar",
2297 err.to_string(),
2298 )?]);
2299 }
2300 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2301 return Ok(vec![control_error_frame(
2302 &frame,
2303 "invalid_manifest",
2304 err.to_string(),
2305 )?]);
2306 }
2307 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2308 return Ok(vec![control_error_frame(
2309 &frame,
2310 "invalid_event_declaration",
2311 err.to_string(),
2312 )?]);
2313 }
2314 if let Some(provenance) = hello_value
2315 .get("manifest")
2316 .and_then(|manifest| manifest.get("provenance"))
2317 {
2318 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2319 return Ok(vec![control_error_frame(
2320 &frame,
2321 "invalid_manifest",
2322 format!("malformed manifest provenance: {err}"),
2323 )?]);
2324 }
2325 }
2326 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2327 Ok(hello) => hello,
2328 Err(err) => {
2329 return Ok(vec![control_error_frame(
2330 &frame,
2331 "invalid_hello",
2332 format!("malformed HELLO body: {err}"),
2333 )?])
2334 }
2335 };
2336
2337 if hello.protocol_ver != hello.manifest.protocol_ver {
2338 return Ok(vec![control_error_frame(
2339 &frame,
2340 "invalid_manifest",
2341 format!(
2342 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2343 hello.protocol_ver, hello.manifest.protocol_ver
2344 ),
2345 )?]);
2346 }
2347
2348 if hello.manifest.module_id.trim().is_empty() {
2349 return Ok(vec![control_error_frame(
2350 &frame,
2351 "invalid_manifest",
2352 "manifest module_id must not be empty",
2353 )?]);
2354 }
2355
2356 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2357 Ok(negotiated_ver) => negotiated_ver,
2358 Err(message) => {
2359 return Ok(vec![control_error_frame(
2360 &frame,
2361 "version_unsupported",
2362 message,
2363 )?])
2364 }
2365 };
2366
2367 let swap_admission = self
2375 .supervisor
2376 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2377 if swap_admission == SwapHelloAdmission::Refused {
2378 warn!(
2379 module_id = %hello.manifest.module_id,
2380 connection_id = connection_id.get(),
2381 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2382 );
2383 return Ok(vec![control_error_frame(
2384 &frame,
2385 "swap_token_invalid",
2386 format!(
2387 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2388 hello.manifest.module_id
2389 ),
2390 )?]);
2391 }
2392 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2393
2394 if let Some(rejection) = (!swap_candidate)
2402 .then(|| {
2403 self.supervisor.reserved_hello_rejection(
2404 &hello.manifest.module_id,
2405 hello.launch_nonce.as_deref(),
2406 )
2407 })
2408 .flatten()
2409 {
2410 let message = match rejection {
2411 ReservedHelloRejection::Exact { module_id } => format!(
2412 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2413 ),
2414 ReservedHelloRejection::Prefix {
2415 prefix,
2416 owner_module_id,
2417 } => format!(
2418 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2419 hello.manifest.module_id
2420 ),
2421 };
2422 return Ok(vec![control_error_frame(
2423 &frame,
2424 "reserved_module",
2425 message,
2426 )?]);
2427 }
2428
2429 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2430 &hello.manifest.module_id,
2431 hello.manifest.capabilities.as_ref(),
2432 );
2433 if let Some(refusal) = reserved_capability_refusals.first() {
2434 let capability = refusal.capability.clone();
2435 let bound_module = refusal.claimants[0].clone();
2436 log_duplicate_claim_events(reserved_capability_refusals);
2437 return Ok(vec![control_error_frame(
2438 &frame,
2439 "reserved_capability",
2440 format!(
2441 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2442 capability, bound_module, hello.manifest.module_id
2443 ),
2444 )?]);
2445 }
2446
2447 if self
2450 .forwarding
2451 .connection_has_client_routes(connection_id)
2452 .map_err(RouterError::Forwarding)?
2453 {
2454 return Ok(vec![control_error_frame(
2455 &frame,
2456 "invalid_hello",
2457 "connection has open client routes and cannot also register as a module",
2458 )?]);
2459 }
2460
2461 self.hello_launch_nonces
2466 .lock()
2467 .unwrap_or_else(|poisoned| poisoned.into_inner())
2468 .record(connection_id, hello.launch_nonce.as_deref());
2469 let control_ops = effective_module_control_ops(hello.control_ops);
2470 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2473 if swap_candidate {
2474 return self.register_swap_candidate(
2475 connection_id,
2476 sink,
2477 &frame,
2478 hello.manifest,
2479 negotiated_ver,
2480 control_ops,
2481 hello_ack,
2482 );
2483 }
2484 let registration = match self.registry.register_unpublished_with_control_ops(
2491 hello.manifest,
2492 negotiated_ver,
2493 connection_id,
2494 control_ops,
2495 ) {
2496 Ok(registration) => registration,
2497 Err(RegistryError::DuplicateModuleId { module_id }) => {
2498 return Ok(vec![control_error_frame(
2499 &frame,
2500 "duplicate_module_id",
2501 format!(
2502 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2503 ),
2504 )?])
2505 }
2506 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2507 return Ok(vec![control_error_frame(
2508 &frame,
2509 "invalid_module_id",
2510 err.to_string(),
2511 )?])
2512 }
2513 Err(err) => {
2514 return Ok(vec![control_error_frame(
2515 &frame,
2516 "registry_error",
2517 err.to_string(),
2518 )?])
2519 }
2520 };
2521 #[cfg(test)]
2522 self.hold_between_registry_and_forwarding();
2523
2524 let reply = if let Some(sink) = sink {
2525 let concurrency = manifest_concurrency(®istration.manifest);
2541 if let Err(err) = self.forwarding.register_module_connection_acked(
2542 connection_id,
2543 registration.manifest.module_id.clone(),
2544 negotiated_ver,
2545 concurrency,
2546 sink,
2547 hello_ack,
2548 ) {
2549 if matches!(
2554 self.deregister_connection(
2555 connection_id,
2556 RegistrationEndReason::RegistrationFailed,
2557 ),
2558 Ok(r) if !r.is_empty()
2559 ) {
2560 crate::supervise::notify_registration_release();
2561 }
2562 return Ok(vec![control_error_frame(
2563 &frame,
2564 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2565 "invalid_hello"
2566 } else {
2567 forwarding_error_code(&err)
2568 },
2569 err.to_string(),
2570 )?]);
2571 }
2572 Vec::new()
2573 } else {
2574 vec![hello_ack]
2577 };
2578
2579 if !self
2584 .registry
2585 .publish_registration(connection_id)
2586 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2587 {
2588 debug!(
2589 module_id = %registration.manifest.module_id,
2590 connection_id = connection_id.get(),
2591 "registration was removed before it could be published"
2592 );
2593 }
2594
2595 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2604 info!(
2605 module_id = %registration.manifest.module_id,
2606 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2607 );
2608 }
2609
2610 self.apply_registration_capabilities(®istration);
2611
2612 self.supervisor
2613 .record_activity_registration(®istration.manifest.module_id);
2614
2615 info!(
2616 module_id = %registration.manifest.module_id,
2617 module_version = %registration.manifest.module_version,
2618 negotiated_ver,
2619 routable_provider = manifest_provides_routable_role(®istration.manifest),
2620 connection_id = connection_id.get(),
2621 "module registered"
2622 );
2623
2624 Ok(reply)
2625 }
2626
2627 #[allow(clippy::too_many_arguments)]
2644 fn register_swap_candidate(
2645 &self,
2646 connection_id: ConnectionId,
2647 sink: Option<crate::FrameSink>,
2648 frame: &Frame,
2649 manifest: ModuleManifest,
2650 negotiated_ver: u8,
2651 control_ops: Vec<String>,
2652 hello_ack: Frame,
2653 ) -> Result<Vec<Frame>, RouterError> {
2654 let module_id = manifest.module_id.clone();
2655 let registration = match self.registry.register_unpublished_candidate_with_control_ops(
2656 manifest,
2657 negotiated_ver,
2658 connection_id,
2659 control_ops,
2660 ) {
2661 Ok(registration) => registration,
2662 Err(RegistryError::DuplicateModuleId { module_id }) => {
2663 return Ok(vec![control_error_frame(
2664 frame,
2665 "duplicate_module_id",
2666 format!(
2667 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2668 ),
2669 )?])
2670 }
2671 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2672 return Ok(vec![control_error_frame(
2673 frame,
2674 "invalid_module_id",
2675 err.to_string(),
2676 )?])
2677 }
2678 Err(err) => {
2679 return Ok(vec![control_error_frame(
2680 frame,
2681 "registry_error",
2682 err.to_string(),
2683 )?])
2684 }
2685 };
2686 #[cfg(test)]
2687 self.hold_between_registry_and_forwarding();
2688 let reply = if let Some(sink) = sink {
2689 let concurrency = manifest_concurrency(®istration.manifest);
2693 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2694 connection_id,
2695 module_id.clone(),
2696 negotiated_ver,
2697 concurrency,
2698 sink,
2699 hello_ack,
2700 ) {
2701 if matches!(
2702 self.deregister_connection(
2703 connection_id,
2704 RegistrationEndReason::RegistrationFailed,
2705 ),
2706 Ok(r) if !r.is_empty()
2707 ) {
2708 crate::supervise::notify_registration_release();
2709 }
2710 return Ok(vec![control_error_frame(
2711 frame,
2712 forwarding_error_code(&err),
2713 err.to_string(),
2714 )?]);
2715 }
2716 Vec::new()
2717 } else {
2718 vec![hello_ack]
2719 };
2720 if !self
2721 .registry
2722 .publish_registration(connection_id)
2723 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2724 {
2725 debug!(
2726 module_id = %module_id,
2727 connection_id = connection_id.get(),
2728 "swap candidate registration was removed before it could be published"
2729 );
2730 }
2731 self.supervisor.mark_swap_candidate_admitted(&module_id);
2732 info!(
2733 module_id = %module_id,
2734 module_version = %registration.manifest.module_version,
2735 negotiated_ver,
2736 ready = registration.ready,
2737 connection_id = connection_id.get(),
2738 "swap candidate registered; not routable until cutover"
2739 );
2740 Ok(reply)
2741 }
2742
2743 fn build_hello_ack(
2744 &self,
2745 frame: &Frame,
2746 negotiated_ver: u8,
2747 module_id: &str,
2748 ) -> Result<Frame, RouterError> {
2749 let ack = ModuleHelloAckBody {
2750 negotiated_ver,
2751 subc_ops: module_subc_ops(),
2752 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2753 storage: self
2754 .storage_config
2755 .as_ref()
2756 .map(|cfg| cfg.descriptor_for(module_id)),
2757 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2758 };
2759 let body = serde_json::to_vec(&ack).map_err(|err| {
2760 RouterError::backend(
2761 0,
2762 frame.header.corr,
2763 format!("failed to encode HELLO_ACK: {err}"),
2764 )
2765 })?;
2766
2767 Frame::build_with_version(
2768 negotiated_ver,
2769 FrameType::HelloAck,
2770 control_flags(),
2771 0,
2772 0,
2773 frame.header.corr,
2774 body,
2775 )
2776 .map_err(RouterError::FrameBuild)
2777 }
2778
2779 async fn handle_client_control_request(
2780 &self,
2781 ctx: &RouteCtx,
2782 frame: Frame,
2783 request: ClientControlRequest,
2784 route_open_timings: &mut Option<RouteOpenTimings>,
2785 ) -> Result<Vec<Frame>, RouterError> {
2786 if let Some(operation) = privileged_operation(&request) {
2787 let caller = self.resolve_caller(ctx.connection_id).await;
2788 let op = client_control_request_op(&request);
2789 let text = prompt_text(operation, &caller);
2790 match self.privileged_operator_prompt {
2791 PrivilegedOperatorPrompt::Log => {
2792 info!(target: "subc_daemon::privileged_prompt", operation = op,
2793 target = ?operation, caller = %caller.description(), summary = %text,
2794 "privileged request would prompt the operator");
2795 }
2796 PrivilegedOperatorPrompt::Enforce => {
2797 let key =
2798 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2799 let outcome = self
2800 .forwarding
2801 .operator_confirms()
2802 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2803 .await;
2804 if outcome.code != "confirmed" {
2805 return Ok(vec![outcome.frame(&frame)]);
2806 }
2807 }
2808 }
2809 }
2810 match request {
2811 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2812 ClientControlRequest::CatalogList { module_id } => {
2813 self.handle_catalog_list(frame, module_id)
2814 }
2815 ClientControlRequest::RouteOpen {
2816 target,
2817 identity,
2818 consumer_identity,
2819 consumer_capabilities,
2820 role_versions,
2821 admission_facts,
2822 scope,
2823 } => {
2824 self.handle_route_open(
2825 ctx,
2826 frame,
2827 RouteOpenRequest {
2828 target,
2829 identity,
2830 consumer_identity,
2831 consumer_capabilities,
2832 role_versions,
2833 admission_facts,
2834 scope,
2835 },
2836 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2837 )
2838 .await
2839 }
2840 ClientControlRequest::RoutePoll {
2841 route_channel,
2842 route_epoch,
2843 kind,
2844 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2845 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2846 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2847 self.handle_supervisor_spawn_snapshot(frame)
2848 }
2849 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2850 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2851 }
2852 ClientControlRequest::SupervisorRestart {
2853 module_id,
2854 drain_timeout_ms,
2855 } => {
2856 self.log_supervisor_request_received(
2857 ctx,
2858 frame.header.corr,
2859 ops::SUPERVISOR_RESTART,
2860 Some(&module_id),
2861 None,
2862 )?;
2863 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2864 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2865 .await
2866 }
2867 ClientControlRequest::SupervisorSwap {
2868 module_id,
2869 ready_timeout_ms,
2870 } => {
2871 self.log_supervisor_request_received(
2872 ctx,
2873 frame.header.corr,
2874 ops::SUPERVISOR_SWAP,
2875 Some(&module_id),
2876 None,
2877 )?;
2878 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2879 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2880 .await
2881 }
2882 ClientControlRequest::SupervisorReload { module_id } => {
2883 self.log_supervisor_request_received(
2884 ctx,
2885 frame.header.corr,
2886 ops::SUPERVISOR_RELOAD,
2887 Some(&module_id),
2888 None,
2889 )?;
2890 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2891 self.handle_supervisor_reload(frame, module_id, requester)
2892 .await
2893 }
2894 ClientControlRequest::SupervisorRescan { preview } => {
2895 if !preview {
2896 self.log_supervisor_request_received(
2897 ctx,
2898 frame.header.corr,
2899 ops::SUPERVISOR_RESCAN,
2900 None,
2901 None,
2902 )?;
2903 }
2904 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2905 self.handle_supervisor_rescan(
2906 frame,
2907 if preview {
2908 RescanMode::Preview
2909 } else {
2910 RescanMode::Execute
2911 },
2912 requester,
2913 )
2914 .await
2915 }
2916 ClientControlRequest::SupervisorRescanChecked {
2917 preview,
2918 candidate,
2919 expect,
2920 } => {
2921 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2922 .await
2923 }
2924 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2925 self.log_supervisor_request_received(
2926 ctx,
2927 frame.header.corr,
2928 ops::SUPERVISOR_RELEASE_RESERVED,
2929 Some(&module_id),
2930 None,
2931 )?;
2932 self.handle_supervisor_release_reserved(frame, module_id)
2933 .await
2934 }
2935 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2936 self.log_supervisor_request_received(
2937 ctx,
2938 frame.header.corr,
2939 ops::SUPERVISOR_SET_ENABLED,
2940 Some(&module_id),
2941 Some(enabled),
2942 )?;
2943 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2944 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2945 .await
2946 }
2947 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2948 self.handle_supervisor_health_probe(frame, module_id).await
2949 }
2950 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2951 ClientControlRequest::SupervisorRoutes { module_id } => {
2952 self.handle_supervisor_routes(frame, module_id)
2953 }
2954 ClientControlRequest::SupervisorProvenance { module_id } => {
2955 self.handle_supervisor_provenance(frame, module_id).await
2956 }
2957 ClientControlRequest::SupervisorStderrTail {
2958 module_id,
2959 max_lines,
2960 max_bytes,
2961 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2962 ClientControlRequest::SupervisorTerminals { module_id } => {
2963 self.handle_supervisor_terminals(frame, module_id).await
2964 }
2965 }
2966 }
2967
2968 fn log_supervisor_request_received(
2969 &self,
2970 ctx: &RouteCtx,
2971 corr: u64,
2972 op: &'static str,
2973 module_id: Option<&str>,
2974 enabled: Option<bool>,
2975 ) -> Result<(), RouterError> {
2976 let caller = self.supervisor_requester(ctx, corr)?;
2977 let caller = principal_label(&caller);
2978 info!(
2981 target: "control",
2982 op,
2983 module_id,
2984 enabled,
2985 connection_id = ctx.connection_id.get(),
2986 caller = %caller,
2987 "supervisor request received"
2988 );
2989 Ok(())
2990 }
2991
2992 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2993 Ok(self
2994 .registry
2995 .get_module_by_connection(ctx.connection_id)
2996 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2997 .map(|registration| Principal::Reserved {
2998 module_id: registration.manifest.module_id,
2999 })
3000 .unwrap_or(Principal::Direct))
3001 }
3002
3003 fn handle_module_control_request(
3004 &self,
3005 connection_id: ConnectionId,
3006 frame: Frame,
3007 request: ModuleControlRequestFromModule,
3008 ) -> Result<Vec<Frame>, RouterError> {
3009 match request {
3010 ModuleControlRequestFromModule::CatalogUpdate {
3011 provides,
3012 capabilities,
3013 ready,
3014 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3015 ModuleControlRequestFromModule::LiveRoots {} => {
3016 let registered = self
3017 .registry
3018 .get_module_by_connection(connection_id)
3019 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3020 let Some(registration) = registered else {
3021 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3022 };
3023 let response = self
3024 .forwarding
3025 .live_roots(®istration.manifest.module_id)
3026 .map_err(RouterError::Forwarding)?;
3027 Ok(vec![control_response_body_frame(
3028 &frame,
3029 &response,
3030 "ModuleControlResponseToModule::LiveRoots",
3031 )?])
3032 }
3033 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3034 self.handle_scope_sync(connection_id, frame, generation, scopes)
3035 }
3036 ModuleControlRequestFromModule::ScopeApply {
3037 generation,
3038 upsert,
3039 end,
3040 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3041 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3042 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3043 }
3044 }
3045 }
3046
3047 fn handle_scope_served_by(
3048 &self,
3049 connection_id: ConnectionId,
3050 frame: Frame,
3051 ) -> Result<Vec<Frame>, RouterError> {
3052 let registered = self
3053 .registry
3054 .get_module_by_connection(connection_id)
3055 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3056 let Some(registered) = registered else {
3057 return Ok(vec![control_error_frame(
3058 &frame,
3059 "not_registered",
3060 "scope.served_by requires an active module registration owned by this connection",
3061 )?]);
3062 };
3063 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3064 {
3065 Ok(request) => request,
3066 Err(err) => {
3067 return Ok(vec![control_error_frame(
3068 &frame,
3069 "invalid_control_body",
3070 format!("malformed scope.served_by body: {err}"),
3071 )?])
3072 }
3073 };
3074 let answer = self
3075 .scopes
3076 .read()
3077 .unwrap_or_else(|poisoned| poisoned.into_inner())
3078 .served_by(
3079 ®istered.manifest.module_id,
3080 &request.scope_ref,
3081 request.scope_epoch,
3082 &request.module_id,
3083 );
3084 let reply = subc_control::ScopeServedByReply::new(
3085 answer,
3086 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3087 );
3088 Ok(vec![control_response_body_frame(
3089 &frame,
3090 &reply,
3091 "ScopeServedByReply",
3092 )?])
3093 }
3094
3095 fn handle_operator_confirm(
3096 &self,
3097 ctx: &RouteCtx,
3098 frame: Frame,
3099 ) -> Result<Vec<Frame>, RouterError> {
3100 use crate::operator_confirm::{audit, Outcome};
3101 let registration = self
3102 .registry
3103 .get_module_by_connection(ctx.connection_id)
3104 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3105 let Some(registration) = registration else {
3106 let outcome = Outcome::refusal("not_registered");
3107 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3108 return Ok(vec![outcome.frame(&frame)]);
3109 };
3110 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3111 Ok(request) => request,
3112 Err(_) => {
3113 let outcome = Outcome::refusal("invalid_control_body");
3114 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3115 return Ok(vec![outcome.frame(&frame)]);
3116 }
3117 };
3118 let module_id = registration.manifest.module_id;
3119 let nonce = self
3123 .hello_launch_nonces
3124 .lock()
3125 .unwrap_or_else(|p| p.into_inner())
3126 .nonce(ctx.connection_id)
3127 .map(str::to_owned);
3128 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3129 self.supervisor
3130 .spawned_consumer_authorized(&module_id, nonce)
3131 });
3132 let confirms = self.forwarding.operator_confirms();
3133 self.forwarding
3134 .with_operator_route(
3135 ctx.connection_id,
3136 request.route_channel,
3137 request.route_epoch,
3138 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3139 )
3140 .map_err(RouterError::Forwarding)
3141 }
3142
3143 fn handle_scope_sync(
3147 &self,
3148 connection_id: ConnectionId,
3149 frame: Frame,
3150 generation: u64,
3151 scopes: Vec<ScopeRecord>,
3152 ) -> Result<Vec<Frame>, RouterError> {
3153 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3154 }
3155
3156 fn handle_scope_change(
3157 &self,
3158 connection_id: ConnectionId,
3159 frame: Frame,
3160 generation: u64,
3161 scopes: Vec<ScopeRecord>,
3162 end: Option<Vec<ScopeEnd>>,
3163 ) -> Result<Vec<Frame>, RouterError> {
3164 let is_apply = end.is_some();
3165 let Some(registration) = self
3166 .registry
3167 .get_module_by_connection(connection_id)
3168 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3169 else {
3170 return Ok(vec![control_error_frame(
3171 &frame,
3172 "not_registered",
3173 "scope.sync and scope.apply require an active module registration owned by this connection",
3174 )?]);
3175 };
3176 let owner = registration.manifest.module_id;
3177 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3178 let is_current_launch = |connection: ConnectionId| {
3179 self.hello_launch_nonces
3180 .lock()
3181 .unwrap_or_else(|poisoned| poisoned.into_inner())
3182 .presented(connection, current_nonce.as_deref())
3183 };
3184 let mut table = self
3189 .scopes
3190 .write()
3191 .unwrap_or_else(|poisoned| poisoned.into_inner());
3192 let outcome = match end {
3193 Some(end) => table.apply(
3194 &owner,
3195 connection_id,
3196 is_current_launch,
3197 generation,
3198 scopes,
3199 end,
3200 ),
3201 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3202 };
3203 let drained = match &outcome {
3204 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3205 Err(_) => Vec::new(),
3206 };
3207 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3208 drop(table);
3209 if has_deadlines {
3210 self.scope_deadline_added.notify_one();
3211 }
3212 match outcome {
3213 Ok(applied) => {
3214 let counts = ScopeOutcomeCounts::of(&applied.results);
3215 info!(
3216 owner = %owner,
3217 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3218 generation,
3219 records = applied.results.len(),
3220 created = counts.created,
3221 replaced = counts.replaced,
3222 updated = counts.updated,
3223 unchanged = counts.unchanged,
3224 refused = counts.refused,
3225 ended = applied.ended.len(),
3226 tag_changes = applied.tag_changes.len(),
3227 routes_closed = drained.len(),
3228 "scope change accepted"
3229 );
3230 for refused in applied
3236 .results
3237 .iter()
3238 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3239 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3240 {
3241 warn!(
3242 owner = %owner,
3243 generation,
3244 scope_ref = %refused.scope_ref,
3245 scope_epoch = refused.scope_epoch,
3246 code = refused.code.as_deref().unwrap_or(""),
3247 "scope record refused"
3248 );
3249 }
3250 self.close_scope_drained_routes(drained);
3251 let response = if is_apply {
3252 ModuleControlResponseToModule::ScopeApply {
3253 generation,
3254 results: applied.results,
3255 end_results: applied.end_results,
3256 ended: applied.ended,
3257 }
3258 } else {
3259 ModuleControlResponseToModule::ScopeSync {
3260 generation,
3261 results: applied.results,
3262 ended: applied.ended,
3263 }
3264 };
3265 Ok(vec![control_response_body_frame(
3266 &frame,
3267 &response,
3268 "scope change response",
3269 )?])
3270 }
3271 Err(refusal) => {
3272 info!(
3273 owner = %owner,
3274 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3275 generation,
3276 code = refusal.code,
3277 "scope change refused"
3278 );
3279 Ok(vec![control_error_frame(
3280 &frame,
3281 refusal.code,
3282 refusal.message,
3283 )?])
3284 }
3285 }
3286 }
3287
3288 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3294 if drained.is_empty() {
3295 return;
3296 }
3297 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3298 BTreeMap::new();
3299 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3300 for route in drained {
3301 warn!(
3302 module_id = %route.module_id,
3303 reason = ?route.reason,
3304 client_connection_id = route.client.connection_id.get(),
3305 route_channel = route.client.channel,
3306 "closing route because its scope changed"
3307 );
3308 pushes
3309 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3310 .or_insert_with(|| (route.reason, Vec::new()))
3311 .1
3312 .push(EndpointRoute {
3313 goodbye_target: route.client.clone(),
3314 principal: Principal::Unverified,
3315 bound_at: Instant::now(),
3316 draining: false,
3317 drain_reason: None,
3318 });
3319 goodbyes.push(route.module);
3320 goodbyes.push(route.client);
3321 }
3322 for ((module_id, _), (reason, routes)) in pushes {
3323 send_route_control_pushes(
3324 &self.forwarding,
3325 routes,
3326 ClientControlPush::RouteClosed {
3327 module_id,
3328 channels: Vec::new(),
3329 reason,
3330 drained: false,
3331 abandoned: 0,
3332 excluded_subscriptions: 0,
3333 terminal: Some(false),
3334 },
3335 );
3336 }
3337 self.emit_route_goodbyes(goodbyes);
3338 }
3339
3340 fn handle_scope_describe(
3343 &self,
3344 connection_id: ConnectionId,
3345 frame: Frame,
3346 owner: Principal,
3347 scope_ref: String,
3348 ) -> Result<Vec<Frame>, RouterError> {
3349 let registered = self
3350 .registry
3351 .get_module_by_connection(connection_id)
3352 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3353 let Some(registered) = registered else {
3354 return Ok(vec![control_error_frame(
3355 &frame,
3356 "not_registered",
3357 "scope.describe requires an active module registration owned by this connection",
3358 )?]);
3359 };
3360 let mut description = self
3361 .scopes
3362 .read()
3363 .unwrap_or_else(|poisoned| poisoned.into_inner())
3364 .describe(&owner, &scope_ref);
3365 if !registered
3368 .manifest
3369 .capabilities
3370 .as_ref()
3371 .is_some_and(|caps| {
3372 caps.provides
3373 .iter()
3374 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3375 })
3376 {
3377 if let Some(stamp) = description.stamp.as_mut() {
3378 stamp.attributes.on_behalf_of_agent = None;
3379 }
3380 }
3381 let owner_configured = match &owner {
3382 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3387 _ => false,
3388 };
3389 let response = ModuleControlResponseToModule::ScopeDescribe {
3390 status: description.status,
3391 scope_epoch: description.scope_epoch,
3392 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3393 owner_synced: description.owner_synced,
3394 owner_configured,
3395 scope: description.stamp,
3396 };
3397 Ok(vec![control_response_body_frame(
3398 &frame,
3399 &response,
3400 "ModuleControlResponseToModule::ScopeDescribe",
3401 )?])
3402 }
3403
3404 fn handle_catalog_update(
3405 &self,
3406 connection_id: ConnectionId,
3407 frame: Frame,
3408 provides: Vec<ProviderRole>,
3409 capabilities: Option<CapabilityDeclarations>,
3410 ready: Option<bool>,
3411 ) -> Result<Vec<Frame>, RouterError> {
3412 self.refresh_capability_requirements();
3413 let Some(registration) = self
3414 .registry
3415 .get_module_by_connection(connection_id)
3416 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3417 else {
3418 return Ok(vec![control_error_frame(
3419 &frame,
3420 "not_registered",
3421 "catalog.update requires an active module registration owned by this connection",
3422 )?]);
3423 };
3424
3425 if let Some(message) =
3426 catalog_update_frozen_field_message(®istration.manifest, &provides)
3427 {
3428 return Ok(vec![control_error_frame(
3429 &frame,
3430 "catalog_update_frozen_field",
3431 message,
3432 )?]);
3433 }
3434
3435 let mut candidate = registration.manifest.clone();
3436 candidate.provides = provides.clone();
3437 candidate.capabilities = capabilities
3438 .clone()
3439 .or_else(|| registration.manifest.capabilities.clone());
3440 if let Err(err) = candidate.validate_capability_grammar() {
3441 return Ok(vec![control_error_frame(
3442 &frame,
3443 "invalid_capability_grammar",
3444 err.to_string(),
3445 )?]);
3446 }
3447
3448 let mut conflicts = self
3451 .capability_evaluator
3452 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3453 if let Some(conflict) = conflicts.first() {
3454 let message = format!(
3455 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3456 conflict.capability, conflict.claimants[0], candidate.module_id
3457 );
3458 for conflict in &mut conflicts {
3459 conflict.source = DuplicateClaimSource::CatalogUpdate;
3460 }
3461 log_duplicate_claim_events(conflicts);
3462 return Ok(vec![control_error_frame(
3463 &frame,
3464 "reserved_capability",
3465 message,
3466 )?]);
3467 }
3468
3469 let updated = self
3470 .registry
3471 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3472 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3473 if updated.is_none() {
3474 return Ok(vec![control_error_frame(
3475 &frame,
3476 "not_registered",
3477 "catalog.update requires an active module registration owned by this connection",
3478 )?]);
3479 }
3480 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3481 log_duplicate_claim_events(
3482 self.capability_evaluator
3483 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3484 );
3485 }
3486 if capability_census_trigger(
3487 registration.manifest.capabilities.as_ref(),
3488 updated
3489 .as_ref()
3490 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3491 ) {
3492 self.enforce_capability_denies();
3493 }
3494 self.refresh_capability_requirements();
3495
3496 let response = ModuleControlResponseToModule::CatalogUpdate {};
3497 control_response_body_frame(
3498 &frame,
3499 &response,
3500 "ModuleControlResponseToModule::CatalogUpdate",
3501 )
3502 .map(|frame| vec![frame])
3503 }
3504
3505 pub(crate) fn with_activity_ledger_config(
3506 mut self,
3507 run_dir: &Path,
3508 incarnation: String,
3509 config: crate::activity::ActivityConfig,
3510 ) -> Self {
3511 self.activity = Some(crate::activity::ActivityLedger::open_config(
3512 run_dir.join("activity.jsonl"),
3513 incarnation,
3514 config,
3515 ));
3516 self.supervisor.set_activity(self.activity.clone());
3517 self
3518 }
3519
3520 pub(crate) async fn configure_activity(
3521 &self,
3522 config: crate::activity::ActivityConfig,
3523 ) -> Result<(), &'static str> {
3524 match &self.activity {
3525 Some(ledger) => ledger.configure(config).await,
3526 None => Ok(()),
3527 }
3528 }
3529
3530 fn cancel_activity_subscription(
3531 &self,
3532 ctx: &RouteCtx,
3533 frame: &Frame,
3534 ) -> Result<bool, RouterError> {
3535 let task = self
3536 .activity_streams
3537 .lock()
3538 .unwrap_or_else(|p| p.into_inner())
3539 .remove(&(ctx.connection_id, frame.header.corr));
3540 let Some(task) = task else {
3541 return Ok(false);
3542 };
3543 task.abort();
3544 let end = Frame::build_with_version(
3545 response_version(frame),
3546 FrameType::StreamEnd,
3547 control_flags(),
3548 0,
3549 0,
3550 frame.header.corr,
3551 Vec::new(),
3552 )
3553 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3554 let sink = ctx.egress.clone();
3555 tokio::spawn(async move {
3556 let _ = sink.send(end).await;
3557 });
3558 Ok(true)
3559 }
3560
3561 async fn handle_activity_read(
3562 &self,
3563 ctx: &RouteCtx,
3564 frame: Frame,
3565 ) -> Result<Vec<Frame>, RouterError> {
3566 use subc_control::{
3567 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3568 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3569 };
3570 #[derive(Deserialize)]
3571 struct Identity {
3572 #[serde(default)]
3573 consumer_identity: Option<ConsumerIdentity>,
3574 }
3575 let principal = match self
3579 .forwarding
3580 .module_id_for_connection(ctx.connection_id)
3581 .map_err(RouterError::Forwarding)?
3582 {
3583 Some(module_id) => Ok(Principal::Reserved { module_id }),
3584 None => {
3585 let identity: Identity = match serde_json::from_slice(&frame.body) {
3586 Ok(identity) => identity,
3587 Err(_) => {
3588 return Ok(vec![control_error_frame(
3589 &frame,
3590 ACTIVITY_MALFORMED,
3591 "malformed activity read",
3592 )?])
3593 }
3594 };
3595 self.route_open_principal(&frame, identity.consumer_identity)?
3596 }
3597 };
3598 let admitted = matches!(principal, Ok(Principal::Direct))
3599 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3600 if !admitted {
3601 return Ok(vec![control_error_frame(
3602 &frame,
3603 ACTIVITY_READ_FORBIDDEN,
3604 "activity reads require an operator or trusted mirror principal",
3605 )?]);
3606 }
3607 let Some(ledger) = &self.activity else {
3608 return Ok(vec![control_error_frame(
3609 &frame,
3610 ACTIVITY_UNAVAILABLE,
3611 "activity ledger unavailable",
3612 )?]);
3613 };
3614 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3615 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3616 {
3617 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3618 Ok(request) => ledger.query(request).await,
3619 Err(_) => Err(ACTIVITY_MALFORMED),
3620 };
3621 return match result {
3622 Ok(reply) => Ok(vec![control_response_body_frame(
3623 &frame,
3624 &reply,
3625 ops::ACTIVITY_QUERY,
3626 )?]),
3627 Err(code) => Ok(vec![control_error_frame(
3628 &frame,
3629 code,
3630 "activity query refused",
3631 )?]),
3632 };
3633 }
3634 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3635 Ok(request) => request,
3636 Err(_) => {
3637 return Ok(vec![control_error_frame(
3638 &frame,
3639 ACTIVITY_MALFORMED,
3640 "malformed activity subscription",
3641 )?])
3642 }
3643 };
3644 let mut subscription = match ledger.subscribe(request.since).await {
3645 Ok(subscription) => subscription,
3646 Err(code) => {
3647 return Ok(vec![control_error_frame(
3648 &frame,
3649 code,
3650 "activity subscription refused",
3651 )?])
3652 }
3653 };
3654 let key = (ctx.connection_id, frame.header.corr);
3655 let sink = ctx.egress.clone();
3656 let mut streams = self
3657 .activity_streams
3658 .lock()
3659 .unwrap_or_else(|p| p.into_inner());
3660 streams.retain(|_, task| !task.is_finished());
3661 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3662 return Ok(vec![control_error_frame(
3663 &frame,
3664 "duplicate_subscription",
3665 "activity subscription correlation id already in use",
3666 )?]);
3667 }
3668 let task = tokio::spawn(async move {
3669 loop {
3670 let entry = if let Some(entry) = subscription.replay.pop_front() {
3671 Some(entry)
3672 } else {
3673 subscription.receiver.recv().await
3674 };
3675 let Some(entry) = entry else {
3676 break;
3677 };
3678 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3679 return;
3680 };
3681 let Ok(next) = Frame::build_with_version(
3682 response_version(&frame),
3683 FrameType::StreamData,
3684 control_flags(),
3685 0,
3686 0,
3687 frame.header.corr,
3688 body,
3689 ) else {
3690 return;
3691 };
3692 if sink.send(next).await.is_err() {
3693 return;
3694 }
3695 }
3696 if let Some(code) = subscription.terminal_code() {
3699 if let Ok(end) =
3700 control_error_frame(&frame, code, "activity subscriber fell behind")
3701 {
3702 let _ = sink.send(end).await;
3703 }
3704 }
3705 });
3706 streams.insert(key, task.abort_handle());
3707 Ok(Vec::new())
3708 }
3709
3710 async fn handle_activity_record(
3711 &self,
3712 ctx: &RouteCtx,
3713 frame: Frame,
3714 ) -> Result<Vec<Frame>, RouterError> {
3715 use subc_control::{
3716 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3717 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3718 };
3719 let result = async {
3720 let module = self
3721 .forwarding
3722 .module_id_for_connection(ctx.connection_id)
3723 .ok()
3724 .flatten()
3725 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3726 let request: ActivityRecordRequest =
3727 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3728 crate::activity::validate(&module, &request)?;
3729 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3730 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3731 let at_ms = std::time::SystemTime::now()
3732 .duration_since(std::time::UNIX_EPOCH)
3733 .unwrap_or_default()
3734 .as_millis() as u64;
3735 ledger.record(module, request, actor, at_ms).await
3736 }
3737 .await;
3738 match result {
3739 Ok(entry) => Ok(vec![control_response_body_frame(
3740 &frame,
3741 &ActivityRecordResponse { seq: entry.seq },
3742 "activity.record",
3743 )?]),
3744 Err(code) => Ok(vec![control_error_frame(
3745 &frame,
3746 code,
3747 "activity recording refused",
3748 )?]),
3749 }
3750 }
3751
3752 fn activity_actor(
3753 &self,
3754 connection: ConnectionId,
3755 module: &str,
3756 on: Option<&subc_control::ActivityOn>,
3757 ) -> Result<subc_control::ActivityActor, &'static str> {
3758 use crate::forwarding::{DataRoute, DataRouteState};
3759 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3760 let principal = |id: &str| Principal::Reserved {
3761 module_id: id.to_owned(),
3762 };
3763 let mut actor = ActivityActor {
3764 principal: principal(module),
3765 scope_owner: None,
3766 agent_id: None,
3767 run_id: None,
3768 flow_id: None,
3769 project_root: None,
3770 };
3771 let scope = match on {
3772 None => return Ok(actor),
3773 Some(ActivityOn::Scope(scope)) => {
3774 let selector = subc_protocol::scope::ScopeSelector::new(
3775 principal(module),
3776 scope.scope_ref.clone(),
3777 Some(scope.epoch),
3778 );
3779 self.scopes
3780 .read()
3781 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3782 .attribute(&principal(module), module, &selector, true)
3783 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3784 }
3785 Some(ActivityOn::Route(route)) => {
3786 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3787 .forwarding
3788 .lookup_data_route(connection, route.channel, route.epoch)
3789 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3790 else {
3791 return Err(ACTIVITY_ACTOR_UNKNOWN);
3792 };
3793 if binding.module_id != module {
3794 return Err(ACTIVITY_ACTOR_UNKNOWN);
3795 }
3796 actor.principal = binding.principal.clone();
3797 actor.project_root = binding
3798 .project_root
3799 .as_ref()
3800 .map(|root| root.as_path().to_path_buf());
3801 let Some(bound) = &binding.scope else {
3802 return Ok(actor);
3803 };
3804 let owner = principal(&bound.owner);
3805 let selector = subc_protocol::scope::ScopeSelector::new(
3806 owner.clone(),
3807 bound.scope_ref.clone(),
3808 Some(bound.tag.scope_epoch),
3809 );
3810 let admitted = self
3811 .scopes
3812 .read()
3813 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3814 .attribute(&owner, module, &selector, true)
3815 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3816 if admitted.tag != bound.tag {
3817 return Err(ACTIVITY_ACTOR_UNKNOWN);
3818 }
3819 admitted
3820 }
3821 };
3822 actor.scope_owner = Some(scope.stamp.owner);
3823 actor.agent_id = scope.stamp.attributes.agent_id;
3824 actor.run_id = scope.stamp.attributes.run_id;
3825 actor.flow_id = scope.stamp.attributes.flow_id;
3826 Ok(actor)
3827 }
3828
3829 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3830 self.refresh_capability_requirements();
3831 let mut counters = self.counters.snapshot();
3837 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3838 obj.insert(
3839 "activity_unparseable_lines".into(),
3840 ledger.read_errors().into(),
3841 );
3842 obj.insert(
3843 "activity_lifecycle_append_failures".into(),
3844 ledger.lifecycle_failures().into(),
3845 );
3846 obj.insert(
3847 "activity_started_digest_skips".into(),
3848 ledger.lifecycle_digest_skips().into(),
3849 );
3850 }
3851 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3852 self.forwarding.client_route_concentration(),
3853 counters.as_object_mut(),
3854 ) {
3855 obj.insert(
3856 "client_connections_with_routes".into(),
3857 connections_with_routes.into(),
3858 );
3859 obj.insert("max_routes_on_one_connection".into(), max.into());
3860 }
3861 if let (Some(open_breakers), Some(obj)) = (
3867 self.route_bind_breakers.open_snapshot(),
3868 counters.as_object_mut(),
3869 ) {
3870 obj.insert("route_bind_breakers_open".into(), open_breakers);
3871 }
3872 let response = ClientControlResponse::ServerDescribe {
3873 protocol_ver: PROTOCOL_VERSION,
3874 subc_ops: subc_ops(),
3875 capabilities: self.describe_capabilities(),
3876 connected_clients: self.connected_clients.count(),
3877 counters: Some(counters),
3878 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3879 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3880 capability_requirements: self.capability_requirement_statuses(),
3881 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3882 config_path: self
3883 .rescan
3884 .as_ref()
3885 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3886 };
3887 Ok(vec![control_response_body_frame(
3888 &frame,
3889 &response,
3890 "ClientControlResponse::ServerDescribe",
3891 )?])
3892 }
3893
3894 fn handle_catalog_list(
3895 &self,
3896 frame: Frame,
3897 module_id: Option<String>,
3898 ) -> Result<Vec<Frame>, RouterError> {
3899 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3900 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3901 })?;
3902 let entries = modules
3903 .into_iter()
3904 .filter(|registration| {
3905 module_id
3906 .as_deref()
3907 .map(|wanted| registration.manifest.module_id == wanted)
3908 .unwrap_or(true)
3909 })
3910 .map(|registration| {
3911 let not_ready = self.not_ready_reason(®istration);
3912 let roles = registration.manifest.provides;
3913 CatalogEntry::new(
3914 registration.manifest.module_id,
3915 roles,
3916 registration.control_ops,
3917 )
3918 .with_ready(not_ready.is_none())
3919 .with_not_ready(not_ready)
3920 .with_module_version(Some(registration.manifest.module_version))
3921 .with_capabilities(registration.manifest.capabilities)
3922 .with_self_signals(registration.manifest.self_signals)
3923 .with_events(registration.manifest.events)
3924 .with_operation_delivery(registration.manifest.operation_delivery)
3925 })
3926 .collect();
3927 let response = ClientControlResponse::CatalogList {
3928 generation,
3929 modules: entries,
3930 subc_ops: subc_ops(),
3931 };
3932 Ok(vec![control_response_body_frame(
3933 &frame,
3934 &response,
3935 "ClientControlResponse::CatalogList",
3936 )?])
3937 }
3938
3939 fn route_open_principal(
3940 &self,
3941 frame: &Frame,
3942 consumer_identity: Option<ConsumerIdentity>,
3943 ) -> Result<Result<Principal, Frame>, RouterError> {
3944 let Some(consumer_identity) = consumer_identity else {
3945 return Ok(Ok(Principal::Direct));
3946 };
3947
3948 if self.supervisor.spawned_consumer_authorized(
3949 &consumer_identity.module_id,
3950 &consumer_identity.launch_nonce,
3951 ) {
3952 return Ok(Ok(Principal::Reserved {
3953 module_id: consumer_identity.module_id,
3954 }));
3955 }
3956
3957 Ok(Err(control_error_frame(
3958 frame,
3959 "bad_consumer_identity",
3960 format!(
3961 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3962 consumer_identity.module_id
3963 ),
3964 )?))
3965 }
3966
3967 fn route_open_refusal_frame(
3974 &self,
3975 ctx: &RouteCtx,
3976 frame: &Frame,
3977 module_id: &str,
3978 reason: &'static str,
3979 code: &'static str,
3980 message: impl Into<String>,
3981 ) -> Result<Frame, RouterError> {
3982 self.observe_route_open_refusal(ctx, module_id, reason, code);
3983 control_error_frame(frame, code, message.into())
3984 }
3985
3986 fn route_open_breaker_refusal_frame(
4004 &self,
4005 ctx: &RouteCtx,
4006 frame: &Frame,
4007 module_id: &str,
4008 consecutive_timeouts: u32,
4009 retry_in: Duration,
4010 probe_in_flight: bool,
4011 ) -> Result<Frame, RouterError> {
4012 self.counters
4013 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4014 debug!(
4015 target: "control",
4016 code = "module_timeout",
4017 module_id = ?module_id,
4018 connection_id = ctx.connection_id.get(),
4019 consecutive_timeouts,
4020 retry_in_ms = retry_in.as_millis() as u64,
4021 probe_in_flight,
4022 "route.open refused by open bind-relay breaker"
4023 );
4024 let detail = if probe_in_flight {
4031 "one test route is already being tried; retry once it settles".to_string()
4032 } else {
4033 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4034 };
4035 control_error_frame(
4036 frame,
4037 "module_timeout",
4038 format!(
4039 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4040 timed out in a row); {detail}; its established routes are unaffected"
4041 ),
4042 )
4043 }
4044
4045 fn observe_route_open_refusal(
4058 &self,
4059 ctx: &RouteCtx,
4060 module_id: &str,
4061 reason: &'static str,
4062 code: &'static str,
4063 ) {
4064 self.counters.increment_route_open_refused(code);
4065 info!(
4066 target: "control",
4067 code,
4068 reason,
4069 module_id = ?module_id,
4070 connection_id = ctx.connection_id.get(),
4071 "route.open refused"
4072 );
4073 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4074 self.route_outages.record_not_serving(module_id, reason);
4075 }
4076 }
4077
4078 fn observe_route_open_accept(
4123 &self,
4124 ctx: &RouteCtx,
4125 module_id: &str,
4126 principal: &str,
4127 scope: &str,
4128 ) {
4129 self.route_outages.record_accepted(module_id);
4130 self.counters.increment_route_open_accepted(principal);
4131 info!(
4132 target: "control",
4133 principal,
4134 module_id,
4135 connection_id = ctx.connection_id.get(),
4136 scope = %scope,
4137 "route.open accepted"
4138 );
4139 }
4140
4141 fn supervised_absent_route_open_refusal_frame(
4142 &self,
4143 ctx: &RouteCtx,
4144 frame: &Frame,
4145 module_id: &str,
4146 code: &'static str,
4147 status: &crate::supervise::ModuleStatus,
4148 ) -> Result<Frame, RouterError> {
4149 self.counters.increment_route_open_refused(code);
4150 info!(
4151 target: "control",
4152 code,
4153 reason = "supervised_not_registered",
4154 module_id = ?module_id,
4155 connection_id = ctx.connection_id.get(),
4156 state = %status.state,
4157 enabled = status.enabled,
4158 live = status.live,
4159 "route.open refused"
4160 );
4161 self.route_outages
4165 .record_not_serving(module_id, "supervised_not_registered");
4166 let message = if let Some(pid) = status.waiting_for_previous_pid {
4167 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4168 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4169 })
4170 } else {
4171 format!(
4172 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4173 status.state, status.enabled, status.live
4174 )
4175 };
4176 control_error_frame(frame, code, message)
4177 }
4178
4179 async fn handle_route_open(
4180 &self,
4181 ctx: &RouteCtx,
4182 frame: Frame,
4183 request: RouteOpenRequest,
4184 timings: &mut RouteOpenTimings,
4185 ) -> Result<Vec<Frame>, RouterError> {
4186 let RouteOpenRequest {
4187 target,
4188 mut identity,
4189 consumer_identity,
4190 consumer_capabilities,
4191 role_versions,
4192 admission_facts,
4193 scope,
4194 } = request;
4195 let target_module_id = target_module_id(&target).to_string();
4196 timings.module_id.clone_from(&target_module_id);
4197 if self
4198 .registry
4199 .get_module_by_connection(ctx.connection_id)
4200 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4201 .is_some()
4202 {
4203 return Ok(vec![control_error_frame(
4204 &frame,
4205 "invalid_request",
4206 "module connections cannot open client routes",
4207 )?]);
4208 }
4209 debug!(
4210 connection_id = ctx.connection_id.get(),
4211 corr = frame.header.corr,
4212 module_id = %target_module_id,
4213 "handling route.open"
4214 );
4215
4216 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4222 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4223 self.observe_route_open_refusal(
4224 ctx,
4225 &target_module_id,
4226 "invalid_role_versions",
4227 error_codes::INVALID_REQUEST,
4228 );
4229 return Ok(vec![control_error_body_frame(
4230 &frame,
4231 ErrorBody {
4232 code: error_codes::INVALID_REQUEST.to_string(),
4233 message: error.to_string(),
4234 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4235 },
4236 )?]);
4237 }
4238
4239 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4240 if status.waiting_for_previous_pid.is_some() {
4241 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4242 ctx,
4243 &frame,
4244 &target_module_id,
4245 "module_warming",
4246 &status,
4247 )?]);
4248 }
4249 }
4250
4251 let Some(registration) = self
4270 .registry
4271 .get_module(&target_module_id)
4272 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4273 else {
4274 if let Some((status, warming)) =
4275 self.supervisor_status(&target_module_id, frame.header.corr)?
4276 {
4277 if status.protocol == ModuleProtocol::None {
4284 return Ok(vec![self.route_open_refusal_frame(
4285 ctx,
4286 &frame,
4287 &target_module_id,
4288 "protocol_none",
4289 error_codes::MODULE_NO_PROTOCOL,
4290 format!(
4291 "module_id '{target_module_id}' is declared protocol: none; \
4292 it speaks no subc wire and serves no routes"
4293 ),
4294 )?]);
4295 }
4296 let code = if warming {
4297 "module_warming"
4298 } else {
4299 "target_unavailable"
4300 };
4301 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4302 ctx,
4303 &frame,
4304 &target_module_id,
4305 code,
4306 &status,
4307 )?]);
4308 }
4309 if let Some(removed_ago_ms) =
4310 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4311 {
4312 return Ok(vec![self.route_open_refusal_frame(
4313 ctx,
4314 &frame,
4315 &target_module_id,
4316 "removed",
4317 error_codes::MODULE_REMOVED,
4318 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4319 )?]);
4320 }
4321 return Ok(vec![self.route_open_refusal_frame(
4322 ctx,
4323 &frame,
4324 &target_module_id,
4325 "not_registered",
4326 error_codes::UNKNOWN_MODULE,
4327 format!("module_id '{target_module_id}' is not registered"),
4328 )?]);
4329 };
4330
4331 if !registration.ready {
4335 self.counters
4336 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4337 info!(
4338 target: "control",
4339 code = error_codes::MODULE_WARMING,
4340 module_id = ?target_module_id,
4341 connection_id = ctx.connection_id.get(),
4342 reason = "declared_not_ready",
4343 "route.open refused"
4344 );
4345 self.route_outages
4348 .record_not_serving(&target_module_id, "declared_not_ready");
4349 return Ok(vec![control_error_body_frame(
4350 &frame,
4351 ErrorBody {
4352 code: error_codes::MODULE_WARMING.to_string(),
4353 message: format!(
4354 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4355 ),
4356 detail: Some(serde_json::json!({
4357 "reason": "declared_not_ready"
4358 })),
4359 },
4360 )?]);
4361 }
4362
4363 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4382 self.counters
4383 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4384 info!(
4385 target: "control",
4386 code = error_codes::MODULE_WARMING,
4387 module_id = ?target_module_id,
4388 connection_id = ctx.connection_id.get(),
4389 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4390 capability = %capability,
4391 "route.open refused"
4392 );
4393 return Ok(vec![control_error_body_frame(
4394 &frame,
4395 ErrorBody {
4396 code: error_codes::MODULE_WARMING.to_string(),
4397 message: format!(
4398 "module_id '{target_module_id}' requires capability '{capability}', \
4399 which no registered module provides; retry"
4400 ),
4401 detail: Some(serde_json::json!({
4402 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4403 "capability": capability,
4404 })),
4405 },
4406 )?]);
4407 }
4408
4409 if !target_has_required_role(&target, ®istration.manifest.provides) {
4410 return Ok(vec![self.route_open_refusal_frame(
4411 ctx,
4412 &frame,
4413 &target_module_id,
4414 "role_not_provided",
4415 "target_unavailable",
4416 format!("module_id '{target_module_id}' does not provide the requested target"),
4417 )?]);
4418 }
4419
4420 if registration.state != ChannelState::Active {
4421 return Ok(vec![self.route_open_refusal_frame(
4422 ctx,
4423 &frame,
4424 &target_module_id,
4425 "registration_not_active",
4426 "target_unavailable",
4427 format!("module_id '{target_module_id}' is not active"),
4428 )?]);
4429 }
4430
4431 if self
4432 .forwarding
4433 .module_is_draining(&target_module_id)
4434 .map_err(RouterError::Forwarding)?
4435 {
4436 return Ok(vec![self.route_open_refusal_frame(
4437 ctx,
4438 &frame,
4439 &target_module_id,
4440 "reloading",
4441 "module_reloading",
4442 format!("module_id '{target_module_id}' is reloading"),
4443 )?]);
4444 }
4445
4446 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4447 process_liveness.process_live(&target_module_id) == Some(false)
4448 }) {
4449 if process_liveness.process_replacing(&target_module_id) {
4456 return Ok(vec![self.route_open_refusal_frame(
4457 ctx,
4458 &frame,
4459 &target_module_id,
4460 "reloading",
4461 "module_reloading",
4462 format!("module_id '{target_module_id}' is reloading"),
4463 )?]);
4464 }
4465 return Ok(vec![self.route_open_refusal_frame(
4466 ctx,
4467 &frame,
4468 &target_module_id,
4469 "supervisor_not_live",
4470 "target_unavailable",
4471 format!("module_id '{target_module_id}' is not live"),
4472 )?]);
4473 }
4474
4475 if !self
4476 .forwarding
4477 .has_live_module_connection(&target_module_id)
4478 .map_err(RouterError::Forwarding)?
4479 {
4480 return Ok(vec![self.route_open_refusal_frame(
4481 ctx,
4482 &frame,
4483 &target_module_id,
4484 "no_forwarding_connection",
4485 "target_unavailable",
4486 format!("module_id '{target_module_id}' has no live forwarding connection"),
4487 )?]);
4488 }
4489
4490 if let Some(error) =
4491 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4492 {
4493 self.observe_route_open_refusal(
4494 ctx,
4495 &target_module_id,
4496 "op_not_allowed",
4497 "op_not_allowed",
4498 );
4499 return Ok(vec![error]);
4500 }
4501
4502 let principal = match self.route_open_principal(&frame, consumer_identity)? {
4503 Ok(principal) => principal,
4504 Err(error) => {
4505 self.observe_route_open_refusal(
4506 ctx,
4507 &target_module_id,
4508 "bad_consumer_identity",
4509 "bad_consumer_identity",
4510 );
4511 return Ok(vec![error]);
4512 }
4513 };
4514 timings.principal = principal_label(&principal);
4515
4516 if let Principal::Reserved {
4520 module_id: opening_module_id,
4521 } = &principal
4522 {
4523 if let Some(opening_registration) = self
4524 .registry
4525 .get_module(opening_module_id)
4526 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4527 {
4528 if let Some(capability) =
4529 denied_capability(&opening_registration.manifest, ®istration.manifest)
4530 {
4531 warn!(
4532 opening_module_id,
4533 target_module_id,
4534 capability,
4535 "refusing route.open because an attested capability deny edge matches"
4536 );
4537 return Ok(vec![self.route_open_refusal_frame(
4538 ctx,
4539 &frame,
4540 &target_module_id,
4541 "capability_deny_edge",
4542 "capability_forbidden",
4543 format!(
4544 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4545 ),
4546 )?]);
4547 }
4548 }
4549 }
4550
4551 if admission_facts.is_some() {
4552 let carrier_matches = matches!(
4553 &principal,
4554 Principal::Reserved { module_id }
4555 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4556 );
4557 if !carrier_matches {
4558 return Ok(vec![self.route_open_refusal_frame(
4559 ctx,
4560 &frame,
4561 &target_module_id,
4562 "admission_facts_carrier_not_permitted",
4563 "admission_facts_not_permitted",
4564 "admission facts may only be carried by the configured reserved module",
4565 )?]);
4566 }
4567
4568 let target_allowed = self
4569 .admission_facts_targets
4570 .as_ref()
4571 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4572 if !target_allowed {
4573 return Ok(vec![self.route_open_refusal_frame(
4574 ctx,
4575 &frame,
4576 &target_module_id,
4577 "admission_facts_target_not_listed",
4578 "admission_facts_target_not_allowed",
4579 format!(
4580 "admission facts are not permitted for target module_id '{target_module_id}'"
4581 ),
4582 )?]);
4583 }
4584
4585 }
4589
4590 let (bound_scope, mut scope_stamp) = match &scope {
4596 None => (None, None),
4597 Some(selector) => {
4598 let owner_configured = match &selector.owner {
4599 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4605 _ => false,
4606 };
4607 let mut scopes = self
4608 .scopes
4609 .write()
4610 .unwrap_or_else(|poisoned| poisoned.into_inner());
4611 let admitted = scopes
4612 .admit(&principal, &target_module_id, selector, owner_configured)
4613 .and_then(|admission| {
4614 crate::scopes::check_target_flow_support(
4615 &admission.stamp,
4616 &target_module_id,
4617 registration.manifest.capabilities.as_ref(),
4618 )?;
4619 crate::scopes::check_target_agent_run_support(
4620 &admission.stamp,
4621 &target_module_id,
4622 registration.manifest.capabilities.as_ref(),
4623 )?;
4624 let bound = scopes.reserve_bind(
4625 &admission,
4626 &principal,
4627 ctx.connection_id,
4628 selector.hold_presence,
4629 );
4630 Ok((admission, bound))
4631 });
4632 match admitted {
4633 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4634 Err(refusal) => {
4635 return Ok(vec![self.route_open_refusal_frame(
4636 ctx,
4637 &frame,
4638 &target_module_id,
4639 refusal.code,
4640 refusal.code,
4641 refusal.message,
4642 )?]);
4643 }
4644 }
4645 }
4646 };
4647
4648 timings.enter(RouteOpenPhase::ProjectRoot);
4670 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4671 Ok(project_root) => project_root,
4672 Err(err) => {
4673 return Ok(vec![control_error_frame(
4674 &frame,
4675 "invalid_project_root",
4676 err.to_string(),
4677 )?])
4678 }
4679 };
4680 timings.enter(RouteOpenPhase::Admission);
4681 identity.project_root = project_root.as_path().to_path_buf();
4682
4683 let _concurrency_guard = match self
4694 .route_bind_concurrency
4695 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4696 {
4697 Ok(guard) => guard,
4698 Err(in_flight) => {
4699 return Ok(vec![self.route_open_target_capacity_refusal(
4700 ctx,
4701 &frame,
4702 &target_module_id,
4703 in_flight,
4704 )?]);
4705 }
4706 };
4707
4708 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4711 RouteBindAdmission::Admitted { guard, probe } => {
4712 if probe {
4713 info!(
4714 module_id = %target_module_id,
4715 connection_id = ctx.connection_id.get(),
4716 "route.bind breaker half-open: admitting one probe"
4717 );
4718 }
4719 guard
4720 }
4721 RouteBindAdmission::Refused {
4722 consecutive_timeouts,
4723 retry_in,
4724 probe_in_flight,
4725 } => {
4726 return Ok(vec![self.route_open_breaker_refusal_frame(
4727 ctx,
4728 &frame,
4729 &target_module_id,
4730 consecutive_timeouts,
4731 retry_in,
4732 probe_in_flight,
4733 )?]);
4734 }
4735 };
4736
4737 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4742 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4743 timings.enter(RouteOpenPhase::EgressReservation);
4744 let pending = match self
4745 .forwarding
4746 .begin_route_bind_relay_for(
4747 ctx.connection_id,
4748 ctx.egress.clone(),
4749 response_version(&frame),
4750 frame.header.corr,
4751 &target_module_id,
4752 principal.clone(),
4753 bound_scope,
4754 Some(project_root),
4755 relay_deadline,
4756 )
4757 .await
4758 {
4759 Ok(pending) => pending,
4760 Err(err) => {
4761 return Ok(vec![self.route_open_refusal_frame(
4762 ctx,
4763 &frame,
4764 &target_module_id,
4765 "relay_reservation_failed",
4766 forwarding_error_code(&err),
4767 err.to_string(),
4768 )?])
4769 }
4770 };
4771 timings.enter(RouteOpenPhase::Admission);
4772 let crate::forwarding::PendingRouteBindRelay {
4773 endpoint,
4774 module_sink,
4775 negotiated_ver,
4776 client_channel,
4777 client_epoch,
4778 module_channel,
4779 module_epoch,
4780 corr: relay_corr,
4781 receiver,
4782 } = pending;
4783 let mut reservation =
4784 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4785
4786 let relay_registration = self
4791 .registry
4792 .get_module_by_connection(endpoint.connection_id)
4793 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4794 let relay_capabilities = relay_registration
4795 .as_ref()
4796 .and_then(|registration| registration.manifest.capabilities.as_ref());
4797 if let Some(selector) = &scope {
4798 if let Err(refusal) = self
4801 .scopes
4802 .read()
4803 .unwrap_or_else(|poisoned| poisoned.into_inner())
4804 .check_bind_proof(&principal, selector)
4805 {
4806 reservation.release_and_disarm();
4807 return Ok(vec![self.route_open_refusal_frame(
4808 ctx,
4809 &frame,
4810 &target_module_id,
4811 refusal.code,
4812 refusal.code,
4813 refusal.message,
4814 )?]);
4815 }
4816 }
4817 if let Some(stamp) = scope_stamp
4818 .as_ref()
4819 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4820 {
4821 if let Err(refusal) = crate::scopes::check_target_flow_support(
4822 stamp,
4823 &target_module_id,
4824 relay_capabilities,
4825 )
4826 .and_then(|()| {
4827 crate::scopes::check_target_agent_run_support(
4828 stamp,
4829 &target_module_id,
4830 relay_capabilities,
4831 )
4832 }) {
4833 reservation.release_and_disarm();
4834 return Ok(vec![self.route_open_refusal_frame(
4835 ctx,
4836 &frame,
4837 &target_module_id,
4838 refusal.code,
4839 refusal.code,
4840 refusal.message,
4841 )?]);
4842 }
4843 }
4844
4845 if !relay_capabilities.is_some_and(|caps| {
4848 caps.provides
4849 .iter()
4850 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
4851 }) {
4852 if let Some(stamp) = scope_stamp.as_mut() {
4853 stamp.attributes.on_behalf_of_agent = None;
4854 }
4855 }
4856 let call_mode = if relay_capabilities
4857 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4858 {
4859 Some(derive_call_mode(scope_stamp.as_ref()))
4860 } else {
4861 if let Some(stamp) = scope_stamp.as_mut() {
4862 stamp.expires_at_ms = None;
4863 }
4864 None
4865 };
4866
4867 debug!(
4868 connection_id = ctx.connection_id.get(),
4869 client_channel,
4870 client_epoch,
4871 module_channel,
4872 module_epoch,
4873 "reserved route handle pair"
4874 );
4875 let route_principal_label = principal_label(&principal);
4878 let scope_label = scope_stamp
4879 .as_ref()
4880 .map_or_else(|| "none".to_string(), route_open_scope_label);
4881 let relay = ModuleControlRequest::RouteBind {
4882 route_channel: module_channel,
4883 epoch: module_epoch,
4884 target,
4885 identity,
4886 principal: Some(principal),
4887 consumer_capabilities,
4888 role_versions,
4889 admission_facts,
4890 scope: scope_stamp,
4891 call_mode,
4892 };
4893 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4894 RouterError::backend(
4895 0,
4896 frame.header.corr,
4897 format!("failed to encode route.bind request: {err}"),
4898 )
4899 })?;
4900 let relay_frame = Frame::build_with_version(
4901 negotiated_ver,
4902 FrameType::Request,
4903 control_flags(),
4904 0,
4905 0,
4906 relay_corr,
4907 relay_body,
4908 )
4909 .map_err(RouterError::FrameBuild)?;
4910
4911 timings.enter(RouteOpenPhase::BindSend);
4912 let sent = module_sink.send(relay_frame).await;
4913 timings.enter(RouteOpenPhase::Admission);
4914 if let Err(err) = sent {
4915 reservation.release_and_disarm();
4916 return Ok(vec![self.route_open_refusal_frame(
4917 ctx,
4918 &frame,
4919 &target_module_id,
4920 "relay_send_failed",
4921 "target_unavailable",
4922 err.to_string(),
4923 )?]);
4924 }
4925
4926 #[cfg(test)]
4927 if let Some(hold) = &self.route_bind_relay_sent_hold {
4928 hold.wait().await;
4932 hold.wait().await;
4933 }
4934
4935 if self
4940 .forwarding
4941 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4942 .map_err(RouterError::Forwarding)?
4943 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4944 {
4945 self.send_abandoned_route_bind_goodbye(
4946 &module_sink,
4947 negotiated_ver,
4948 module_channel,
4949 module_epoch,
4950 );
4951 }
4952
4953 timings.enter(RouteOpenPhase::BindAckWait);
4954 let outcome = timeout_at(relay_deadline, receiver).await;
4955 timings.enter(RouteOpenPhase::Admission);
4956 match outcome {
4957 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4958 reservation.disarm();
4959 if breaker.record_accepted() {
4960 info!(
4961 module_id = %target_module_id,
4962 "route.bind breaker closed: the probe was accepted"
4963 );
4964 }
4965 self.observe_route_open_accept(
4966 ctx,
4967 &target_module_id,
4968 &route_principal_label,
4969 &scope_label,
4970 );
4971 Ok(Vec::new())
4972 }
4973 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4974 reservation.release_and_disarm();
4975 breaker.record_inconclusive();
4979 let scope_code = match body.code.as_str() {
4983 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4984 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4985 _ => None,
4986 };
4987 if let Some(code) = scope_code {
4988 self.observe_route_open_refusal(
4989 ctx,
4990 &target_module_id,
4991 "scope_changed_before_commit",
4992 code,
4993 );
4994 return Ok(vec![control_error_body_frame(&frame, body)?]);
4995 }
4996 self.counters
4997 .increment_route_open_refused("module_rejected");
4998 info!(
4999 target: "control",
5000 code = "module_rejected",
5001 module_code = ?body.code,
5002 module_id = ?target_module_id,
5003 connection_id = ctx.connection_id.get(),
5004 "route.open refused"
5005 );
5006 Ok(vec![control_error_body_frame(&frame, body)?])
5007 }
5008 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5009 reservation.release_and_disarm();
5010 breaker.record_inconclusive();
5011 tracing::warn!(
5018 module_id = %target_module_id,
5019 "route.bind relay abandoned: {message}"
5020 );
5021 Ok(vec![self.route_open_refusal_frame(
5022 ctx,
5023 &frame,
5024 &target_module_id,
5025 "relay_abandoned",
5026 "target_unavailable",
5027 message,
5028 )?])
5029 }
5030 Ok(Err(_)) => {
5031 reservation.release_and_disarm();
5032 breaker.record_inconclusive();
5033 Ok(vec![self.route_open_refusal_frame(
5034 ctx,
5035 &frame,
5036 &target_module_id,
5037 "relay_waiter_canceled",
5038 "target_unavailable",
5039 "route.bind relay waiter was canceled before the module responded",
5040 )?])
5041 }
5042 Err(_) => {
5043 reservation.release_and_disarm();
5044 if let Some(opened) = breaker.record_timeout(
5048 self.route_bind_breaker_threshold,
5049 self.route_bind_breaker_cooldown,
5050 ) {
5051 warn!(
5052 module_id = %target_module_id,
5053 consecutive_timeouts = opened.consecutive_timeouts,
5054 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5055 reopened_after_probe = opened.reopened_after_probe,
5056 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5057 );
5058 }
5059 tracing::warn!(
5067 module_id = %target_module_id,
5068 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5069 "route.bind relay timed out: module did not ack within budget"
5070 );
5071 Ok(vec![self.route_open_refusal_frame(
5072 ctx,
5073 &frame,
5074 &target_module_id,
5075 "relay_timed_out",
5076 "module_timeout",
5077 format!(
5078 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5079 route_bind_relay_timeout
5080 ),
5081 )?])
5082 }
5083 }
5084 }
5085
5086 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5087 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5088 snapshot: self.supervisor.spawn_snapshot(),
5089 };
5090 Ok(vec![control_response_body_frame(
5091 &frame,
5092 &response,
5093 "ClientControlResponse::SupervisorSpawnSnapshot",
5094 )?])
5095 }
5096
5097 fn handle_supervisor_spawn_subscribe(
5098 &self,
5099 ctx: &RouteCtx,
5100 frame: Frame,
5101 since: Option<SpawnCursor>,
5102 ) -> Result<Vec<Frame>, RouterError> {
5103 match self.supervisor.subscribe_spawns(
5104 ctx.connection_id,
5105 frame.header.corr,
5106 response_version(&frame),
5107 since,
5108 ctx.egress.clone(),
5109 ) {
5110 Ok(()) => Ok(Vec::new()),
5111 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5112 Ok(vec![control_error_body_frame(
5113 &frame,
5114 ErrorBody {
5115 code: "spawn_cursor_incarnation_mismatch".to_string(),
5116 message: "spawn cursor belongs to a different daemon incarnation"
5117 .to_string(),
5118 detail: Some(serde_json::json!({
5119 "current_daemon_incarnation": current
5120 })),
5121 },
5122 )?])
5123 }
5124 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5125 &frame,
5126 ErrorBody {
5127 code: "spawn_cursor_too_old".to_string(),
5128 message: "spawn cursor predates the retained event ring".to_string(),
5129 detail: Some(serde_json::json!({
5130 "oldest_retained_cursor": oldest
5131 })),
5132 },
5133 )?]),
5134 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5135 0,
5136 frame.header.corr,
5137 format!("failed to open supervisor spawn subscription: {error}"),
5138 )),
5139 }
5140 }
5141
5142 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5143 let generation = self
5144 .registry
5145 .generation()
5146 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5147 let mut modules = Vec::new();
5148 for module in self.supervisor.list() {
5149 let status = module.status_for_control("list").map_err(|err| {
5150 RouterError::backend(
5151 0,
5152 frame.header.corr,
5153 format!("failed to read supervisor status: {err}"),
5154 )
5155 })?;
5156 let (configured, _) = module.configuration().map_err(|err| {
5157 RouterError::backend(
5158 0,
5159 frame.header.corr,
5160 format!("failed to read module configuration: {err}"),
5161 )
5162 })?;
5163 let image = module.running_image_agreement().await;
5165 let resources = Some(module.child_resource_usage());
5168 let mut verdict =
5169 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5170 verdict.spec = Some(launch_spec_agreement(
5171 &configured,
5172 status.process_alive,
5173 status.launch_spec_digest.as_ref(),
5174 ));
5175 let pending_reload = Some(verdict);
5176 modules.push(
5179 SupervisorEntry::new(
5180 status.module_id,
5181 status.state.to_string(),
5182 status.enabled,
5183 status.live,
5184 status.health.status,
5185 )
5186 .with_launch_nonce_env(Some(!cfg!(unix)))
5187 .with_launch_nonce_source(status.launch_nonce_source)
5188 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5189 .with_protocol(status.protocol)
5190 .with_pending_reload(pending_reload)
5191 .with_last_probe_ms(status.health.last_probe_ms)
5192 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5193 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5194 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5195 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5196 .with_restart_count(Some(status.restart_count))
5197 .with_max_restarts(Some(status.max_restarts))
5198 .with_lifetime_restarts(Some(status.lifetime_restarts))
5199 .with_spawn_generation(Some(status.spawn_generation))
5200 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5201 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5202 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5203 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5204 .with_resources(resources),
5205 );
5206 }
5207 let response = ClientControlResponse::SupervisorList {
5208 generation,
5209 modules,
5210 };
5211 Ok(vec![control_response_body_frame(
5212 &frame,
5213 &response,
5214 "ClientControlResponse::SupervisorList",
5215 )?])
5216 }
5217
5218 fn handle_supervisor_stderr_tail(
5219 &self,
5220 frame: Frame,
5221 module_id: String,
5222 max_lines: Option<u32>,
5223 max_bytes: Option<u32>,
5224 ) -> Result<Vec<Frame>, RouterError> {
5225 let Some(module) = self.supervisor.get(&module_id) else {
5226 return Ok(vec![control_error_frame(
5227 &frame,
5228 "unknown_module",
5229 format!("module_id '{module_id}' is not supervised"),
5230 )?]);
5231 };
5232
5233 let snapshot = module.stderr_tail(
5234 max_lines.map(|value| value as usize),
5235 max_bytes.map(|value| value as usize),
5236 );
5237
5238 let response = ClientControlResponse::SupervisorStderrTail {
5239 module_id,
5240 tail: StderrTail {
5241 capture: match snapshot.capture {
5242 CaptureState::Captured => StderrCaptureState::Captured,
5243 CaptureState::Incomplete { reason } => {
5244 StderrCaptureState::Incomplete { reason }
5245 }
5246 CaptureState::NotCaptured { reason } => {
5247 StderrCaptureState::NotCaptured { reason }
5248 }
5249 },
5250 entries: snapshot
5251 .entries
5252 .into_iter()
5253 .map(|entry| match entry {
5254 TailEntry::Line {
5255 text,
5256 truncated,
5257 at_ms,
5258 } => StderrTailEntry::Line {
5259 text,
5260 truncated,
5261 at_ms,
5262 },
5263 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5264 })
5265 .collect(),
5266 dropped_lines: snapshot.dropped_lines,
5267 },
5268 };
5269 Ok(vec![control_response_body_frame(
5270 &frame,
5271 &response,
5272 "ClientControlResponse::SupervisorStderrTail",
5273 )?])
5274 }
5275
5276 async fn handle_supervisor_terminals(
5277 &self,
5278 frame: Frame,
5279 module_id: String,
5280 ) -> Result<Vec<Frame>, RouterError> {
5281 let Some(module) = self.supervisor.get(&module_id) else {
5282 return Ok(vec![control_error_frame(
5283 &frame,
5284 "unknown_module",
5285 format!("module_id '{module_id}' is not supervised"),
5286 )?]);
5287 };
5288
5289 let terminals = module
5292 .read_durable_terminal_history()
5293 .await
5294 .map_err(|error| {
5295 RouterError::backend(
5296 0,
5297 frame.header.corr,
5298 format!("failed to read terminal history: {error}"),
5299 )
5300 })?;
5301 let response = ClientControlResponse::SupervisorTerminals {
5302 module_id,
5303 terminals,
5304 };
5305 Ok(vec![control_response_body_frame(
5306 &frame,
5307 &response,
5308 "ClientControlResponse::SupervisorTerminals",
5309 )?])
5310 }
5311
5312 fn handle_supervisor_routes(
5313 &self,
5314 frame: Frame,
5315 module_id: Option<String>,
5316 ) -> Result<Vec<Frame>, RouterError> {
5317 let modules = self
5318 .forwarding
5319 .route_census(module_id.as_deref())
5320 .map_err(RouterError::Forwarding)?
5321 .into_iter()
5322 .map(|(module_id, routes)| SupervisorRouteModule {
5323 module_id,
5324 routes: routes
5325 .into_iter()
5326 .map(|route| SupervisorRoute {
5327 consumer: match route.principal {
5328 Principal::Reserved { module_id } => {
5329 SupervisorRouteConsumer::Reserved { module_id }
5330 }
5331 Principal::Direct | Principal::Unverified => {
5332 SupervisorRouteConsumer::Direct {
5333 connection_id: route.goodbye_target.connection_id.get(),
5334 }
5335 }
5336 },
5337 age_ms: Instant::now()
5338 .saturating_duration_since(route.bound_at)
5339 .as_millis()
5340 .try_into()
5341 .unwrap_or(u64::MAX),
5342 draining: route.draining,
5343 drain_reason: route.drain_reason,
5344 })
5345 .collect(),
5346 })
5347 .collect();
5348 let response = ClientControlResponse::SupervisorRoutes { modules };
5349 Ok(vec![control_response_body_frame(
5350 &frame,
5351 &response,
5352 "ClientControlResponse::SupervisorRoutes",
5353 )?])
5354 }
5355
5356 async fn handle_supervisor_provenance(
5357 &self,
5358 frame: Frame,
5359 module_id: Option<String>,
5360 ) -> Result<Vec<Frame>, RouterError> {
5361 let mut selected = if let Some(module_id) = module_id {
5362 let Some(module) = self.supervisor.get(&module_id) else {
5363 return Ok(vec![control_error_frame(
5364 &frame,
5365 "unknown_module",
5366 format!("module_id '{module_id}' is not supervised"),
5367 )?]);
5368 };
5369 vec![module]
5370 } else {
5371 self.supervisor.list()
5372 };
5373
5374 let mut modules = Vec::with_capacity(selected.len());
5375 for module in selected.drain(..) {
5376 let (status, observed_image) = module
5377 .status_and_running_image_agreement()
5378 .await
5379 .map_err(|err| {
5380 RouterError::backend(
5381 0,
5382 frame.header.corr,
5383 format!("failed to read supervisor status: {err}"),
5384 )
5385 })?;
5386 let module_declared = self
5387 .registry
5388 .get_module(&status.module_id)
5389 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5390 .and_then(|registration| registration.manifest.provenance)
5391 .map(|build| ModuleDeclaredProvenance::Reported { build })
5392 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5393 #[cfg(test)]
5394 let running_image = match &self.provenance_probe_override {
5395 Some(result) => result.clone(),
5396 None => observed_image,
5397 };
5398 #[cfg(not(test))]
5399 let running_image = observed_image;
5400 modules.push(SupervisorModuleProvenance {
5401 module_id: status.module_id,
5402 module_declared,
5403 daemon_observed: SupervisorObservedProcess {
5404 launch_nonce_source: status.launch_nonce_source,
5405 pid: status.pid,
5406 spawned_at_ms: status.spawned_at_ms,
5407 spawned_from: status.spawned_from,
5408 running_image,
5409 },
5410 });
5411 }
5412 let daemon = SupervisorDaemonProvenance {
5413 daemon_build: self.daemon_provenance.build.clone(),
5414 daemon_observed: DaemonObservedProcess {
5415 pid: self.daemon_provenance.pid,
5416 started_at_ms: self
5417 .daemon_provenance
5418 .start_clock
5419 .map(|clock| clock.started_at_ms())
5420 .or(self.daemon_provenance.started_at_ms),
5421 running_image: self
5422 .daemon_provenance
5423 .probe
5424 .observe(
5425 self.daemon_provenance.pid,
5426 self.daemon_provenance.executable_path.as_deref(),
5427 self.daemon_provenance.executable_identity.clone(),
5428 self.daemon_provenance.process_start_time,
5429 )
5430 .await,
5431 },
5432 };
5433 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5434 Ok(vec![control_response_body_frame(
5435 &frame,
5436 &response,
5437 "ClientControlResponse::SupervisorProvenance",
5438 )?])
5439 }
5440
5441 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5442 self.refresh_capability_requirements();
5443 let generation = self
5444 .registry
5445 .generation()
5446 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5447 let modules = self
5448 .supervisor
5449 .list()
5450 .into_iter()
5451 .map(|module| {
5452 let status = module.status_for_control("health").map_err(|err| {
5453 RouterError::backend(
5454 0,
5455 frame.header.corr,
5456 format!("failed to read supervisor health: {err}"),
5457 )
5458 })?;
5459 let module_id = status.module_id;
5460 let capability_detail = self
5461 .capability_evaluator
5462 .required_problem_detail(&module_id);
5463 Ok(SupervisorHealthEntry {
5464 module_id,
5465 status: status.health.status,
5466 detail: append_capability_problem_detail(
5467 status.health.detail,
5468 capability_detail,
5469 ),
5470 metrics: status.health.metrics,
5471 consecutive_failures: status.health.consecutive_failures,
5472 late_answer_count: status.health.late_answer_count,
5473 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5474 last_action: status.health.last_action,
5475 last_action_ms: status.health.last_action_ms,
5476 last_probe_ms: status.health.last_probe_ms,
5477 })
5478 })
5479 .collect::<Result<Vec<_>, RouterError>>()?;
5480 let response = ClientControlResponse::SupervisorHealth {
5481 generation,
5482 modules,
5483 };
5484 Ok(vec![control_response_body_frame(
5485 &frame,
5486 &response,
5487 "ClientControlResponse::SupervisorHealth",
5488 )?])
5489 }
5490
5491 async fn handle_supervisor_restart(
5492 &self,
5493 frame: Frame,
5494 module_id: String,
5495 drain_timeout_ms: Option<u64>,
5496 requester: Principal,
5497 ) -> Result<Vec<Frame>, RouterError> {
5498 let operation_lock = self.supervisor.operation_lock();
5499 let _operation_guard = operation_lock.lock().await;
5500 let Some(module) = self.supervisor.get(&module_id) else {
5501 return Ok(vec![control_error_frame(
5502 &frame,
5503 "unknown_module",
5504 format!("module_id '{module_id}' is not supervised"),
5505 )?]);
5506 };
5507
5508 self.route_outages.mark_operator_action(&module_id);
5509 if let Err(err) = module
5510 .restart_requested(drain_timeout_ms, Some(requester))
5511 .await
5512 {
5513 self.route_outages
5514 .operator_action_ended_unrefused(&module_id);
5515 let (code, message) = match err {
5516 crate::supervise::SuperviseError::Disabled { .. } => {
5517 ("module_disabled", err.to_string())
5518 }
5519 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5520 ("swap_in_progress", err.to_string())
5521 }
5522 _ => (
5523 "target_unavailable",
5524 format!("failed to restart module_id '{module_id}': {err}"),
5525 ),
5526 };
5527 return Ok(vec![control_error_frame(&frame, code, message)?]);
5528 }
5529
5530 let response = ClientControlResponse::SupervisorAck {
5531 module_id,
5532 applied: true,
5533 };
5534 Ok(vec![control_response_body_frame(
5535 &frame,
5536 &response,
5537 "ClientControlResponse::SupervisorAck",
5538 )?])
5539 }
5540
5541 async fn handle_supervisor_swap(
5545 &self,
5546 frame: Frame,
5547 module_id: String,
5548 ready_timeout_ms: Option<u64>,
5549 requester: Principal,
5550 ) -> Result<Vec<Frame>, RouterError> {
5551 let module = {
5558 let operation_lock = self.supervisor.operation_lock();
5559 let _operation_guard = operation_lock.lock().await;
5560 self.supervisor.get(&module_id)
5561 };
5562 let Some(module) = module else {
5563 return Ok(vec![control_error_frame(
5564 &frame,
5565 "unknown_module",
5566 format!("module_id '{module_id}' is not supervised"),
5567 )?]);
5568 };
5569
5570 self.route_outages.mark_operator_action(&module_id);
5571 if let Err(err) = module
5572 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5573 .await
5574 {
5575 self.route_outages
5576 .operator_action_ended_unrefused(&module_id);
5577 use crate::supervise::SuperviseError;
5578 let message = err.to_string();
5579 let error = match err {
5580 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5581 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5582 code: "swap_refused".to_string(),
5583 message,
5584 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5585 },
5586 SuperviseError::SwapFailed {
5587 arm,
5588 candidate_exit,
5589 ..
5590 } => ErrorBody {
5591 code: "swap_failed".to_string(),
5592 message,
5593 detail: Some(serde_json::json!({
5594 "arm": arm.as_str(),
5595 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5596 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5597 })),
5598 },
5599 _ => ErrorBody::new(
5600 "target_unavailable",
5601 format!("failed to swap module_id '{module_id}': {message}"),
5602 ),
5603 };
5604 return Ok(vec![control_error_body_frame(&frame, error)?]);
5605 }
5606 self.route_outages
5610 .operator_action_ended_unrefused(&module_id);
5611
5612 let response = ClientControlResponse::SupervisorAck {
5613 module_id,
5614 applied: true,
5615 };
5616 Ok(vec![control_response_body_frame(
5617 &frame,
5618 &response,
5619 "ClientControlResponse::SupervisorAck",
5620 )?])
5621 }
5622
5623 async fn handle_supervisor_reload(
5624 &self,
5625 frame: Frame,
5626 module_id: String,
5627 requester: Principal,
5628 ) -> Result<Vec<Frame>, RouterError> {
5629 let operation_lock = self.supervisor.operation_lock();
5630 let _operation_guard = operation_lock.lock().await;
5631 let Some(module) = self.supervisor.get(&module_id) else {
5632 return Ok(vec![control_error_frame(
5633 &frame,
5634 "unknown_module",
5635 format!("module_id '{module_id}' is not supervised"),
5636 )?]);
5637 };
5638
5639 self.route_outages.mark_operator_action(&module_id);
5640 if let Err(err) = module.reload_requested(Some(requester)).await {
5641 self.route_outages
5642 .operator_action_ended_unrefused(&module_id);
5643 let (code, message) = match err {
5644 crate::supervise::SuperviseError::Disabled { .. } => {
5645 ("module_disabled", err.to_string())
5646 }
5647 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5648 ("swap_in_progress", err.to_string())
5649 }
5650 _ => (
5651 "reload_failed",
5652 format!("failed to reload module_id '{module_id}': {err}"),
5653 ),
5654 };
5655 return Ok(vec![control_error_frame(&frame, code, message)?]);
5656 }
5657
5658 let response = ClientControlResponse::SupervisorAck {
5659 module_id,
5660 applied: true,
5661 };
5662 Ok(vec![control_response_body_frame(
5663 &frame,
5664 &response,
5665 "ClientControlResponse::SupervisorAck",
5666 )?])
5667 }
5668
5669 async fn handle_supervisor_rescan(
5673 &self,
5674 frame: Frame,
5675 mode: RescanMode,
5676 requester: Principal,
5677 ) -> Result<Vec<Frame>, RouterError> {
5678 let preview = mode.previews();
5679 let Some(context) = self.rescan.clone() else {
5680 return Ok(vec![control_error_frame(
5681 &frame,
5682 "rescan_unavailable",
5683 "the daemon was not started with a reloadable config path".to_string(),
5684 )?]);
5685 };
5686
5687 let operation_lock = self.supervisor.operation_lock();
5688 let _operation_guard = operation_lock.lock().await;
5689 let loaded = match self.load_rescan_config(&mode, &context) {
5693 Ok(config) => config,
5694 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5695 };
5696 let Some((config, activity_config)) = loaded else {
5704 return Ok(vec![control_error_frame(
5705 &frame,
5706 "invalid_daemon_config",
5707 format!(
5708 "daemon config not found at {}; refusing to rescan (an absent config would \
5709 retire every supervised module)",
5710 context.config_path.display()
5711 ),
5712 )?]);
5713 };
5714 let (
5715 configured_port,
5716 storage_config,
5717 admission_facts_carrier_module_id,
5718 admission_facts_targets,
5719 scope_authority_owners,
5720 privileged_operator_prompt,
5721 modules,
5722 reserved_capabilities,
5723 ) = (
5724 config.port,
5725 config.storage,
5726 config.admission_facts_carrier_module_id,
5727 config.admission_facts_targets,
5728 config.scope_authority_owners,
5729 config.privileged_operator_prompt,
5730 config.modules,
5731 config.reserved_capabilities,
5732 );
5733
5734 let mut restart_required = Vec::new();
5743 for section in RestartRequiredSection::ALL {
5744 let changed = match section {
5745 RestartRequiredSection::Port => configured_port != context.configured_port,
5746 RestartRequiredSection::Storage => storage_config != context.storage_config,
5747 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5748 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5749 }
5750 RestartRequiredSection::AdmissionFactsTargets => {
5751 admission_facts_targets != context.admission_facts_targets
5752 }
5753 RestartRequiredSection::ScopeAuthorityOwners => {
5754 scope_authority_owners != context.scope_authority_owners
5755 }
5756 RestartRequiredSection::PrivilegedOperatorPrompt => {
5757 privileged_operator_prompt != context.privileged_operator_prompt
5758 }
5759 };
5760 if changed {
5761 restart_required.push(section.label().to_string());
5762 }
5763 }
5764 if !restart_required.is_empty() {
5765 warn!(
5766 config_path = %context.config_path.display(),
5767 sections = %restart_required.join(", "),
5768 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5769 );
5770 }
5771
5772 for configured in &modules {
5773 if let Err(err) = validate_spec(&configured.module_spec()) {
5774 return Ok(vec![control_error_frame(
5775 &frame,
5776 "invalid_daemon_config",
5777 format!("supervisor rescan rejected daemon config: {err}"),
5778 )?]);
5779 }
5780 }
5781
5782 let configured_capabilities = modules
5783 .iter()
5784 .map(|module| (module.module_id.clone(), module.enabled))
5785 .collect::<Vec<_>>();
5786 let preview_capability_warnings = if preview {
5787 let (_, registrations) = self.runtime_capability_snapshot()?;
5788 let current_modules = self
5789 .supervisor
5790 .list()
5791 .into_iter()
5792 .map(|module| module.module_id().to_string())
5793 .collect::<BTreeSet<_>>();
5794 let resulting_modules = configured_capabilities.clone();
5795 let removed = current_modules
5796 .into_iter()
5797 .filter(|module_id| {
5798 !resulting_modules
5799 .iter()
5800 .any(|(configured_id, _)| configured_id == module_id)
5801 })
5802 .collect::<Vec<_>>();
5803 let disabled = match &mode {
5806 RescanMode::Candidate(_) => resulting_modules
5807 .iter()
5808 .filter(|(_, enabled)| !enabled)
5809 .map(|(module_id, _)| module_id.clone())
5810 .collect::<Vec<_>>(),
5811 _ => Vec::new(),
5812 };
5813 self.capability_evaluator.preview_warnings(
5814 resulting_modules,
5815 &removed,
5816 &disabled,
5817 ®istrations,
5818 )
5819 } else {
5820 Vec::new()
5821 };
5822 let reconciled = match &mode {
5823 RescanMode::Candidate(candidate) => self
5824 .preview_candidate(&context, modules, candidate)
5825 .map(|result| (result, Vec::new())),
5826 _ => {
5827 self.reconcile_supervised_modules(
5828 &context.supervisor,
5829 modules,
5830 preview,
5831 requester.clone(),
5832 )
5833 .await
5834 }
5835 };
5836 let (result, changed) = match reconciled {
5837 Ok(result) => result,
5838 Err(message) => {
5839 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5840 }
5841 };
5842 if !preview {
5843 if let Some(ledger) = &self.activity {
5844 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
5845 ledger.config_changed(&result.added, &result.removed, &changed, requester);
5846 }
5847 if let Err(code) = self.configure_activity(activity_config).await {
5848 warn!(
5849 code,
5850 "activity retention could not be applied during rescan"
5851 );
5852 }
5853 self.capability_evaluator
5854 .configure(configured_capabilities, reserved_capabilities);
5855 self.capability_evaluator.wake_deadline_loop();
5856 self.refresh_capability_requirements();
5857 }
5858 let mut result = result;
5859 result.restart_required = restart_required;
5860 result.capability_warnings = preview_capability_warnings;
5861 let response = ClientControlResponse::SupervisorRescan { result };
5862 Ok(vec![control_response_body_frame(
5863 &frame,
5864 &response,
5865 "ClientControlResponse::SupervisorRescan",
5866 )?])
5867 }
5868
5869 async fn handle_supervisor_release_reserved(
5870 &self,
5871 frame: Frame,
5872 module_id: String,
5873 ) -> Result<Vec<Frame>, RouterError> {
5874 let Some(context) = self.rescan.clone() else {
5875 return Ok(vec![control_error_frame(
5876 &frame,
5877 "release_unavailable",
5878 "reserved-id release requires a daemon started with a reloadable config path",
5879 )?]);
5880 };
5881 let operation_lock = self.supervisor.operation_lock();
5882 let _operation_guard = operation_lock.lock().await;
5883 let loaded = match crate::daemon_config::load(&context.config_path) {
5884 Ok(Some(config)) => config,
5885 Ok(None) => {
5886 return Ok(vec![control_error_frame(
5887 &frame,
5888 "invalid_daemon_config",
5889 format!(
5890 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5891 context.config_path.display()
5892 ),
5893 )?])
5894 }
5895 Err(err) => {
5896 return Ok(vec![control_error_frame(
5897 &frame,
5898 "invalid_daemon_config",
5899 format!("unable to verify reserved-id release against daemon config: {err}"),
5900 )?])
5901 }
5902 };
5903 if loaded
5904 .modules
5905 .iter()
5906 .any(|configured| configured.module_id == module_id)
5907 {
5908 return Ok(vec![control_error_frame(
5909 &frame,
5910 "reserved_module_configured",
5911 format!(
5912 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5913 ),
5914 )?]);
5915 }
5916 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5917 return Ok(vec![control_error_frame(
5918 &frame,
5919 "reserved_gate_not_retained",
5920 format!(
5921 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5922 ),
5923 )?]);
5924 }
5925
5926 let response = ClientControlResponse::SupervisorAck {
5927 module_id,
5928 applied: true,
5929 };
5930 Ok(vec![control_response_body_frame(
5931 &frame,
5932 &response,
5933 "ClientControlResponse::SupervisorAck",
5934 )?])
5935 }
5936
5937 fn plan_rescan(
5941 &self,
5942 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5943 ) -> Result<RescanPlan, String> {
5944 let mut current = BTreeMap::new();
5945 for module in self.supervisor.list() {
5946 let (spec, health) = module.configuration().map_err(|err| {
5947 format!(
5948 "failed to read configuration for module_id '{}': {err}",
5949 module.module_id()
5950 )
5951 })?;
5952 let enabled = module
5953 .status()
5954 .map_err(|err| {
5955 format!(
5956 "failed to read status for module_id '{}': {err}",
5957 module.module_id()
5958 )
5959 })?
5960 .enabled;
5961 current.insert(
5962 module.module_id().to_string(),
5963 (module, spec, health, enabled),
5964 );
5965 }
5966 let configured = configured_modules
5967 .into_iter()
5968 .map(|module| (module.module_id.clone(), module))
5969 .collect::<BTreeMap<_, _>>();
5970
5971 let added = configured
5972 .keys()
5973 .filter(|module_id| !current.contains_key(*module_id))
5974 .cloned()
5975 .collect::<Vec<_>>();
5976 let removed = current
5977 .keys()
5978 .filter(|module_id| !configured.contains_key(*module_id))
5979 .cloned()
5980 .collect::<Vec<_>>();
5981 let mut changed_pending_reload = Vec::new();
5982 let mut configuration_changes = BTreeSet::new();
5983 let mut enabled_changes = BTreeSet::new();
5984 let mut unchanged = 0_u32;
5985
5986 for (module_id, configured_module) in &configured {
5987 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5988 else {
5989 continue;
5990 };
5991 let launch_changed = *current_spec != configured_module.module_spec();
5995 let configuration_changed =
5996 launch_changed || *current_health != configured_module.health;
5997 let enabled_changed = *current_enabled != configured_module.enabled;
5998 if configuration_changed {
5999 configuration_changes.insert(module_id.clone());
6000 }
6001 if launch_changed {
6002 changed_pending_reload.push(module_id.clone());
6003 }
6004 if enabled_changed {
6005 enabled_changes.insert(module_id.clone());
6006 }
6007 if !configuration_changed && !enabled_changed {
6008 unchanged = unchanged.saturating_add(1);
6009 }
6010 }
6011 Ok(RescanPlan {
6012 current,
6013 configured,
6014 added,
6015 removed,
6016 changed_pending_reload,
6017 configuration_changes,
6018 enabled_changes,
6019 unchanged,
6020 })
6021 }
6022
6023 async fn reconcile_supervised_modules(
6032 &self,
6033 supervisor: &Supervisor,
6034 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6035 preview: bool,
6036 requester: Principal,
6037 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6038 let plan = self.plan_rescan(configured_modules)?;
6039
6040 if preview {
6046 return Ok((plan.preview_result(), Vec::new()));
6047 }
6048 let changed = plan
6049 .configuration_changes
6050 .union(&plan.enabled_changes)
6051 .cloned()
6052 .collect::<Vec<_>>();
6053 let RescanPlan {
6054 current,
6055 configured,
6056 added,
6057 removed,
6058 changed_pending_reload,
6059 configuration_changes,
6060 enabled_changes,
6061 unchanged,
6062 } = plan;
6063 if !(added.is_empty()
6068 && removed.is_empty()
6069 && configuration_changes.is_empty()
6070 && enabled_changes.is_empty())
6071 {
6072 if let Some(context) = &self.rescan {
6073 context
6074 .generation
6075 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6076 }
6077 }
6078
6079 for module_id in &removed {
6080 let module = ¤t
6081 .get(module_id)
6082 .expect("removed module came from current supervisor state")
6083 .0;
6084 module
6085 .retire_requested(Some(requester.clone()))
6086 .await
6087 .map_err(|err| {
6088 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6089 })?;
6090 self.supervisor.record_rescan_removal(module_id);
6124 self.supervisor.retire(module_id);
6125 self.route_outages.forget(module_id);
6126 }
6127
6128 for module_id in configured.keys() {
6129 let Some((module, _, _, _)) = current.get(module_id) else {
6130 continue;
6131 };
6132 let configured_module = configured
6133 .get(module_id)
6134 .expect("configured module id came from configured map");
6135 if configuration_changes.contains(module_id) {
6136 module
6137 .update_configuration(
6138 configured_module.module_spec(),
6139 configured_module.health.clone(),
6140 configured_module.drain_timeout_ms,
6141 )
6142 .await
6143 .map_err(|err| {
6144 format!(
6145 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6146 )
6147 })?;
6148 }
6149 if enabled_changes.contains(module_id) {
6150 self.route_outages.mark_operator_action(module_id);
6153 module
6154 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6155 .await
6156 .map_err(|err| {
6157 self.route_outages.operator_action_ended_unrefused(module_id);
6158 format!(
6159 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6160 configured_module.enabled
6161 )
6162 })?;
6163 }
6164 }
6165
6166 for module_id in &added {
6167 let configured_module = configured
6168 .get(module_id)
6169 .expect("added module id came from configured map");
6170 supervisor
6171 .supervise_configured_requested(
6172 configured_module.module_spec(),
6173 configured_module.enabled,
6174 configured_module.health.clone(),
6175 configured_module.drain_timeout_ms,
6176 configured_module.restart,
6177 Some(requester.clone()),
6178 )
6179 .map_err(|err| {
6180 format!("failed to add module_id '{module_id}' during rescan: {err}")
6181 })?;
6182 }
6183
6184 Ok((
6185 SupervisorRescanResult {
6186 added,
6187 removed,
6188 changed_pending_reload,
6189 enabled_changes: enabled_changes.iter().cloned().collect(),
6190 unchanged,
6191 preview: false,
6192 restart_required: Vec::new(),
6195 capability_warnings: Vec::new(),
6196 module_actions: Vec::new(),
6197 earlier_pending: Vec::new(),
6198 expect: None,
6199 },
6200 changed,
6201 ))
6202 }
6203
6204 async fn handle_supervisor_set_enabled(
6205 &self,
6206 frame: Frame,
6207 module_id: String,
6208 enabled: bool,
6209 requester: Principal,
6210 ) -> Result<Vec<Frame>, RouterError> {
6211 let operation_lock = self.supervisor.operation_lock();
6212 let _operation_guard = operation_lock.lock().await;
6213 let Some(module) = self.supervisor.get(&module_id) else {
6214 return Ok(vec![control_error_frame(
6215 &frame,
6216 "unknown_module",
6217 format!("module_id '{module_id}' is not supervised"),
6218 )?]);
6219 };
6220
6221 self.route_outages.mark_operator_action(&module_id);
6224 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6225 Ok(applied) => applied,
6226 Err(err) => {
6227 self.route_outages
6228 .operator_action_ended_unrefused(&module_id);
6229 return Ok(vec![control_error_frame(
6230 &frame,
6231 "target_unavailable",
6232 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6233 )?]);
6234 }
6235 };
6236 if !applied {
6237 self.route_outages
6240 .operator_action_ended_unrefused(&module_id);
6241 }
6242
6243 self.capability_evaluator.wake_deadline_loop();
6244 self.refresh_capability_requirements();
6245 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6246 Ok(vec![control_response_body_frame(
6247 &frame,
6248 &response,
6249 "ClientControlResponse::SupervisorAck",
6250 )?])
6251 }
6252
6253 async fn handle_supervisor_health_probe(
6254 &self,
6255 frame: Frame,
6256 module_id: String,
6257 ) -> Result<Vec<Frame>, RouterError> {
6258 self.refresh_capability_requirements();
6259 let Some(registration) = self
6260 .registry
6261 .get_module(&module_id)
6262 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6263 else {
6264 return Ok(vec![control_error_frame(
6265 &frame,
6266 "unknown_module",
6267 format!("module_id '{module_id}' is not registered"),
6268 )?]);
6269 };
6270
6271 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6284 {
6285 return Ok(vec![control_error_frame(
6286 &frame,
6287 "health_not_advertised",
6288 format!("module_id '{module_id}' did not advertise health.check"),
6289 )?]);
6290 }
6291
6292 let deadline = Instant::now() + self.health_probe_timeout;
6293 let pending = match self.forwarding.begin_module_control_rpc_for(
6294 &module_id,
6295 MODULE_CONTROL_OP_HEALTH_CHECK,
6296 deadline,
6297 ) {
6298 Ok(pending) => pending,
6299 Err(err) => {
6300 return Ok(vec![control_error_frame(
6301 &frame,
6302 forwarding_error_code(&err),
6303 err.to_string(),
6304 )?])
6305 }
6306 };
6307
6308 let PendingModuleControlRpc {
6309 endpoint,
6310 module_sink,
6311 negotiated_ver,
6312 corr: probe_corr,
6313 receiver,
6314 } = pending;
6315 let mut guard =
6316 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6317 let probe_body =
6318 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6319 RouterError::backend(
6320 0,
6321 frame.header.corr,
6322 format!("failed to encode health.check request: {err}"),
6323 )
6324 })?;
6325 let probe_frame = Frame::build_with_version(
6326 negotiated_ver,
6327 FrameType::Request,
6328 control_flags(),
6329 0,
6330 0,
6331 probe_corr,
6332 probe_body,
6333 )
6334 .map_err(RouterError::FrameBuild)?;
6335
6336 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6339 Ok(Ok(())) => {}
6340 Ok(Err(err)) => {
6341 return Ok(vec![control_error_frame(
6342 &frame,
6343 "target_unavailable",
6344 err.to_string(),
6345 )?]);
6346 }
6347 Err(_) => {
6348 return Ok(vec![control_error_frame(
6349 &frame,
6350 "module_timeout",
6351 format!(
6352 "module_id '{module_id}' did not answer health.check within {:?}",
6353 self.health_probe_timeout
6354 ),
6355 )?]);
6356 }
6357 }
6358
6359 match timeout_at(deadline, receiver).await {
6360 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6361 guard.disarm();
6362 let Some(report) = response.health_report() else {
6363 return Ok(vec![control_error_frame(
6364 &frame,
6365 "invalid_control_body",
6366 "health.check RPC returned a non-health response",
6367 )?]);
6368 };
6369 let HealthReport {
6374 status,
6375 detail,
6376 metrics,
6377 } = report;
6378 let capability_detail = self
6379 .capability_evaluator
6380 .required_problem_detail(&module_id);
6381 let response = ClientControlResponse::SupervisorHealthProbe {
6382 module_id,
6383 status,
6384 detail: append_capability_problem_detail(detail, capability_detail),
6385 metrics,
6386 };
6387 Ok(vec![control_response_body_frame(
6388 &frame,
6389 &response,
6390 "ClientControlResponse::SupervisorHealthProbe",
6391 )?])
6392 }
6393 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6394 guard.disarm();
6395 Ok(vec![control_error_body_frame(&frame, body)?])
6396 }
6397 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6398 guard.disarm();
6399 Ok(vec![control_error_frame(
6400 &frame,
6401 "target_unavailable",
6402 message,
6403 )?])
6404 }
6405 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6406 guard.disarm();
6407 Ok(vec![control_error_frame(
6408 &frame,
6409 "invalid_control_body",
6410 message,
6411 )?])
6412 }
6413 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6414 guard.disarm();
6415 Ok(vec![control_error_frame(
6416 &frame,
6417 "invalid_control_body",
6418 format!("expected module-control op '{expected}', got '{actual}'"),
6419 )?])
6420 }
6421 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6422 guard.disarm();
6423 Ok(vec![control_error_frame(
6424 &frame,
6425 "module_timeout",
6426 format!(
6427 "module_id '{module_id}' answered health.check after {:?}",
6428 self.health_probe_timeout
6429 ),
6430 )?])
6431 }
6432 Ok(Err(_)) => Ok(vec![control_error_frame(
6433 &frame,
6434 "target_unavailable",
6435 "health.check waiter was canceled before the module responded",
6436 )?]),
6437 Err(_) => Ok(vec![control_error_frame(
6438 &frame,
6439 "module_timeout",
6440 format!(
6441 "module_id '{module_id}' did not answer health.check within {:?}",
6442 self.health_probe_timeout
6443 ),
6444 )?]),
6445 }
6446 }
6447
6448 fn supervisor_status(
6449 &self,
6450 module_id: &str,
6451 corr: u64,
6452 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6453 self.supervisor
6454 .get(module_id)
6455 .map(|module| {
6456 let warming = module.is_warming_for_control("status").map_err(|err| {
6457 RouterError::backend(
6458 0,
6459 corr,
6460 format!(
6461 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6462 ),
6463 )
6464 })?;
6465 module.status_for_control("status").map_err(|err| {
6466 RouterError::backend(
6467 0,
6468 corr,
6469 format!(
6470 "failed to read supervisor status for module_id '{module_id}': {err}"
6471 ),
6472 )
6473 }).map(|status| (status, warming))
6474 })
6475 .transpose()
6476 }
6477
6478 fn guard_module_control_op(
6479 &self,
6480 frame: &Frame,
6481 module_id: &str,
6482 op: &str,
6483 ) -> Result<Option<Frame>, RouterError> {
6484 if self.module_grants_op(module_id, op, frame.header.corr)? {
6485 return Ok(None);
6486 }
6487
6488 Ok(Some(control_error_frame(
6489 frame,
6490 "op_not_allowed",
6491 format!("module_id '{module_id}' did not grant control op '{op}'"),
6492 )?))
6493 }
6494
6495 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6496 let Some(registration) = self
6497 .registry
6498 .get_module(module_id)
6499 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6500 else {
6501 return Ok(false);
6502 };
6503 Ok(module_registration_grants_op(®istration.control_ops, op))
6504 }
6505
6506 fn handle_status_update(
6507 &self,
6508 endpoint: ModuleEndpointId,
6509 frame: Frame,
6510 ) -> Result<Vec<Frame>, RouterError> {
6511 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6512 Ok(update) => update,
6513 Err(err) => {
6514 if is_known_module_push_op(&frame.body) {
6519 return Ok(vec![control_error_frame(
6520 &frame,
6521 "invalid_control_body",
6522 format!("malformed module control push body: {err}"),
6523 )?]);
6524 }
6525 return Ok(Vec::new());
6526 }
6527 };
6528
6529 match update {
6530 ModuleControlPush::RouteStatus {
6531 route_channel,
6532 route_epoch,
6533 status,
6534 } => {
6535 self.forwarding
6536 .cache_status(endpoint, route_channel, route_epoch, status)
6537 .map_err(RouterError::Forwarding)?;
6538 }
6539 }
6540 Ok(Vec::new())
6541 }
6542
6543 fn handle_route_poll(
6544 &self,
6545 ctx: &RouteCtx,
6546 frame: Frame,
6547 route_channel: u16,
6548 route_epoch: u32,
6549 kind: PollKind,
6550 ) -> Result<Vec<Frame>, RouterError> {
6551 let snapshot = self
6552 .forwarding
6553 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6554 .map_err(RouterError::Forwarding)?;
6555 let response = match (kind, snapshot) {
6556 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6557 ClientControlResponse::RoutePoll {
6558 route_channel,
6559 route_epoch,
6560 status,
6561 live: None,
6562 }
6563 }
6564 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6565 route_channel,
6566 route_epoch,
6567 status: None,
6568 live: None,
6569 },
6570 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6571 let live = self
6597 .process_liveness
6598 .as_ref()
6599 .and_then(|source| source.process_live(&module_id))
6600 .unwrap_or(true);
6601 ClientControlResponse::RoutePoll {
6602 route_channel,
6603 route_epoch,
6604 status: None,
6605 live: Some(live),
6606 }
6607 }
6608 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6609 route_channel,
6610 route_epoch,
6611 status: None,
6612 live: Some(false),
6613 },
6614 };
6615
6616 Ok(vec![control_response_body_frame(
6617 &frame,
6618 &response,
6619 "ClientControlResponse::RoutePoll",
6620 )?])
6621 }
6622
6623 pub(crate) fn observe_module_control_completion(
6624 &self,
6625 completion: ModuleControlRpcCompletion,
6626 ) -> bool {
6627 match completion {
6628 ModuleControlRpcCompletion::Unknown => false,
6629 ModuleControlRpcCompletion::Settled => true,
6630 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6631 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6632 info!(
6633 module_id = %module_id,
6634 latency_ms,
6635 "late health.check answer proves the module is alive"
6636 );
6637 match self
6638 .supervisor
6639 .record_late_health_answer(&module_id, latency_ms)
6640 {
6641 Ok(true) => {}
6642 Ok(false) => debug!(
6643 module_id = %module_id,
6644 latency_ms,
6645 "late health.check answer has no active supervisor snapshot"
6646 ),
6647 Err(err) => warn!(
6648 module_id = %module_id,
6649 latency_ms,
6650 error = %err,
6651 "failed to record late health.check answer"
6652 ),
6653 }
6654 true
6655 }
6656 }
6657 }
6658
6659 fn refuse_to_end_module_connection_for_a_client(
6680 &self,
6681 module_connection_id: ConnectionId,
6682 corr: u64,
6683 err: ForwardingError,
6684 ) -> Result<(), RouterError> {
6685 if let ForwardingError::ConnectionClosing { connection_id } = err {
6686 if connection_id != module_connection_id {
6687 warn!(
6688 module_connection_id = module_connection_id.get(),
6689 client_connection_id = connection_id.get(),
6690 corr,
6691 "dropping a route.bind response for a closing client; the module connection keeps serving"
6692 );
6693 return Ok(());
6694 }
6695 }
6696 Err(RouterError::Forwarding(err))
6697 }
6698
6699 fn handle_module_relay_response(
6700 &self,
6701 connection_id: ConnectionId,
6702 frame: Frame,
6703 ) -> Result<Vec<Frame>, RouterError> {
6704 let mut secondary_error = None;
6705 let outcome = match frame.header.ty {
6706 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6707 Ok(probe) if probe.op == "route.bind" => {
6708 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6709 Ok(ModuleControlResponse::RouteBindAck {}) => {
6710 RouteBindRelayOutcome::Accepted
6711 }
6712 Ok(other) => {
6713 let message =
6714 format!("route.bind response carried unexpected body: {other:?}");
6715 secondary_error = Some(control_error_frame(
6716 &frame,
6717 "invalid_control_body",
6718 message.clone(),
6719 )?);
6720 RouteBindRelayOutcome::ModuleGone(message)
6721 }
6722 Err(err) => {
6723 let message = format!("malformed route.bind response body: {err}");
6724 secondary_error = Some(control_error_frame(
6725 &frame,
6726 "invalid_control_body",
6727 message.clone(),
6728 )?);
6729 RouteBindRelayOutcome::ModuleGone(message)
6730 }
6731 }
6732 }
6733 Ok(probe) => {
6734 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6735 {
6736 Ok(response) => ModuleControlRpcOutcome::Response(response),
6737 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6738 "malformed {} response body: {err}",
6739 probe.op
6740 )),
6741 };
6742 let completion = self
6743 .forwarding
6744 .complete_module_control_rpc(
6745 connection_id,
6746 frame.header.corr,
6747 Some(&probe.op),
6748 outcome,
6749 )
6750 .map_err(RouterError::Forwarding)?;
6751 if !self.observe_module_control_completion(completion) {
6752 debug!(
6753 connection_id = connection_id.get(),
6754 corr = frame.header.corr,
6755 op = %probe.op,
6756 "dropping late or unknown module-control RPC response"
6757 );
6758 }
6759 return Ok(Vec::new());
6760 }
6761 Err(err) => {
6762 if let Some(expected_op) = self
6763 .forwarding
6764 .pending_module_control_op(connection_id, frame.header.corr)
6765 .map_err(RouterError::Forwarding)?
6766 {
6767 let completion = self
6768 .forwarding
6769 .complete_module_control_rpc(
6770 connection_id,
6771 frame.header.corr,
6772 None,
6773 ModuleControlRpcOutcome::MalformedResponse(format!(
6774 "malformed {expected_op} response body: {err}"
6775 )),
6776 )
6777 .map_err(RouterError::Forwarding)?;
6778 if !self.observe_module_control_completion(completion) {
6779 debug!(
6780 connection_id = connection_id.get(),
6781 corr = frame.header.corr,
6782 "dropping late malformed module-control RPC response"
6783 );
6784 }
6785 return Ok(Vec::new());
6786 }
6787 let message = format!("malformed route.bind response body: {err}");
6788 secondary_error = Some(control_error_frame(
6789 &frame,
6790 "invalid_control_body",
6791 message.clone(),
6792 )?);
6793 RouteBindRelayOutcome::ModuleGone(message)
6794 }
6795 },
6796 FrameType::Error => {
6797 if self
6798 .forwarding
6799 .pending_module_control_op(connection_id, frame.header.corr)
6800 .map_err(RouterError::Forwarding)?
6801 .is_some()
6802 {
6803 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6804 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6805 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6806 "malformed module-control ERROR body: {err}"
6807 )),
6808 };
6809 let completion = self
6810 .forwarding
6811 .complete_module_control_rpc(
6812 connection_id,
6813 frame.header.corr,
6814 None,
6815 outcome,
6816 )
6817 .map_err(RouterError::Forwarding)?;
6818 if !self.observe_module_control_completion(completion) {
6819 debug!(
6820 connection_id = connection_id.get(),
6821 corr = frame.header.corr,
6822 "dropping late or unknown module-control RPC error"
6823 );
6824 }
6825 return Ok(Vec::new());
6826 }
6827 match serde_json::from_slice::<ErrorBody>(&frame.body) {
6828 Ok(body) => RouteBindRelayOutcome::Rejected(body),
6829 Err(err) => {
6830 let message = format!("malformed route.bind ERROR body: {err}");
6831 secondary_error = Some(control_error_frame(
6832 &frame,
6833 "invalid_control_body",
6834 message.clone(),
6835 )?);
6836 RouteBindRelayOutcome::ModuleGone(message)
6837 }
6838 }
6839 }
6840 ty => {
6841 return Ok(vec![control_error_frame(
6842 &frame,
6843 "unsupported_control_frame",
6844 format!("unsupported module channel-0 frame {ty:?}"),
6845 )?])
6846 }
6847 };
6848
6849 let settled =
6850 self.forwarding
6851 .complete_pending_relay(connection_id, frame.header.corr, outcome);
6852 let completion = match settled {
6853 Ok(completion) => completion,
6854 Err(err) => {
6855 self.refuse_to_end_module_connection_for_a_client(
6856 connection_id,
6857 frame.header.corr,
6858 err,
6859 )?;
6860 return Ok(secondary_error.into_iter().collect());
6861 }
6862 };
6863 if let Some(target) = completion.abandoned.as_ref() {
6864 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6865 }
6866 if !completion.settled {
6867 debug!(
6868 connection_id = connection_id.get(),
6869 corr = frame.header.corr,
6870 frame_type = ?frame.header.ty,
6871 "dropping late or unknown route.bind relay response"
6872 );
6873 }
6874 Ok(secondary_error.into_iter().collect())
6875 }
6876
6877 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6878 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6879 self.cleanup_connection_with_end_reason(
6883 connection_id,
6884 RegistrationEndReason::ExplicitGoodbye,
6885 )
6886 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6887 Ok(Vec::new())
6888 }
6889}
6890
6891impl Default for ControlHandler {
6892 fn default() -> Self {
6893 Self::new(Arc::new(Registry::default()))
6894 }
6895}
6896
6897impl crate::supervise::SwapPromotionObserver for ControlHandler {
6898 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6899 self.apply_registration_capabilities(registration);
6900 }
6901}
6902
6903fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6904 CapabilityRequirementStatus {
6905 consumer: status.consumer,
6906 capability: status.capability,
6907 need: match status.need {
6908 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6909 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6910 },
6911 verdict: status.verdict.as_str().to_string(),
6912 episode_seq: status.episode_seq,
6913 config_satisfiable: status.config_satisfiable,
6914 runtime_available: status.runtime_available,
6915 detail: status.detail,
6916 }
6917}
6918
6919fn append_capability_problem_detail(
6920 detail: Option<String>,
6921 capability_detail: Option<String>,
6922) -> Option<String> {
6923 match (detail, capability_detail) {
6924 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6925 (Some(detail), None) => Some(detail),
6926 (None, Some(capability_detail)) => Some(capability_detail),
6927 (None, None) => None,
6928 }
6929}
6930
6931fn subc_ops() -> Vec<String> {
6932 SUBC_CONTROL_OPS
6933 .iter()
6934 .map(|op| (*op).to_string())
6935 .collect()
6936}
6937
6938fn module_subc_ops() -> Vec<String> {
6939 SUBC_CONTROL_OPS
6940 .iter()
6941 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6942 .map(|op| (*op).to_string())
6943 .collect()
6944}
6945
6946#[cfg(test)]
6947fn module_baseline_control_ops() -> Vec<String> {
6948 MODULE_BASELINE_CONTROL_OPS
6949 .iter()
6950 .map(|op| (*op).to_string())
6951 .collect()
6952}
6953
6954fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6955 let mut seen = HashSet::new();
6956 let mut effective = Vec::new();
6957 for op in MODULE_BASELINE_CONTROL_OPS {
6958 if seen.insert((*op).to_string()) {
6959 effective.push((*op).to_string());
6960 }
6961 }
6962 for op in declared.unwrap_or_default() {
6963 if seen.insert(op.clone()) {
6964 effective.push(op);
6965 }
6966 }
6967 effective
6968}
6969
6970fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6971 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6972}
6973
6974fn target_module_id(target: &RouteTarget) -> &str {
6975 match target {
6976 RouteTarget::ToolProvider { module_id }
6977 | RouteTarget::ManagementSurface { module_id }
6978 | RouteTarget::InternalService { module_id, .. } => module_id,
6979 }
6980}
6981
6982fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6983 roles.iter().any(|role| match (target, role) {
6984 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6985 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6986 (
6987 RouteTarget::InternalService { service_id, .. },
6988 ProviderRole::InternalService {
6989 service_id: provided,
6990 ..
6991 },
6992 ) => service_id == provided,
6993 _ => false,
6994 })
6995}
6996
6997fn is_routable_role(role: &ProviderRole) -> bool {
6998 matches!(
6999 role,
7000 ProviderRole::ToolProvider { .. }
7001 | ProviderRole::ManagementSurface { .. }
7002 | ProviderRole::InternalService { .. }
7003 )
7004}
7005
7006#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7007enum ControlRequestBodyError {
7008 UnknownOp,
7009 InvalidBody,
7010}
7011
7012#[derive(Debug, Deserialize)]
7013struct ControlOpProbe {
7014 op: String,
7015}
7016
7017const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7020
7021fn is_known_module_push_op(body: &[u8]) -> bool {
7022 serde_json::from_slice::<ControlOpProbe>(body)
7023 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7024 .unwrap_or(false)
7025}
7026
7027fn is_known_module_request_op(body: &[u8]) -> bool {
7028 serde_json::from_slice::<ControlOpProbe>(body)
7029 .map(|probe| is_module_to_subc_op(&probe.op))
7030 .unwrap_or(false)
7031}
7032
7033fn is_module_to_subc_op(op: &str) -> bool {
7034 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7035}
7036
7037fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7038 debug!(
7039 op = %op,
7040 connection_id = connection_id.get(),
7041 corr,
7042 "control dispatch"
7043 );
7044}
7045
7046fn log_slow_control_dispatch(
7047 dispatch_started_at: Option<StdInstant>,
7048 op: &'static str,
7049 connection_id: ConnectionId,
7050 corr: u64,
7051 route_open_timings: Option<&RouteOpenTimings>,
7052) {
7053 let Some(dispatch_started_at) = dispatch_started_at else {
7054 return;
7055 };
7056 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7058 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7059 if let Some(timings) = route_open_timings {
7060 warn!(
7061 op = %op,
7062 connection_id = connection_id.get(),
7063 corr,
7064 elapsed_ms = elapsed.as_millis() as u64,
7065 module_id = ?timings.module_id,
7066 principal = %timings.principal,
7067 admission_ms = timings.admission.as_millis() as u64,
7068 project_root_ms = timings.project_root.as_millis() as u64,
7069 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7070 bind_send_ms = timings.bind_send.as_millis() as u64,
7071 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7072 "slow control dispatch"
7073 );
7074 return;
7075 }
7076 warn!(
7077 op = %op,
7078 connection_id = connection_id.get(),
7079 corr,
7080 elapsed_ms = elapsed.as_millis() as u64,
7081 "slow control dispatch"
7082 );
7083 }
7084}
7085
7086#[cfg(test)]
7088type TestCallerLookup =
7089 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7090
7091fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7093 match request {
7094 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7095 Some(PrivilegedOperation::Restart { module_id })
7096 }
7097 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7098 Some(PrivilegedOperation::Swap { module_id })
7099 }
7100 ClientControlRequest::SupervisorReload { module_id } => {
7101 Some(PrivilegedOperation::Reload { module_id })
7102 }
7103 ClientControlRequest::SupervisorRescan { preview } => {
7104 if *preview {
7105 None
7106 } else {
7107 Some(PrivilegedOperation::Rescan)
7108 }
7109 }
7110 ClientControlRequest::SupervisorRescanChecked {
7115 preview,
7116 candidate,
7117 expect,
7118 } => (!*preview && candidate.is_none() && expect.is_some())
7119 .then_some(PrivilegedOperation::Rescan),
7120 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7121 Some(PrivilegedOperation::SetEnabled {
7122 module_id,
7123 enabled: *enabled,
7124 })
7125 }
7126 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7127 Some(PrivilegedOperation::ReleaseReserved { module_id })
7128 }
7129 ClientControlRequest::ServerDescribe {}
7130 | ClientControlRequest::CatalogList { .. }
7131 | ClientControlRequest::RouteOpen { .. }
7132 | ClientControlRequest::RoutePoll { .. }
7133 | ClientControlRequest::SupervisorList {}
7134 | ClientControlRequest::SupervisorHealth {}
7135 | ClientControlRequest::SupervisorHealthProbe { .. }
7136 | ClientControlRequest::SupervisorRoutes { .. }
7137 | ClientControlRequest::SupervisorProvenance { .. }
7138 | ClientControlRequest::SupervisorStderrTail { .. }
7139 | ClientControlRequest::SupervisorTerminals { .. }
7140 | ClientControlRequest::SupervisorSpawnSnapshot {}
7141 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7142 }
7143}
7144
7145fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7146 match request {
7147 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7148 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7149 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7150 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7151 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7152 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7153 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7154 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7155 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7156 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7157 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7158 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7159 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7160 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7161 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7162 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7163 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7164 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7165 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7166 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7167 }
7168}
7169
7170fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7176 use subc_protocol::scope::ScopeKind;
7177 const MAX_REF_CHARS: usize = 64;
7178 let kind = match stamp.kind {
7179 ScopeKind::Head => "head",
7180 ScopeKind::Worker => "worker",
7181 ScopeKind::Ephemeral => "ephemeral",
7182 };
7183 let mut scope_ref: String = stamp.scope_ref.chars().take(MAX_REF_CHARS).collect();
7184 if stamp.scope_ref.chars().nth(MAX_REF_CHARS).is_some() {
7185 scope_ref.push('…');
7186 }
7187 format!(
7188 "{kind}/{}/{scope_ref:?}@{}",
7189 principal_label(&stamp.owner),
7190 stamp.scope_epoch
7191 )
7192}
7193
7194fn principal_label(principal: &Principal) -> String {
7195 match principal {
7196 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7197 Principal::Direct => "direct".to_string(),
7198 other => format!("{other:?}"),
7199 }
7200}
7201
7202fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7203 match request {
7204 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7205 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7206 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7207 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7208 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7209 }
7210}
7211
7212fn parse_client_control_request(
7213 body: &[u8],
7214) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7215 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7216 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7217 Ok(probe)
7218 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7219 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7220 {
7221 ControlRequestBodyError::InvalidBody
7222 }
7223 Ok(_) => ControlRequestBodyError::UnknownOp,
7224 Err(_) => ControlRequestBodyError::InvalidBody,
7225 };
7226 (err, classification)
7227 })
7228}
7229
7230fn parse_module_control_request_from_module(
7231 body: &[u8],
7232) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7233 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7234 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7235 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7236 Ok(_) => ControlRequestBodyError::UnknownOp,
7237 Err(_) => ControlRequestBodyError::InvalidBody,
7238 };
7239 (err, classification)
7240 })
7241}
7242
7243#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7244enum ProviderRoleKind {
7245 ToolProvider,
7246 PipelineStage,
7247 ManagementSurface,
7248 InternalService,
7249}
7250
7251fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7252 match role {
7253 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7254 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7255 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7256 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7257 }
7258}
7259
7260fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7261 roles.iter().map(provider_role_kind).collect()
7262}
7263
7264const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7267
7268#[derive(Debug, Default, PartialEq, Eq)]
7270struct ScopeOutcomeCounts {
7271 created: usize,
7272 replaced: usize,
7273 updated: usize,
7274 unchanged: usize,
7275 refused: usize,
7276}
7277
7278impl ScopeOutcomeCounts {
7279 fn of(results: &[ScopeRecordResult]) -> Self {
7280 let mut counts = Self::default();
7281 for result in results {
7282 let slot = match result.outcome {
7283 ScopeRecordOutcome::Created => &mut counts.created,
7284 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7285 ScopeRecordOutcome::Updated => &mut counts.updated,
7286 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7287 ScopeRecordOutcome::Refused => &mut counts.refused,
7288 };
7289 *slot += 1;
7290 }
7291 counts
7292 }
7293}
7294
7295#[cfg(test)]
7296mod scope_outcome_count_tests {
7297 use super::*;
7298
7299 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7300 ScopeRecordResult {
7301 scope_ref: "r".to_string(),
7302 scope_epoch: 1,
7303 outcome,
7304 code: None,
7305 message: None,
7306 version: None,
7307 parent_state: None,
7308 }
7309 }
7310
7311 #[test]
7314 fn every_outcome_is_counted_in_its_own_field() {
7315 let results = [
7316 result(ScopeRecordOutcome::Created),
7317 result(ScopeRecordOutcome::Created),
7318 result(ScopeRecordOutcome::Replaced),
7319 result(ScopeRecordOutcome::Updated),
7320 result(ScopeRecordOutcome::Unchanged),
7321 result(ScopeRecordOutcome::Refused),
7322 result(ScopeRecordOutcome::Refused),
7323 result(ScopeRecordOutcome::Refused),
7324 ];
7325 assert_eq!(
7326 ScopeOutcomeCounts::of(&results),
7327 ScopeOutcomeCounts {
7328 created: 2,
7329 replaced: 1,
7330 updated: 1,
7331 unchanged: 1,
7332 refused: 3,
7333 }
7334 );
7335 }
7336}
7337
7338fn capability_census_trigger(
7342 old: Option<&CapabilityDeclarations>,
7343 new: Option<&CapabilityDeclarations>,
7344) -> bool {
7345 let old_provides = old
7346 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7347 .unwrap_or_default();
7348 let old_denies = old
7349 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7350 .unwrap_or_default();
7351 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7352 provides: Vec::new(),
7353 requires: Vec::new(),
7354 must_never_reach: Vec::new(),
7355 });
7356
7357 new.provides
7358 .iter()
7359 .any(|capability| !old_provides.contains(capability))
7360 || new
7361 .must_never_reach
7362 .iter()
7363 .any(|capability| !old_denies.contains(capability))
7364}
7365
7366fn denied_capability<'a>(
7369 opening_manifest: &'a ModuleManifest,
7370 target_manifest: &ModuleManifest,
7371) -> Option<&'a str> {
7372 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7373 let target_capabilities = target_manifest.capabilities.as_ref()?;
7374 opening_capabilities
7375 .must_never_reach
7376 .iter()
7377 .find(|denied| {
7378 target_capabilities
7379 .provides
7380 .iter()
7381 .any(|provided| provided == *denied)
7382 })
7383 .map(String::as_str)
7384}
7385
7386fn catalog_update_frozen_field_message(
7387 registered: &ModuleManifest,
7388 provides: &[ProviderRole],
7389) -> Option<String> {
7390 let old_has_provides = !registered.provides.is_empty();
7391 let new_has_provides = !provides.is_empty();
7392 if old_has_provides != new_has_provides {
7393 return Some(format!(
7394 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7395 registered.module_id
7396 ));
7397 }
7398
7399 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7400 return Some(format!(
7401 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7402 registered.module_id
7403 ));
7404 }
7405
7406 let registered_concurrency = manifest_concurrency(registered);
7407 let mut candidate = registered.clone();
7408 candidate.provides = provides.to_vec();
7409 let candidate_concurrency = manifest_concurrency(&candidate);
7410 if candidate_concurrency != registered_concurrency {
7411 return Some(format!(
7412 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7413 registered.module_id, registered_concurrency, candidate_concurrency
7414 ));
7415 }
7416
7417 None
7420}
7421
7422fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7423 manifest.provides.iter().any(is_routable_role)
7424}
7425
7426fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7432 manifest
7433 .provides
7434 .iter()
7435 .find_map(|provider| match provider {
7436 ProviderRole::ToolProvider { concurrency, .. }
7437 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7438 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7439 })
7440 .unwrap_or(Concurrency::ModuleManaged)
7441}
7442
7443fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7450 let has_management_surface = manifest
7451 .provides
7452 .iter()
7453 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7454 if !has_management_surface {
7455 return false;
7456 }
7457 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7458 return false;
7459 };
7460 let Some(provides) = raw
7461 .get("manifest")
7462 .and_then(|manifest| manifest.get("provides"))
7463 .and_then(serde_json::Value::as_array)
7464 else {
7465 return false;
7466 };
7467 provides.iter().any(|role| {
7472 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7473 && role.get("concurrency").is_none()
7474 })
7475}
7476
7477fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7478 if peer_version != PROTOCOL_VERSION {
7479 return Err(format!(
7480 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7481 ));
7482 }
7483 Ok(PROTOCOL_VERSION)
7484}
7485
7486fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7487 Frame::build_with_version(
7488 response_version(frame),
7489 FrameType::Pong,
7490 frame.header.flags,
7491 0,
7492 0,
7493 frame.header.corr,
7494 Vec::new(),
7495 )
7496 .map_err(RouterError::FrameBuild)
7497}
7498
7499pub(crate) fn control_error_frame(
7500 frame: &Frame,
7501 code: &'static str,
7502 message: impl Into<String>,
7503) -> Result<Frame, RouterError> {
7504 control_error_body_frame(
7505 frame,
7506 ErrorBody {
7507 code: code.to_string(),
7508 message: message.into(),
7509 detail: None,
7510 },
7511 )
7512}
7513
7514fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7515 let body = serde_json::to_vec(&error).map_err(|err| {
7516 RouterError::backend(
7517 0,
7518 frame.header.corr,
7519 format!("failed to encode control ERROR: {err}"),
7520 )
7521 })?;
7522
7523 Frame::build_with_version(
7524 response_version(frame),
7525 FrameType::Error,
7526 control_flags(),
7527 0,
7528 0,
7529 frame.header.corr,
7530 body,
7531 )
7532 .map_err(RouterError::FrameBuild)
7533}
7534
7535fn control_response_body_frame<T: Serialize>(
7536 frame: &Frame,
7537 reply: &T,
7538 label: &'static str,
7539) -> Result<Frame, RouterError> {
7540 let body = serde_json::to_vec(reply).map_err(|err| {
7541 RouterError::backend(
7542 0,
7543 frame.header.corr,
7544 format!("failed to encode {label}: {err}"),
7545 )
7546 })?;
7547
7548 Frame::build_with_version(
7549 response_version(frame),
7550 FrameType::Response,
7551 control_flags(),
7552 0,
7553 0,
7554 frame.header.corr,
7555 body,
7556 )
7557 .map_err(RouterError::FrameBuild)
7558}
7559
7560fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7590 match err {
7591 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7592 ForwardingError::NoModuleConnection => "target_unavailable",
7593 ForwardingError::ModuleReloading { .. } => "module_reloading",
7594 ForwardingError::ClientRouteChannelExhausted { .. }
7595 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7596 ForwardingError::StaleModuleEndpoint
7597 | ForwardingError::UnknownReservation { .. }
7598 | ForwardingError::ConnectionClosing { .. }
7599 | ForwardingError::ClientEgressClosed { .. }
7600 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7601 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7604 ForwardingError::RelayCorrelationExhausted
7605 | ForwardingError::RouteOpenBuild(_)
7606 | ForwardingError::Poisoned => "forwarding_error",
7607 }
7608}
7609
7610fn response_version(frame: &Frame) -> u8 {
7611 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7612 frame.header.ver
7613 } else {
7614 PROTOCOL_VERSION
7615 }
7616}
7617
7618fn control_flags() -> Flags {
7619 Flags::new(false, Priority::Passive, false)
7620}
7621
7622fn send_goodbye_target_best_effort(
7627 counters: &DaemonCounters,
7628 target: &GoodbyeTarget,
7629 context: &'static str,
7630) {
7631 let Ok(frame) = Frame::build_with_version(
7632 target.negotiated_ver,
7633 FrameType::Goodbye,
7634 control_flags(),
7635 target.channel,
7636 target.epoch,
7637 0,
7638 Vec::new(),
7639 ) else {
7640 return;
7641 };
7642 crate::forwarding::send_module_route_goodbye(
7643 counters,
7644 &target.sink,
7645 frame,
7646 target.module_id.as_deref(),
7647 context,
7648 );
7649}
7650
7651pub(crate) fn send_route_control_pushes(
7652 forwarding: &ForwardingTable,
7653 routes: Vec<EndpointRoute>,
7654 push: ClientControlPush,
7655) {
7656 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7657 for route in routes {
7658 let target = route.goodbye_target;
7659 if let Some((existing, channels)) = targets
7660 .iter_mut()
7661 .find(|(existing, _)| existing.connection_id == target.connection_id)
7662 {
7663 debug_assert_eq!(
7664 existing.negotiated_ver, target.negotiated_ver,
7665 "one connection cannot negotiate multiple frame versions"
7666 );
7667 if !channels.contains(&target.channel) {
7668 channels.push(target.channel);
7669 }
7670 continue;
7671 }
7672 let channel = target.channel;
7673 targets.push((target, vec![channel]));
7674 }
7675 for (target, mut channels) in targets {
7676 channels.sort_unstable();
7677 let mut push = push.clone();
7678 match &mut push {
7679 ClientControlPush::RouteClosing {
7680 channels: covered, ..
7681 }
7682 | ClientControlPush::RouteClosed {
7683 channels: covered, ..
7684 } => *covered = channels,
7685 }
7686 let body = match serde_json::to_vec(&push) {
7687 Ok(body) => body,
7688 Err(err) => {
7689 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7690 continue;
7691 }
7692 };
7693 let frame = match Frame::build_with_version(
7694 target.negotiated_ver,
7695 FrameType::Push,
7696 control_flags(),
7697 0,
7698 0,
7699 0,
7700 body.clone(),
7701 ) {
7702 Ok(frame) => frame,
7703 Err(err) => {
7704 warn!(
7705 route_channel = target.channel,
7706 error = %err,
7707 "failed to build route lifecycle control PUSH frame"
7708 );
7709 continue;
7710 }
7711 };
7712 if let Err(err) = target.sink.try_send(frame) {
7713 if target.close_on_delivery_failure() {
7714 warn!(
7715 target_connection_id = target.connection_id.get(),
7716 route_channel = target.channel,
7717 error = %err,
7718 "route lifecycle control PUSH was not delivered to client; closing target connection"
7719 );
7720 let _ = forwarding.escalate_client_delivery_failure(
7721 target.connection_id,
7722 target.channel,
7723 target.epoch,
7724 CloseReason::new(
7725 "route_lifecycle_push_delivery_failed",
7726 format!(
7727 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7728 target.channel
7729 ),
7730 ),
7731 crate::forwarding::UndeliveredFrame {
7732 module_id: target.module_id.as_deref(),
7733 sink: &target.sink,
7734 },
7735 );
7736 }
7737 }
7738 }
7739}
7740
7741#[cfg(test)]
7742mod tests {
7743 mod activity_read {
7744 include!("activity_read_tests.rs");
7745 }
7746
7747 mod lifecycle_activity {
7748 include!("control_lifecycle_tests.rs");
7749 }
7750 #[cfg(unix)]
7751 #[tokio::test]
7752 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7753 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7754 let path = dir.join("subc.jsonc");
7755 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7756 "program":"/bin/sleep","args":["60"],"protocol":"none",
7757 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7758 }}}).to_string()).unwrap();
7759 let mut configured = crate::daemon_config::load(&path)
7760 .unwrap()
7761 .unwrap()
7762 .modules
7763 .pop()
7764 .unwrap();
7765 let registry = std::sync::Arc::new(crate::Registry::default());
7766 let handle = crate::SupervisorHandle::new();
7767 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7768 .with_handle(handle.clone());
7769 let module = supervisor
7770 .supervise_configured_with_health(
7771 configured.module_spec(),
7772 true,
7773 configured.health.clone(),
7774 None,
7775 configured.restart,
7776 )
7777 .unwrap();
7778 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7779 let before = module.status().unwrap().pid;
7780 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7781 configured.health.cadence = std::time::Duration::from_secs(3600);
7782 let health_only = handler
7783 .reconcile_supervised_modules(
7784 &supervisor,
7785 vec![configured.clone()],
7786 false,
7787 Principal::Direct,
7788 )
7789 .await
7790 .unwrap();
7791 assert!(
7792 health_only.0.changed_pending_reload.is_empty(),
7793 "health policy is already applied live"
7794 );
7795 assert_eq!(module.status().unwrap().pid, before);
7796 assert_eq!(
7797 module.configuration().unwrap().1.http,
7798 configured.health.http
7799 );
7800 configured.args = vec!["61".into()];
7801 let launch = handler
7802 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7803 .await
7804 .unwrap();
7805 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
7806 assert_eq!(
7807 module.status().unwrap().pid,
7808 before,
7809 "a launch edit is stored until reload"
7810 );
7811 module.drain().await.unwrap();
7812 }
7813 use cortexkit_test_support::ScratchDir;
7814 use std::{
7815 collections::BTreeMap,
7816 fmt,
7817 path::PathBuf,
7818 sync::{Arc, Mutex},
7819 time::Duration,
7820 };
7821
7822 use serde_json::{json, Value};
7823 use subc_protocol::{
7824 manifest::{
7825 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7826 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7827 },
7828 session::HealthStatus,
7829 FrameType,
7830 };
7831
7832 use super::*;
7833 use crate::{
7834 forwarding::{DataRoute, DataRouteState},
7835 registry::ChannelState,
7836 router::FrameSink,
7837 stderr_tail::DEFAULT_MAX_LINE_BYTES,
7838 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7839 RouteCtx, Router,
7840 };
7841 use tokio::{
7842 sync::mpsc,
7843 time::{sleep, Instant},
7844 };
7845 use tracing::{
7846 field::{Field, Visit},
7847 Event, Subscriber,
7848 };
7849 use tracing_subscriber::{layer::Context, prelude::*, Layer};
7850
7851 fn fake_aft_stub_path() -> PathBuf {
7870 let mut path = std::env::current_exe().expect("current_exe available in tests");
7871 path.pop(); path.pop(); path.push(if cfg!(windows) {
7874 "fake-aft-stub.exe"
7875 } else {
7876 "fake-aft-stub"
7877 });
7878 assert!(
7879 path.exists(),
7880 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7881 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7882 path.display()
7883 );
7884 path
7885 }
7886
7887 fn client_retries(code: &str) -> bool {
7892 subc_protocol::error_codes::is_retryable_route_open(code)
7893 }
7894
7895 #[test]
7906 fn retryability_of_forwarding_codes_matches_the_failure() {
7907 let transient = [
7910 ForwardingError::NoModuleConnection,
7911 ForwardingError::ModuleReloading {
7912 module_id: "m".into(),
7913 },
7914 ForwardingError::StaleModuleEndpoint,
7915 ForwardingError::UnknownReservation {
7916 client_channel: 1,
7917 module_channel: 1,
7918 },
7919 ForwardingError::ConnectionClosing {
7920 connection_id: ConnectionId::new(1),
7921 },
7922 ForwardingError::ClientEgressClosed {
7923 connection_id: ConnectionId::new(1),
7924 },
7925 ForwardingError::ModuleEgressUnavailable {
7926 connection_id: ConnectionId::new(1),
7927 },
7928 ];
7929 for err in transient {
7930 let code = forwarding_error_code(&err);
7931 assert!(
7932 client_retries(code),
7933 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7934 );
7935 }
7936
7937 let permanent = [
7942 ForwardingError::ConnectionRoleConflict {
7943 connection_id: ConnectionId::new(1),
7944 },
7945 ForwardingError::ClientRouteChannelExhausted {
7946 connection_id: ConnectionId::new(1),
7947 },
7948 ForwardingError::ModuleRouteChannelExhausted {
7949 endpoint: ModuleEndpointId {
7950 connection_id: ConnectionId::new(1),
7951 generation: 1,
7952 },
7953 },
7954 ForwardingError::RelayCorrelationExhausted,
7955 ForwardingError::RouteOpenBuild("x".into()),
7956 ForwardingError::Poisoned,
7957 ];
7958 for err in permanent {
7959 let code = forwarding_error_code(&err);
7960 assert!(
7961 !client_retries(code),
7962 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7963 );
7964 }
7965 }
7966
7967 #[tokio::test]
7995 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7996 let handler = ControlHandler::default();
7997 let frame =
7998 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7999
8000 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8004 assert_eq!(
8005 stamped,
8006 Principal::Direct,
8007 "a caller that proved nothing must not be stamped as a supervised module"
8008 );
8009
8010 let forged = handler
8015 .route_open_principal(
8016 &frame,
8017 Some(ConsumerIdentity {
8018 module_id: "aft".to_string(),
8019 launch_nonce: "not-a-real-nonce".to_string(),
8020 }),
8021 )
8022 .unwrap();
8023 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8024 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8025 }
8026
8027 #[test]
8044 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8045 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8046 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8047 let ClientControlRequest::RouteOpen {
8048 consumer_identity, ..
8049 } = parsed
8050 else {
8051 panic!("route.open body must parse as RouteOpen");
8052 };
8053 assert_eq!(
8054 consumer_identity,
8055 Some(ConsumerIdentity {
8056 module_id: "aft".to_string(),
8057 launch_nonce: "n".to_string(),
8058 }),
8059 "the wire field name must reach the value route_open_principal reads"
8060 );
8061 }
8062
8063 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8064 ModuleManifest::builder(module_id, "0.1.0")
8065 .protocol_ver(protocol_ver)
8066 .provides(vec![ProviderRole::ToolProvider {
8067 tools: vec![Tool {
8068 name: "read".to_string(),
8069 description: None,
8070 execution_mode: ExecutionMode::Pure,
8071 schema: json!({"type": "object"}),
8072 }],
8073 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8074 concurrency: Concurrency::ModuleManaged,
8075 emits_push: true,
8076 sub_supervises: true,
8077 }])
8078 .build()
8079 }
8080
8081 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8082 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8083 }
8084
8085 fn hello_frame_with_control_ops(
8086 module_id: &str,
8087 protocol_ver: u8,
8088 corr: u64,
8089 control_ops: Option<Vec<String>>,
8090 ) -> Frame {
8091 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8092 }
8093
8094 fn hello_frame_with_nonce(
8095 module_id: &str,
8096 protocol_ver: u8,
8097 corr: u64,
8098 launch_nonce: Option<&str>,
8099 ) -> Frame {
8100 hello_frame_full(
8101 module_id,
8102 protocol_ver,
8103 corr,
8104 None,
8105 launch_nonce.map(ToOwned::to_owned),
8106 )
8107 }
8108
8109 fn hello_frame_full(
8110 module_id: &str,
8111 protocol_ver: u8,
8112 corr: u64,
8113 control_ops: Option<Vec<String>>,
8114 launch_nonce: Option<String>,
8115 ) -> Frame {
8116 let body = serde_json::to_vec(&ModuleHelloBody {
8117 manifest: manifest(module_id, protocol_ver),
8118 protocol_ver,
8119 control_ops,
8120 launch_nonce,
8121 })
8122 .unwrap();
8123 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8124 }
8125
8126 fn non_routable_hello_frame_with_control_ops(
8127 module_id: &str,
8128 corr: u64,
8129 control_ops: Option<Vec<String>>,
8130 ) -> Frame {
8131 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8132 manifest.provides.clear();
8133 let body = serde_json::to_vec(&ModuleHelloBody {
8134 manifest,
8135 protocol_ver: PROTOCOL_VERSION,
8136 control_ops,
8137 launch_nonce: None,
8138 })
8139 .unwrap();
8140 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8141 }
8142
8143 fn capability_grammar_hello_frame(
8144 capabilities: Value,
8145 runtime_computed: Option<Value>,
8146 corr: u64,
8147 ) -> Frame {
8148 let mut body = serde_json::to_value(ModuleHelloBody {
8149 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8150 protocol_ver: PROTOCOL_VERSION,
8151 control_ops: None,
8152 launch_nonce: None,
8153 })
8154 .expect("HELLO body serializes");
8155 body["manifest"]["capabilities"] = capabilities;
8156 if let Some(runtime_computed) = runtime_computed {
8157 body["runtime_computed"] = runtime_computed;
8158 }
8159 Frame::build(
8160 FrameType::Hello,
8161 control_flags(),
8162 0,
8163 0,
8164 corr,
8165 serde_json::to_vec(&body).expect("HELLO body reserializes"),
8166 )
8167 .expect("HELLO frame builds")
8168 }
8169
8170 fn channel_request(channel: u16, corr: u64) -> Frame {
8171 Frame::build(
8172 FrameType::Request,
8173 Flags::new(true, Priority::Interactive, false),
8174 channel,
8175 0,
8176 corr,
8177 b"opaque".to_vec(),
8178 )
8179 .unwrap()
8180 }
8181
8182 fn route_ctx(
8183 connection_id: ConnectionId,
8184 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8185 let (tx, rx) = mpsc::channel(8);
8186 (
8187 RouteCtx {
8188 connection_id,
8189 egress: FrameSink::new(tx),
8190 },
8191 rx,
8192 )
8193 }
8194
8195 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8196 serde_json::from_slice(&frame.body).unwrap()
8197 }
8198
8199 async fn hello_via_sink(
8204 handler: &ControlHandler,
8205 ctx: &RouteCtx,
8206 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8207 hello: Frame,
8208 ) -> Frame {
8209 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8210 assert!(
8211 replies.is_empty(),
8212 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8213 );
8214 let ack = rx
8215 .try_recv()
8216 .expect("HELLO_ACK is queued on the module sink")
8217 .frame;
8218 assert_eq!(ack.header.ty, FrameType::HelloAck);
8219 ack
8220 }
8221
8222 fn parse_error(frame: &Frame) -> Value {
8223 serde_json::from_slice(&frame.body).unwrap()
8224 }
8225
8226 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8227 serde_json::from_slice(&frame.body).unwrap()
8228 }
8229
8230 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8231 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8232 route_channel,
8233 route_epoch: 0,
8234 kind,
8235 })
8236 .unwrap();
8237 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8238 }
8239
8240 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8241 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8242 module_id: module_id.to_string(),
8243 })
8244 .unwrap();
8245 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8246 }
8247
8248 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8249 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8250 }
8251
8252 fn route_open_frame_with_consumer_capabilities(
8253 corr: u64,
8254 module_id: &str,
8255 project_root: ScratchDir,
8256 consumer_capabilities: Option<Vec<String>>,
8257 ) -> Frame {
8258 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8259 target: RouteTarget::ToolProvider {
8260 module_id: module_id.to_string(),
8261 },
8262 identity: BindIdentity::new(
8263 project_root.path().to_path_buf(),
8264 "unit".to_string(),
8265 "session".to_string(),
8266 ),
8267 consumer_identity: None,
8268 consumer_capabilities,
8269 role_versions: None,
8270 admission_facts: None,
8271 scope: None,
8272 })
8273 .unwrap();
8274 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8275 }
8276
8277 fn route_open_frame_with_role_versions(
8278 corr: u64,
8279 module_id: &str,
8280 project_root: ScratchDir,
8281 role_versions: Option<BTreeMap<String, String>>,
8282 ) -> Frame {
8283 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8284 target: RouteTarget::ToolProvider {
8285 module_id: module_id.to_string(),
8286 },
8287 identity: BindIdentity::new(
8288 project_root.path().to_path_buf(),
8289 "unit".to_string(),
8290 format!("session-{corr}"),
8291 ),
8292 consumer_identity: None,
8293 consumer_capabilities: None,
8294 role_versions,
8295 admission_facts: None,
8296 scope: None,
8297 })
8298 .unwrap();
8299 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8300 }
8301
8302 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8303 entries
8304 .iter()
8305 .map(|(role, version)| (role.to_string(), version.to_string()))
8306 .collect()
8307 }
8308
8309 fn route_open_frame_with_admission_facts(
8310 corr: u64,
8311 module_id: &str,
8312 project_root: ScratchDir,
8313 consumer_identity: Option<subc_control::ConsumerIdentity>,
8314 facts: Option<Value>,
8315 ) -> Frame {
8316 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8317 target: RouteTarget::ToolProvider {
8318 module_id: module_id.to_string(),
8319 },
8320 identity: BindIdentity::new(
8321 project_root.path().to_path_buf(),
8322 "unit".to_string(),
8323 format!("session-{corr}"),
8324 ),
8325 consumer_identity,
8326 consumer_capabilities: None,
8327 role_versions: None,
8328 admission_facts: facts,
8329 scope: None,
8330 })
8331 .unwrap();
8332 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8333 }
8334
8335 #[derive(Clone, Default)]
8336 struct EventCapture {
8337 events: Arc<Mutex<Vec<CapturedEvent>>>,
8338 }
8339
8340 #[derive(Clone, Debug)]
8341 struct CapturedEvent {
8342 target: String,
8343 level: tracing::Level,
8344 fields: BTreeMap<String, String>,
8345 }
8346
8347 impl EventCapture {
8348 fn events(&self) -> Vec<CapturedEvent> {
8349 self.events.lock().unwrap().clone()
8350 }
8351 }
8352
8353 impl<S> Layer<S> for EventCapture
8354 where
8355 S: Subscriber,
8356 {
8357 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8358 let mut visitor = EventFieldVisitor::default();
8359 event.record(&mut visitor);
8360 self.events.lock().unwrap().push(CapturedEvent {
8361 target: event.metadata().target().to_string(),
8362 level: *event.metadata().level(),
8363 fields: visitor.fields,
8364 });
8365 }
8366 }
8367
8368 #[derive(Default)]
8369 struct EventFieldVisitor {
8370 fields: BTreeMap<String, String>,
8371 }
8372
8373 impl Visit for EventFieldVisitor {
8374 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8375 self.fields
8376 .insert(field.name().to_string(), format!("{value:?}"));
8377 }
8378 }
8379
8380 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8381 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8382 status,
8383 detail: Some("warming".to_string()),
8384 metrics: Some(json!({"queue_depth": 3})),
8385 })
8386 .unwrap();
8387 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8388 }
8389
8390 fn route_bind_ack(corr: u64) -> Frame {
8391 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8392 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8393 }
8394
8395 fn unique_project_root(label: &str) -> ScratchDir {
8396 ScratchDir::new(label)
8397 }
8398
8399 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8400 match parse_route_poll(frame) {
8401 ClientControlResponse::RoutePoll {
8402 status: None,
8403 live: Some(live),
8404 ..
8405 } => assert_eq!(live, expected_live),
8406 other => panic!("unexpected route.poll response: {other:?}"),
8407 }
8408 }
8409
8410 fn bind_liveness_route(
8411 registry: &Registry,
8412 forwarding: &ForwardingTable,
8413 module_id: &str,
8414 ) -> (RouteCtx, u16, u32) {
8415 let module_connection = ConnectionId::new(101);
8416 let client_connection = ConnectionId::new(202);
8417 let registration = registry
8418 .register_with_control_ops(
8419 manifest(module_id, PROTOCOL_VERSION),
8420 PROTOCOL_VERSION,
8421 module_connection,
8422 module_baseline_control_ops(),
8423 )
8424 .unwrap();
8425 let (module_tx, _module_rx) = mpsc::channel(8);
8426 let endpoint = forwarding
8427 .register_module_connection(
8428 module_connection,
8429 module_id.to_string(),
8430 PROTOCOL_VERSION,
8431 manifest_concurrency(®istration.manifest),
8432 FrameSink::new(module_tx),
8433 )
8434 .unwrap();
8435 let (client_ctx, _client_rx) = route_ctx(client_connection);
8436 let pending = forwarding
8437 .begin_route_bind_relay_for_test(
8438 client_connection,
8439 client_ctx.egress.clone(),
8440 1,
8441 module_id,
8442 )
8443 .unwrap();
8444 assert_eq!(pending.endpoint, endpoint);
8445 let route_channel = pending.client_channel;
8446 let route_epoch = pending.client_epoch;
8447 forwarding
8448 .complete_pending_relay(
8449 module_connection,
8450 pending.corr,
8451 RouteBindRelayOutcome::Accepted,
8452 )
8453 .unwrap();
8454 (client_ctx, route_channel, route_epoch)
8455 }
8456
8457 struct FakeProcessLiveness {
8458 live: Option<bool>,
8459 }
8460
8461 impl ModuleProcessLiveness for FakeProcessLiveness {
8462 fn process_live(&self, _module_id: &str) -> Option<bool> {
8463 self.live
8464 }
8465 }
8466
8467 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8468 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8469 {
8470 let registry = Arc::new(Registry::default());
8471 let supervisor_handle = SupervisorHandle::new();
8472 let supervisor = Supervisor::new_for_test(
8473 Arc::clone(®istry),
8474 RestartPolicy::new(1, Duration::from_millis(10)),
8475 )
8476 .with_handle(supervisor_handle.clone());
8477 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8478 let module = supervisor
8479 .spawn(ModuleSpec {
8480 module_id: "stderr-tail-wire".to_string(),
8481 program: fake_aft_stub_path(),
8482 args: Vec::new(),
8483 env: vec![
8484 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8485 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8486 ],
8487 reserved: false,
8488 reserved_prefixes: Vec::new(),
8489 protocol: ModuleProtocol::Subc,
8490 overlap: Default::default(),
8491 })
8492 .unwrap();
8493
8494 let deadline = Instant::now() + Duration::from_secs(5);
8495 loop {
8496 let tail = module.stderr_tail(None, None);
8497 if tail
8498 .entries
8499 .iter()
8500 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8501 && tail.entries.iter().any(|entry| {
8502 matches!(
8503 entry,
8504 TailEntry::Line {
8505 truncated: true,
8506 ..
8507 }
8508 )
8509 })
8510 {
8511 break;
8512 }
8513 assert!(
8514 Instant::now() < deadline,
8515 "module did not produce a truncated line and restart boundary: {tail:?}"
8516 );
8517 sleep(Duration::from_millis(10)).await;
8518 }
8519
8520 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8521 let request = ClientControlRequest::SupervisorStderrTail {
8522 module_id: "stderr-tail-wire".to_string(),
8523 max_lines: None,
8524 max_bytes: None,
8525 };
8526 let frame = Frame::build(
8527 FrameType::Request,
8528 control_flags(),
8529 0,
8530 0,
8531 1,
8532 serde_json::to_vec(&request).unwrap(),
8533 )
8534 .unwrap();
8535 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8536 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8537 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8538 serde_json::from_slice(&responses[0].body).unwrap()
8539 else {
8540 panic!("expected supervisor.stderr_tail response");
8541 };
8542
8543 assert!(
8544 tail.entries
8545 .iter()
8546 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8547 "the control response lost the restart boundary"
8548 );
8549 let Some(StderrTailEntry::Line {
8550 text,
8551 truncated,
8552 at_ms,
8553 }) = tail.entries.iter().find(|entry| {
8554 matches!(
8555 entry,
8556 StderrTailEntry::Line {
8557 truncated: true,
8558 ..
8559 }
8560 )
8561 })
8562 else {
8563 panic!("the control response lost the truncated line");
8564 };
8565 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8566 assert!(*truncated);
8567 assert!(
8568 at_ms.is_some(),
8569 "the control response lost the line's capture time"
8570 );
8571 }
8572
8573 #[tokio::test(flavor = "current_thread")]
8578 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8579 let dir = ScratchDir::new("terminals-off-worker");
8580 let journal_path = dir.join("terminals.jsonl");
8581 let registry = Arc::new(Registry::default());
8582 let supervisor_handle = SupervisorHandle::new();
8583 let supervisor =
8584 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8585 .with_handle(supervisor_handle.clone())
8586 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8587 let module = supervisor
8588 .spawn(ModuleSpec {
8589 module_id: "terminal-off-worker".to_string(),
8590 program: fake_aft_stub_path(),
8591 args: Vec::new(),
8592 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8593 reserved: false,
8594 reserved_prefixes: Vec::new(),
8595 protocol: ModuleProtocol::Subc,
8596 overlap: Default::default(),
8597 })
8598 .unwrap();
8599 let deadline = Instant::now() + Duration::from_secs(5);
8600 while module.terminal_history().entries.len() != 2 {
8601 assert!(Instant::now() < deadline, "module did not record two exits");
8602 sleep(Duration::from_millis(10)).await;
8603 }
8604
8605 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8606 let handler =
8607 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8608 let frame = Frame::build(
8609 FrameType::Request,
8610 control_flags(),
8611 0,
8612 0,
8613 1,
8614 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8615 module_id: "terminal-off-worker".to_string(),
8616 })
8617 .unwrap(),
8618 )
8619 .unwrap();
8620 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8621 let spawned_at = std::time::Instant::now();
8622 let read = tokio::spawn({
8623 let handler = Arc::clone(&handler);
8624 async move { handler.handle_control_frame(&ctx, frame).await }
8625 });
8626 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8629 .await
8630 .unwrap()
8631 .expect("the history read reached its pause");
8632 let elapsed = spawned_at.elapsed();
8633 assert!(
8634 elapsed < Duration::from_secs(2) && !read.is_finished(),
8635 "this task could not run while the history read was paused \
8636 (resumed after {elapsed:?}, read finished: {})",
8637 read.is_finished()
8638 );
8639
8640 drop(release);
8641 let responses = read.await.unwrap().unwrap();
8642 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8643 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8644 panic!("expected supervisor.terminals response");
8645 };
8646 assert_eq!(terminals.entries.len(), 2);
8647 assert_eq!(terminals.journal_skipped_lines, 0);
8648 assert_eq!(terminals.journal_read_errors, 0);
8649 }
8650
8651 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8652 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8653 let registry = Arc::new(Registry::default());
8654 let supervisor_handle = SupervisorHandle::new();
8655 let supervisor =
8656 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8657 .with_handle(supervisor_handle.clone());
8658 let module = supervisor
8659 .spawn(ModuleSpec {
8660 module_id: "terminal-golden".to_string(),
8661 program: fake_aft_stub_path(),
8662 args: Vec::new(),
8663 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8664 reserved: false,
8665 reserved_prefixes: Vec::new(),
8666 protocol: ModuleProtocol::Subc,
8667 overlap: Default::default(),
8668 })
8669 .unwrap();
8670
8671 let deadline = Instant::now() + Duration::from_secs(5);
8672 while module.terminal_history().entries.len() != 2 {
8673 assert!(
8674 Instant::now() < deadline,
8675 "module did not retain two terminal exits: {:?}",
8676 module.terminal_history()
8677 );
8678 sleep(Duration::from_millis(10)).await;
8679 }
8680
8681 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8682 let request = ClientControlRequest::SupervisorTerminals {
8683 module_id: "terminal-golden".to_string(),
8684 };
8685 let frame = Frame::build(
8686 FrameType::Request,
8687 control_flags(),
8688 0,
8689 0,
8690 1,
8691 serde_json::to_vec(&request).unwrap(),
8692 )
8693 .unwrap();
8694 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8695 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8696 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8697 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8698 panic!("expected supervisor.terminals response");
8699 };
8700 assert_eq!(terminals.entries.len(), 2);
8701 assert_eq!(terminals.dropped, 0);
8702
8703 let mut rendered = serde_json::to_value(response).unwrap();
8704 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8707 for (index, entry) in rendered["entries"]
8708 .as_array_mut()
8709 .expect("terminal response entries array")
8710 .iter_mut()
8711 .enumerate()
8712 {
8713 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8714 }
8715
8716 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8717 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8718 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8719 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8720 std::fs::write(&golden_path, &serialized).unwrap();
8721 }
8722 let expected: Value =
8723 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8724 assert_eq!(rendered, expected);
8725 }
8726
8727 #[test]
8728 fn hello_registers_manifest_and_returns_ack() {
8729 let registry = Arc::new(Registry::default());
8730 let handler = ControlHandler::new(Arc::clone(®istry));
8731 let conn = ConnectionId::new(1);
8732
8733 let responses = handler
8734 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8735 .unwrap();
8736
8737 assert_eq!(responses.len(), 1);
8738 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8739 assert_eq!(responses[0].header.channel, 0);
8740 assert_eq!(responses[0].header.corr, 7);
8741 let ack = parse_ack(&responses[0]);
8742 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8743 assert!(ack
8744 .subc_capabilities
8745 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8746 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8747 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8748 assert!(ack
8749 .subc_ops
8750 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8751 assert!(ack
8752 .subc_ops
8753 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8754
8755 let registration = registry.get_module("aft").unwrap().unwrap();
8756 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8757 assert_eq!(registration.state, ChannelState::Active);
8758 assert_eq!(registration.connection_id, conn);
8759 assert_eq!(registration.control_ops, module_baseline_control_ops());
8760 }
8761
8762 #[test]
8763 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8764 let invalid_identifiers = [
8765 ("case_change", "credentials-Provider/v1"),
8766 ("leading_zero", "credentials-provider/v01"),
8767 ("trailing_hyphen", "credentials-provider-/v1"),
8768 ("consecutive_hyphens", "credentials--provider/v1"),
8769 ("uppercase", "Credentials-provider/v1"),
8770 ("missing_v", "credentials-provider/1"),
8771 ("whitespace", "credentials provider/v1"),
8772 ("zero_version", "credentials-provider/v0"),
8773 ("out_of_range_version", "credentials-provider/v4294967296"),
8774 (
8775 "overlength_name",
8776 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8777 ),
8778 ];
8779 let mut cases = invalid_identifiers
8780 .into_iter()
8781 .map(|(name, identifier)| {
8782 (
8783 format!("identifier_{name}"),
8784 "capabilities.provides[0]".to_string(),
8785 identifier.to_string(),
8786 json!({ "provides": [identifier] }),
8787 None,
8788 )
8789 })
8790 .collect::<Vec<_>>();
8791 cases.extend([
8792 (
8793 "unknown_need".to_string(),
8794 "capabilities.requires[0].need".to_string(),
8795 "deferred".to_string(),
8796 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8797 None,
8798 ),
8799 (
8800 "duplicate_provides".to_string(),
8801 "capabilities.provides[1]".to_string(),
8802 "credentials-provider/v1".to_string(),
8803 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8804 None,
8805 ),
8806 (
8807 "duplicate_must_never_reach".to_string(),
8808 "capabilities.must_never_reach[1]".to_string(),
8809 "credentials-provider/v1".to_string(),
8810 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8811 None,
8812 ),
8813 (
8814 "duplicate_requires_same_need".to_string(),
8815 "capabilities.requires[1]".to_string(),
8816 "credentials-provider/v1".to_string(),
8817 json!({ "requires": [
8818 { "capability": "credentials-provider/v1", "need": "required" },
8819 { "capability": "credentials-provider/v1", "need": "required" }
8820 ] }),
8821 None,
8822 ),
8823 (
8824 "duplicate_requires_conflicting_need".to_string(),
8825 "capabilities.requires[1]".to_string(),
8826 "credentials-provider/v1".to_string(),
8827 json!({ "requires": [
8828 { "capability": "credentials-provider/v1", "need": "required" },
8829 { "capability": "credentials-provider/v1", "need": "optional" }
8830 ] }),
8831 None,
8832 ),
8833 (
8834 "capabilities_root_pointer".to_string(),
8835 "runtime_computed[0]".to_string(),
8836 "/capabilities".to_string(),
8837 json!({}),
8838 Some(json!(["/capabilities"])),
8839 ),
8840 (
8841 "capabilities_descendant_pointer".to_string(),
8842 "runtime_computed[0]".to_string(),
8843 "/capabilities/provides".to_string(),
8844 json!({}),
8845 Some(json!(["/capabilities/provides"])),
8846 ),
8847 (
8848 "malformed_pointer_without_leading_slash".to_string(),
8849 "runtime_computed[0]".to_string(),
8850 "capabilities".to_string(),
8851 json!({}),
8852 Some(json!(["capabilities"])),
8853 ),
8854 (
8855 "malformed_pointer_escape".to_string(),
8856 "runtime_computed[0]".to_string(),
8857 "/roles/~2/tools".to_string(),
8858 json!({}),
8859 Some(json!(["/roles/~2/tools"])),
8860 ),
8861 (
8862 "unknown_capabilities_field".to_string(),
8863 "capabilities.future".to_string(),
8864 "<array>".to_string(),
8865 json!({ "future": [] }),
8866 None,
8867 ),
8868 ]);
8869
8870 for (index, (name, field, value, capabilities, runtime_computed)) in
8871 cases.into_iter().enumerate()
8872 {
8873 let registry = Arc::new(Registry::default());
8874 let handler = ControlHandler::new(Arc::clone(®istry));
8875 let response = handler
8876 .handle_control(
8877 ConnectionId::new((index + 1) as u64),
8878 capability_grammar_hello_frame(
8879 capabilities,
8880 runtime_computed,
8881 index as u64 + 1,
8882 ),
8883 )
8884 .expect("invalid HELLO returns a refusal");
8885
8886 assert_eq!(response.len(), 1, "{name} must emit one refusal");
8887 let error = parse_error(&response[0]);
8888 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8889 let message = error["message"]
8890 .as_str()
8891 .expect("error message is a string");
8892 assert!(
8893 message.contains(&field),
8894 "{name}: field missing from {message}"
8895 );
8896 assert!(
8897 message.contains(&value),
8898 "{name}: value missing from {message}"
8899 );
8900 assert_eq!(
8901 registry
8902 .active_registration_count()
8903 .expect("registry reads"),
8904 0,
8905 "{name}: refused HELLO must not create a catalog entry"
8906 );
8907 }
8908 }
8909
8910 #[test]
8911 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8912 let registry = Arc::new(Registry::default());
8913 let handler = ControlHandler::new(Arc::clone(®istry));
8914 let capabilities = json!({
8915 "provides": ["credentials-provider/v1"],
8916 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8917 "must_never_reach": ["federation-transport/v1"]
8918 });
8919 let response = handler
8920 .handle_control(
8921 ConnectionId::new(99),
8922 capability_grammar_hello_frame(
8923 capabilities.clone(),
8924 Some(json!(["/roles/0/tools"])),
8925 99,
8926 ),
8927 )
8928 .expect("valid HELLO registers");
8929 assert_eq!(response[0].header.ty, FrameType::HelloAck);
8930
8931 let request = Frame::build(
8932 FrameType::Request,
8933 control_flags(),
8934 0,
8935 0,
8936 100,
8937 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8938 .expect("catalog request serializes"),
8939 )
8940 .expect("catalog request frame builds");
8941 let response = handler
8942 .handle_catalog_list(request, None)
8943 .expect("catalog list succeeds");
8944 let ClientControlResponse::CatalogList { modules, .. } =
8945 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8946 else {
8947 panic!("catalog request must return catalog.list");
8948 };
8949 assert_eq!(modules.len(), 1);
8950 assert_eq!(
8951 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8952 capabilities
8953 );
8954 }
8955
8956 #[test]
8957 fn catalog_list_mirrors_management_operation_description() {
8958 let registry = Arc::new(Registry::default());
8959 let handler = ControlHandler::new(Arc::clone(®istry));
8960 let description = "List managed records and return their identifiers and metadata.";
8961 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8962 manifest.provides = vec![ProviderRole::ManagementSurface {
8963 operations: vec![ManagementOperation {
8964 name: "records.list".to_string(),
8965 kind: ManagementOperationKind::Query,
8966 description: Some(description.to_string()),
8967 }],
8968 config_schema: json!({"type": "object"}),
8969 observability: vec![ObservabilitySurface {
8970 name: "records.stats".to_string(),
8971 kind: ObservabilityKind::Snapshot,
8972 }],
8973 identity_scope: vec![IdentityScope::Project],
8974 concurrency: Concurrency::ModuleManaged,
8975 }];
8976 registry
8977 .register_with_control_ops(
8978 manifest,
8979 PROTOCOL_VERSION,
8980 ConnectionId::new(99),
8981 Vec::new(),
8982 )
8983 .expect("described management manifest registers");
8984
8985 let request = Frame::build(
8986 FrameType::Request,
8987 control_flags(),
8988 0,
8989 0,
8990 100,
8991 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8992 .expect("catalog request serializes"),
8993 )
8994 .expect("catalog request frame builds");
8995 let response = handler
8996 .handle_catalog_list(request, None)
8997 .expect("catalog list succeeds");
8998 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8999 assert_eq!(
9000 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9001 "catalog.list must preserve the declared operation description verbatim"
9002 );
9003 }
9004
9005 #[test]
9006 fn catalog_list_mirrors_registered_operation_delivery() {
9007 let registry = Arc::new(Registry::default());
9008 let handler = ControlHandler::new(Arc::clone(®istry));
9009 let delivery =
9010 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9011 let mut body = serde_json::to_value(ModuleHelloBody {
9012 manifest: manifest("watch-management", PROTOCOL_VERSION),
9013 protocol_ver: PROTOCOL_VERSION,
9014 control_ops: None,
9015 launch_nonce: None,
9016 })
9017 .unwrap();
9018 body["manifest"]["provides"] = json!([{
9019 "role": "management_surface",
9020 "operations": [
9021 {"name": "status", "kind": "query"},
9022 {"name": "future", "kind": "query"},
9023 {"name": "reset", "kind": "mutate"}
9024 ],
9025 "config_schema": {}, "observability": [], "identity_scope": []
9026 }]);
9027 body["manifest"]["operation_delivery"] = delivery.clone();
9028 let hello = Frame::build(
9029 FrameType::Hello,
9030 control_flags(),
9031 0,
9032 0,
9033 99,
9034 serde_json::to_vec(&body).unwrap(),
9035 )
9036 .unwrap();
9037 let response = handler
9038 .handle_control(ConnectionId::new(99), hello)
9039 .unwrap();
9040 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9041
9042 let (_, registrations) = registry.list_modules().unwrap();
9043 let registered = ®istrations[0].manifest;
9044 assert!(registered.is_watch_operation("status"));
9045 assert!(!registered.is_watch_operation("future"));
9046 assert!(!registered.is_watch_operation("reset"));
9047 assert!(!registered.is_watch_operation("missing"));
9048
9049 let request = Frame::build(
9050 FrameType::Request,
9051 control_flags(),
9052 0,
9053 0,
9054 100,
9055 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9056 )
9057 .unwrap();
9058 let response = handler.handle_catalog_list(request, None).unwrap();
9059 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9060 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9061 let ClientControlResponse::CatalogList { modules, .. } =
9062 serde_json::from_slice(&response[0].body).unwrap()
9063 else {
9064 panic!("expected catalog.list");
9065 };
9066 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9067 }
9068
9069 #[test]
9070 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9071 let registry = Arc::new(Registry::default());
9072 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9073 [("vault".to_string(), true), ("squatter".to_string(), true)],
9074 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9075 );
9076 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9077 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9078 provides: vec!["credentials-provider/v1".to_string()],
9079 requires: Vec::new(),
9080 must_never_reach: Vec::new(),
9081 });
9082 let frame = Frame::build(
9083 FrameType::Hello,
9084 control_flags(),
9085 0,
9086 0,
9087 77,
9088 serde_json::to_vec(&ModuleHelloBody {
9089 manifest: squatter,
9090 protocol_ver: PROTOCOL_VERSION,
9091 control_ops: None,
9092 launch_nonce: None,
9093 })
9094 .expect("HELLO serializes"),
9095 )
9096 .expect("HELLO frame builds");
9097 let response = handler
9098 .handle_control(ConnectionId::new(77), frame)
9099 .expect("reserved claim receives a typed refusal");
9100 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9101 assert_eq!(
9102 registry
9103 .active_registration_count()
9104 .expect("registry reads"),
9105 0,
9106 "a reserved capability refusal must not leave a catalog entry"
9107 );
9108 }
9109
9110 #[test]
9111 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
9112 for settlement in ["timeout", "inconclusive", "drop"] {
9113 let breakers = RouteBindBreakers::default();
9114 let RouteBindAdmission::Admitted {
9115 guard: mut old,
9116 probe: false,
9117 } = breakers.admit("prov")
9118 else {
9119 panic!("ordinary relay admitted")
9120 };
9121 let RouteBindAdmission::Admitted {
9122 guard: mut opener, ..
9123 } = breakers.admit("prov")
9124 else {
9125 panic!("second relay admitted")
9126 };
9127 assert!(
9128 !opener
9129 .record_timeout(1, Duration::ZERO)
9130 .unwrap()
9131 .reopened_after_probe
9132 );
9133 let RouteBindAdmission::Admitted {
9134 guard: mut probe,
9135 probe: true,
9136 } = breakers.admit("prov")
9137 else {
9138 panic!("one cooldown probe admitted")
9139 };
9140 match settlement {
9141 "timeout" => assert!(
9142 !old.record_timeout(1, Duration::ZERO)
9143 .unwrap()
9144 .reopened_after_probe
9145 ),
9146 "inconclusive" => old.record_inconclusive(),
9147 "drop" => drop(old),
9148 _ => unreachable!(),
9149 }
9150 assert!(
9151 matches!(
9152 breakers.admit("prov"),
9153 RouteBindAdmission::Refused {
9154 probe_in_flight: true,
9155 ..
9156 }
9157 ),
9158 "{settlement} of a pre-open relay cannot release the real probe"
9159 );
9160 assert!(
9161 probe
9162 .record_timeout(1, Duration::ZERO)
9163 .unwrap()
9164 .reopened_after_probe
9165 );
9166 assert!(matches!(
9167 breakers.admit("prov"),
9168 RouteBindAdmission::Admitted { probe: true, .. }
9169 ));
9170 }
9171 let breakers = RouteBindBreakers::default();
9172 let admit = || match breakers.admit("prov") {
9173 RouteBindAdmission::Admitted { guard, .. } => guard,
9174 _ => panic!("relay admitted"),
9175 };
9176 admit().record_timeout(1, Duration::ZERO);
9177 let mut old_probe = admit();
9178 breakers.reset_for_new_module_connection("prov");
9179 admit().record_timeout(1, Duration::ZERO);
9180 let _new_probe = admit();
9181 old_probe.record_inconclusive();
9182 assert!(matches!(
9183 breakers.admit("prov"),
9184 RouteBindAdmission::Refused {
9185 probe_in_flight: true,
9186 ..
9187 }
9188 ));
9189 }
9190
9191 #[tokio::test]
9192 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9193 for candidate in [false, true] {
9194 let registry = Arc::new(Registry::default());
9195 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9196 [("vault".to_string(), true), ("squatter".to_string(), true)],
9197 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9198 );
9199 let conn = ConnectionId::new(77);
9200 let (ctx, mut rx) = route_ctx(conn);
9201 let initial = capability_manifest("squatter", &[], &[]);
9202 if candidate {
9203 registry
9204 .register_candidate_with_control_ops(
9205 initial.clone(),
9206 PROTOCOL_VERSION,
9207 conn,
9208 module_baseline_control_ops(),
9209 )
9210 .unwrap();
9211 handler
9212 .forwarding
9213 .register_candidate_module_connection(
9214 conn,
9215 "squatter".to_string(),
9216 PROTOCOL_VERSION,
9217 manifest_concurrency(&initial),
9218 ctx.egress.clone(),
9219 )
9220 .unwrap();
9221 } else {
9222 hello_via_sink(
9223 &handler,
9224 &ctx,
9225 &mut rx,
9226 hello_frame_with_manifest(initial.clone(), 1),
9227 )
9228 .await;
9229 }
9230 let response = handler
9231 .handle_control_frame(
9232 &ctx,
9233 catalog_update_with_capabilities_frame(
9234 2,
9235 capability_manifest("squatter", &["credentials-provider/v1"], &[])
9236 .capabilities
9237 .unwrap(),
9238 ),
9239 )
9240 .await
9241 .unwrap();
9242 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9243 assert_eq!(
9244 registry
9245 .get_module_by_connection(conn)
9246 .unwrap()
9247 .unwrap()
9248 .manifest,
9249 initial
9250 );
9251 }
9252 }
9253
9254 #[test]
9255 fn server_describe_surfaces_required_capability_verdict_fields() {
9256 let registry = Arc::new(Registry::default());
9257 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9258 [
9259 ("consumer".to_string(), true),
9260 ("provider".to_string(), false),
9261 ],
9262 BTreeMap::new(),
9263 );
9264 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9265 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9266 provides: Vec::new(),
9267 requires: vec![subc_protocol::manifest::CapabilityRequirement {
9268 capability: "credentials-provider/v1".to_string(),
9269 need: subc_protocol::manifest::CapabilityNeed::Required,
9270 }],
9271 must_never_reach: Vec::new(),
9272 });
9273 let hello = Frame::build(
9274 FrameType::Hello,
9275 control_flags(),
9276 0,
9277 0,
9278 78,
9279 serde_json::to_vec(&ModuleHelloBody {
9280 manifest: consumer,
9281 protocol_ver: PROTOCOL_VERSION,
9282 control_ops: None,
9283 launch_nonce: None,
9284 })
9285 .expect("HELLO serializes"),
9286 )
9287 .expect("HELLO frame builds");
9288 handler
9289 .handle_control(ConnectionId::new(78), hello)
9290 .expect("consumer registers");
9291 let describe = Frame::build(
9292 FrameType::Request,
9293 control_flags(),
9294 0,
9295 0,
9296 79,
9297 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
9298 .expect("request serializes"),
9299 )
9300 .expect("describe frame builds");
9301 let response = handler
9302 .handle_server_describe(describe)
9303 .expect("server.describe succeeds");
9304 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
9305 let requirement = &rendered["capability_requirements"][0];
9306 assert_eq!(requirement["consumer"], "consumer");
9307 assert_eq!(requirement["verdict"], "never_provided");
9308 assert_eq!(requirement["episode_seq"], 1);
9309 assert_eq!(requirement["config_satisfiable"], false);
9310 assert_eq!(requirement["runtime_available"], false);
9311 assert!(requirement["detail"]
9312 .as_str()
9313 .expect("detail string")
9314 .contains("credentials-provider/v1"));
9315 }
9316
9317 fn deregister_required_provider(
9323 daemon_shutdown_begun: bool,
9324 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
9325 const CAPABILITY: &str = "project-identity/v1";
9326 let registry = Arc::new(Registry::default());
9327 let daemon_shutdown = DaemonShutdownFlag::default();
9328 let handler = ControlHandler::new(Arc::clone(®istry))
9329 .with_capability_config(
9330 [
9331 ("consumer".to_string(), true),
9332 ("provider".to_string(), true),
9333 ],
9334 BTreeMap::new(),
9335 )
9336 .with_daemon_shutdown_flag(daemon_shutdown.clone());
9337 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9338 consumer.capabilities = Some(CapabilityDeclarations {
9339 provides: Vec::new(),
9340 requires: vec![subc_protocol::manifest::CapabilityRequirement {
9341 capability: CAPABILITY.to_string(),
9342 need: subc_protocol::manifest::CapabilityNeed::Required,
9343 }],
9344 must_never_reach: Vec::new(),
9345 });
9346 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
9347 let provider_conn = ConnectionId::new(182);
9348 for (conn, manifest, corr) in [
9349 (ConnectionId::new(181), consumer, 181),
9350 (provider_conn, provider, 182),
9351 ] {
9352 let hello = handler
9353 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
9354 .expect("module registers");
9355 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9356 }
9357 assert_eq!(
9358 handler.capability_evaluator.verdict("consumer", CAPABILITY),
9359 Some(CapabilityVerdict::Provided),
9360 "the registered provider must satisfy the requirement first"
9361 );
9362 if daemon_shutdown_begun {
9363 daemon_shutdown.set_for_test();
9364 }
9365
9366 let capture = EventCapture::default();
9367 let _subscriber =
9368 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9369 handler
9370 .cleanup_connection(provider_conn)
9371 .expect("provider deregisters");
9372 let events = capture
9373 .events()
9374 .into_iter()
9375 .filter(|event| {
9376 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
9377 })
9378 .collect();
9379 (
9380 events,
9381 handler.capability_evaluator.verdict("consumer", CAPABILITY),
9382 )
9383 }
9384
9385 #[test]
9386 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
9387 let (events, verdict) = deregister_required_provider(true);
9388 assert_eq!(
9389 verdict,
9390 Some(CapabilityVerdict::NeverProvided),
9391 "shutdown must not change the verdict route.open refusals read"
9392 );
9393 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9394 let event = &events[0];
9395 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
9396 assert_eq!(
9397 event.fields.get("cause").map(String::as_str),
9398 Some("\"daemon_shutdown\""),
9399 "{event:?}"
9400 );
9401 assert_eq!(
9402 event.fields.get("verdict").map(String::as_str),
9403 Some("\"never_provided\""),
9404 "{event:?}"
9405 );
9406 }
9407
9408 #[test]
9409 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
9410 let (events, verdict) = deregister_required_provider(false);
9411 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
9412 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9413 let event = &events[0];
9414 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
9415 assert_eq!(event.fields.get("cause"), None, "{event:?}");
9416 assert_eq!(
9417 event.fields.get("verdict").map(String::as_str),
9418 Some("\"never_provided\""),
9419 "{event:?}"
9420 );
9421 }
9422
9423 #[test]
9424 fn catalog_list_omits_capabilities_for_legacy_manifest() {
9425 let registry = Arc::new(Registry::default());
9426 let handler = ControlHandler::new(Arc::clone(®istry));
9427 let hello = handler
9428 .handle_control(
9429 ConnectionId::new(101),
9430 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
9431 )
9432 .expect("legacy HELLO registers");
9433 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9434
9435 let request = Frame::build(
9436 FrameType::Request,
9437 control_flags(),
9438 0,
9439 0,
9440 102,
9441 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9442 .expect("catalog request serializes"),
9443 )
9444 .expect("catalog request frame builds");
9445 let response = handler
9446 .handle_catalog_list(request, None)
9447 .expect("catalog list succeeds");
9448 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9449 assert!(
9450 body["modules"][0].get("capabilities").is_none(),
9451 "legacy manifest must retain an absent capabilities field on catalog.list"
9452 );
9453 }
9454
9455 #[test]
9456 fn hello_ack_omits_storage_when_no_storage_config() {
9457 let registry = Arc::new(Registry::default());
9458 let handler = ControlHandler::new(Arc::clone(®istry));
9459 let responses = handler
9460 .handle_control(
9461 ConnectionId::new(1),
9462 hello_frame("aft", PROTOCOL_VERSION, 7),
9463 )
9464 .unwrap();
9465 let ack = parse_ack(&responses[0]);
9466 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
9467 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
9468 }
9469
9470 #[tokio::test]
9471 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
9472 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
9473 let registry = Arc::new(Registry::default());
9474 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
9475 let responses = handler
9476 .handle_control(
9477 ConnectionId::new(1),
9478 hello_frame("aft", PROTOCOL_VERSION, 7),
9479 )
9480 .unwrap();
9481 let ack = parse_ack(&responses[0]);
9482 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
9483
9484 let described = handler
9485 .handle_control_frame(
9486 &route_ctx(ConnectionId::new(2)).0,
9487 Frame::build(
9488 FrameType::Request,
9489 control_flags(),
9490 0,
9491 0,
9492 9,
9493 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9494 )
9495 .unwrap(),
9496 )
9497 .await
9498 .unwrap();
9499 let ClientControlResponse::ServerDescribe { machine_id, .. } =
9500 serde_json::from_slice(&described[0].body).unwrap()
9501 else {
9502 panic!("server.describe answered with another shape");
9503 };
9504 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
9505 }
9506
9507 #[test]
9508 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
9509 let registry = Arc::new(Registry::default());
9512 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
9513 crate::daemon_config::StorageConfig::Sqlite {
9514 data_home: std::path::PathBuf::from("/data"),
9515 },
9516 ));
9517
9518 let responses = handler
9519 .handle_control(
9520 ConnectionId::new(1),
9521 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
9522 )
9523 .unwrap();
9524 let ack = parse_ack(&responses[0]);
9525 assert_eq!(
9526 ack.storage,
9527 Some(serde_json::json!({
9528 "module_id": "alfonso-routing",
9529 "storage_namespace": "default",
9530 "isolation": { "kind": "module" },
9531 "backend": {
9532 "backend": "sqlite",
9533 "path": "/data/cortexkit/alfonso-routing/store.db"
9534 }
9535 })),
9536 "the delivered descriptor is the module's own sqlite store path"
9537 );
9538 }
9539
9540 #[test]
9541 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
9542 let registry = Arc::new(Registry::default());
9543 let handler = ControlHandler::new(Arc::clone(®istry));
9544 let conn = ConnectionId::new(1);
9545 let responses = handler
9546 .handle_control(
9547 conn,
9548 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9549 )
9550 .unwrap();
9551 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9552 let registration = registry.get_module("aft").unwrap().unwrap();
9553 assert_eq!(registration.control_ops, module_baseline_control_ops());
9554
9555 let frame =
9556 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
9557 assert!(handler
9558 .guard_module_control_op(&frame, "aft", "route.bind")
9559 .unwrap()
9560 .is_none());
9561 let error = handler
9562 .guard_module_control_op(&frame, "aft", "test.synthetic")
9563 .unwrap()
9564 .expect("synthetic ungranted op should be rejected");
9565 assert_eq!(error.header.ty, FrameType::Error);
9566 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
9567 }
9568
9569 #[test]
9570 fn hello_control_ops_some_adds_optional_grants() {
9571 let registry = Arc::new(Registry::default());
9572 let handler = ControlHandler::new(Arc::clone(®istry));
9573 handler
9574 .handle_control(
9575 ConnectionId::new(1),
9576 hello_frame_with_control_ops(
9577 "aft",
9578 PROTOCOL_VERSION,
9579 7,
9580 Some(vec![
9581 "future.synthetic".to_string(),
9582 "route.bind".to_string(),
9583 ]),
9584 ),
9585 )
9586 .unwrap();
9587 let registration = registry.get_module("aft").unwrap().unwrap();
9588 assert_eq!(
9589 registration.control_ops,
9590 vec![
9591 "route.bind".to_string(),
9592 "route.status".to_string(),
9593 "future.synthetic".to_string(),
9594 ]
9595 );
9596 let frame =
9597 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9598 assert!(handler
9599 .guard_module_control_op(&frame, "aft", "future.synthetic")
9600 .unwrap()
9601 .is_none());
9602 }
9603
9604 #[tokio::test]
9605 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9606 let registry = Arc::new(Registry::default());
9607 let forwarding = Arc::new(ForwardingTable::default());
9608 let handler =
9609 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9610 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9611 hello_via_sink(
9612 &handler,
9613 &module_ctx,
9614 &mut module_rx,
9615 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9616 )
9617 .await;
9618
9619 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9620 let responses = handler
9621 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9622 .await
9623 .unwrap();
9624 assert_eq!(responses.len(), 1);
9625 assert_eq!(responses[0].header.ty, FrameType::Error);
9626 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9627 assert!(module_rx.try_recv().is_err());
9628 }
9629
9630 #[tokio::test]
9631 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9632 let registry = Arc::new(Registry::default());
9633 let forwarding = Arc::new(ForwardingTable::default());
9634 let handler =
9635 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9636 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9637 hello_via_sink(
9638 &handler,
9639 &module_ctx,
9640 &mut module_rx,
9641 hello_frame_with_control_ops(
9642 "aft",
9643 PROTOCOL_VERSION,
9644 7,
9645 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9646 ),
9647 )
9648 .await;
9649
9650 let project_root = unique_project_root("demux");
9651 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9652 let route_handler = handler.clone();
9653 let route_task = tokio::spawn(async move {
9654 route_handler
9655 .handle_control_frame(
9656 &route_client_ctx,
9657 route_open_frame(100, "aft", project_root),
9658 )
9659 .await
9660 .unwrap()
9661 });
9662 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9663 .await
9664 .unwrap()
9665 .unwrap();
9666 assert!(matches!(
9667 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
9668 ModuleControlRequest::RouteBind { .. }
9669 ));
9670
9671 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
9672 let health_handler = handler.clone();
9673 let health_task = tokio::spawn(async move {
9674 health_handler
9675 .handle_control_frame(
9676 &health_client_ctx,
9677 supervisor_health_probe_frame(101, "aft"),
9678 )
9679 .await
9680 .unwrap()
9681 });
9682 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9683 .await
9684 .unwrap()
9685 .unwrap();
9686 assert_eq!(
9687 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9688 ModuleControlRequest::HealthCheck {}
9689 );
9690
9691 handler
9692 .handle_control_frame(
9693 &module_ctx,
9694 health_response(health_frame.header.corr, HealthStatus::Degraded),
9695 )
9696 .await
9697 .unwrap();
9698 let health_response = health_task.await.unwrap();
9699 assert_eq!(health_response.len(), 1);
9700 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9701 ClientControlResponse::SupervisorHealthProbe {
9702 module_id,
9703 status,
9704 detail,
9705 metrics,
9706 } => {
9707 assert_eq!(module_id, "aft");
9708 assert_eq!(status, HealthStatus::Degraded);
9709 assert_eq!(detail.as_deref(), Some("warming"));
9710 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
9711 }
9712 other => panic!("unexpected health response: {other:?}"),
9713 }
9714
9715 handler
9716 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9717 .await
9718 .unwrap();
9719 let route_response = route_task.await.unwrap();
9720 assert!(route_response.is_empty());
9721 let published = route_client_rx.recv().await.unwrap();
9722 assert!(matches!(
9723 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9724 ClientControlResponse::RouteOpen { .. }
9725 ));
9726 }
9727
9728 async fn relay_route_open(
9733 handler: &ControlHandler,
9734 client_connection: ConnectionId,
9735 client_egress: &FrameSink,
9736 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9737 corr: u64,
9738 module_id: &str,
9739 project_root_label: &str,
9740 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
9741 let ctx = RouteCtx {
9742 connection_id: client_connection,
9743 egress: client_egress.clone(),
9744 };
9745 let handler = handler.clone();
9746 let project_root = unique_project_root(project_root_label);
9747 let module_id = module_id.to_string();
9748 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
9749 let task = tokio::spawn(async move {
9750 let _guard = tracing::dispatcher::set_default(&dispatch);
9751 handler
9752 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
9753 .await
9754 .unwrap()
9755 });
9756 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
9757 .await
9758 .expect("module receives the relayed route.bind")
9759 .expect("module egress is open");
9760 (task, bind.frame)
9761 }
9762
9763 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
9764 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
9765 ModuleControlRequest::RouteBind {
9766 route_channel,
9767 epoch,
9768 ..
9769 } => (route_channel, epoch),
9770 other => panic!("expected a route.bind request, got {other:?}"),
9771 }
9772 }
9773
9774 fn published_route(frame: &Frame) -> (u16, u32) {
9775 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
9776 ClientControlResponse::RouteOpen {
9777 route_channel,
9778 route_epoch,
9779 } => (route_channel, route_epoch),
9780 other => panic!("expected a route.open response, got {other:?}"),
9781 }
9782 }
9783
9784 #[tokio::test]
9804 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
9805 let registry = Arc::new(Registry::default());
9806 let forwarding = Arc::new(ForwardingTable::default());
9807 let handler =
9808 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9809
9810 let module_connection = ConnectionId::new(30);
9811 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9812 hello_via_sink(
9813 &handler,
9814 &module_ctx,
9815 &mut module_rx,
9816 hello_frame("aft", PROTOCOL_VERSION, 7),
9817 )
9818 .await;
9819
9820 let dying_client = ConnectionId::new(31);
9821 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
9822
9823 let (first_task, first_bind) = relay_route_open(
9826 &handler,
9827 dying_client,
9828 &dying_ctx.egress,
9829 &mut module_rx,
9830 100,
9831 "aft",
9832 "closing-first",
9833 )
9834 .await;
9835 handler
9836 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
9837 .await
9838 .unwrap();
9839 assert!(first_task.await.unwrap().is_empty());
9840 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
9841
9842 let (second_task, second_bind) = relay_route_open(
9844 &handler,
9845 dying_client,
9846 &dying_ctx.egress,
9847 &mut module_rx,
9848 101,
9849 "aft",
9850 "closing-second",
9851 )
9852 .await;
9853 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9854
9855 assert!(forwarding
9858 .escalate_client_delivery_failure(
9859 dying_client,
9860 first_channel,
9861 first_epoch,
9862 CloseReason::new(
9863 "module_to_client_delivery_failed",
9864 "client egress refused a module frame",
9865 ),
9866 crate::forwarding::UndeliveredFrame {
9867 module_id: None,
9868 sink: &dying_ctx.egress,
9869 },
9870 )
9871 .unwrap());
9872 assert!(!dying_ctx.egress.is_closed());
9873
9874 let ack = handler
9876 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9877 .await;
9878 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9879 if module_loop_error.is_some() {
9880 handler.cleanup_connection(module_connection).unwrap();
9884 }
9885 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9889 .await
9890 .ok()
9891 .flatten();
9892
9893 assert!(
9895 registry
9896 .get_module_by_connection(module_connection)
9897 .unwrap()
9898 .is_some(),
9899 "one client's closing connection ended the shared module connection: \
9900 {module_loop_error:?}"
9901 );
9902 let cotenant = ConnectionId::new(32);
9904 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9905 let (cotenant_task, cotenant_bind) = relay_route_open(
9906 &handler,
9907 cotenant,
9908 &cotenant_ctx.egress,
9909 &mut module_rx,
9910 102,
9911 "aft",
9912 "closing-cotenant",
9913 )
9914 .await;
9915 handler
9916 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9917 .await
9918 .unwrap();
9919 assert!(cotenant_task.await.unwrap().is_empty());
9920 let (cotenant_channel, cotenant_epoch) =
9921 published_route(&cotenant_rx.recv().await.unwrap());
9922 assert!(matches!(
9923 forwarding
9924 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9925 .unwrap(),
9926 DataRoute::Client(DataRouteState::Bound(_))
9927 ));
9928
9929 let goodbye = post_ack_module_frame
9932 .expect("module receives a GOODBYE for the abandoned route channel");
9933 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9934 assert_eq!(goodbye.header.channel, abandoned_channel);
9935 assert_eq!(goodbye.header.epoch, abandoned_epoch);
9936
9937 assert!(dying_rx.try_recv().is_err());
9941 let second_response = second_task.await.unwrap();
9942 assert_eq!(second_response.len(), 1);
9943 assert_eq!(
9944 parse_error(&second_response[0])["code"],
9945 "target_unavailable"
9946 );
9947 }
9948
9949 async fn catalog_snapshot(
9952 handler: &ControlHandler,
9953 client_ctx: &RouteCtx,
9954 corr: u64,
9955 ) -> (u64, Vec<(String, bool)>) {
9956 let body =
9957 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
9958 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
9959 let reply = handler
9960 .handle_control_frame(client_ctx, request)
9961 .await
9962 .unwrap()
9963 .pop()
9964 .unwrap();
9965 let ClientControlResponse::CatalogList {
9966 generation,
9967 modules,
9968 ..
9969 } = serde_json::from_slice(&reply.body).unwrap()
9970 else {
9971 panic!("not a catalog.list reply");
9972 };
9973 (
9974 generation,
9975 modules
9976 .into_iter()
9977 .map(|entry| (entry.module_id, entry.ready))
9978 .collect(),
9979 )
9980 }
9981
9982 fn hello_held_before_forwarding() -> (
9988 ControlHandler,
9989 Arc<std::sync::Barrier>,
9990 std::thread::JoinHandle<Vec<Frame>>,
9991 RouteCtx,
9992 mpsc::Receiver<crate::router::OutboundFrame>,
9993 ) {
9994 let hold = Arc::new(std::sync::Barrier::new(2));
9995 let handler = ControlHandler::with_forwarding(
9996 Arc::new(Registry::default()),
9997 Arc::new(ForwardingTable::default()),
9998 )
9999 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10000 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10001 let hello = {
10002 let handler = handler.clone();
10003 let module_ctx = module_ctx.clone();
10004 std::thread::spawn(move || {
10005 handler
10006 .handle_hello(
10007 module_ctx.connection_id,
10008 Some(module_ctx.egress.clone()),
10009 hello_frame("aft", PROTOCOL_VERSION, 7),
10010 )
10011 .unwrap()
10012 })
10013 };
10014 (handler, hold, hello, module_ctx, module_rx)
10015 }
10016
10017 #[tokio::test]
10026 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10027 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10028 let client = ConnectionId::new(41);
10029 let (client_ctx, mut client_rx) = route_ctx(client);
10030
10031 hold.wait();
10032 assert!(handler.registry.get_module("aft").unwrap().is_some());
10034 assert!(!handler
10035 .forwarding
10036 .has_live_module_connection("aft")
10037 .unwrap());
10038 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10039 assert!(
10040 listed.is_empty(),
10041 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10042 );
10043 let refused = handler
10044 .handle_control_frame(
10045 &client_ctx,
10046 route_open_frame(301, "aft", unique_project_root("listed-window")),
10047 )
10048 .await
10049 .unwrap();
10050 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10051 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10052 let duplicate = handler
10053 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10054 .await
10055 .unwrap();
10056 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10057
10058 hold.wait();
10059 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10060 let ack = module_rx.try_recv().unwrap().frame;
10061 assert_eq!(ack.header.ty, FrameType::HelloAck);
10062 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10063 assert_eq!(listed, vec![("aft".to_string(), true)]);
10064
10065 let (open_task, bind) = relay_route_open(
10066 &handler,
10067 client,
10068 &client_ctx.egress,
10069 &mut module_rx,
10070 303,
10071 "aft",
10072 "listed-after",
10073 )
10074 .await;
10075 assert!(handler
10076 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10077 .await
10078 .unwrap()
10079 .is_empty());
10080 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10081 published_route(&client_rx.try_recv().unwrap().frame);
10082 }
10083
10084 #[tokio::test]
10091 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
10092 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
10093 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10094 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10095
10096 hold.wait();
10097 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
10098 assert!(listed.is_empty());
10099 assert_eq!(
10100 during, before,
10101 "the generation moved while the module was not listed"
10102 );
10103
10104 hold.wait();
10105 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10106 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10107 assert_eq!(listed, vec![("aft".to_string(), true)]);
10108 assert_eq!(
10109 after,
10110 before + 1,
10111 "the module appeared without exactly one generation bump"
10112 );
10113 }
10114
10115 #[tokio::test]
10120 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10121 let registry = Arc::new(Registry::default());
10122 let handler = ControlHandler::with_forwarding(
10123 Arc::clone(®istry),
10124 Arc::new(ForwardingTable::default()),
10125 );
10126 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10127 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10128 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10129 drop(module_rx);
10130
10131 let replies = handler
10132 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10133 .await
10134 .unwrap();
10135 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10136 assert!(registry.get_module("aft").unwrap().is_none());
10137 assert!(registry
10138 .get_module_by_connection(module_ctx.connection_id)
10139 .unwrap()
10140 .is_none());
10141 assert_eq!(
10142 catalog_snapshot(&handler, &client_ctx, 301).await,
10143 (before, Vec::new())
10144 );
10145 }
10146
10147 async fn route_open_held_after_bind_send() -> (
10151 ControlHandler,
10152 Arc<tokio::sync::Barrier>,
10153 RouteCtx,
10154 mpsc::Receiver<crate::router::OutboundFrame>,
10155 ) {
10156 let hold = Arc::new(tokio::sync::Barrier::new(2));
10157 let handler = ControlHandler::with_forwarding(
10158 Arc::new(Registry::default()),
10159 Arc::new(ForwardingTable::default()),
10160 )
10161 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10162 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10163 hello_via_sink(
10164 &handler,
10165 &module_ctx,
10166 &mut module_rx,
10167 hello_frame("aft", PROTOCOL_VERSION, 7),
10168 )
10169 .await;
10170 (handler, hold, module_ctx, module_rx)
10171 }
10172
10173 #[tokio::test]
10180 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10181 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10182 let client = ConnectionId::new(41);
10183 let (client_ctx, mut client_rx) = route_ctx(client);
10184
10185 let (open_task, bind) = relay_route_open(
10186 &handler,
10187 client,
10188 &client_ctx.egress,
10189 &mut module_rx,
10190 200,
10191 "aft",
10192 "acked-before-mark",
10193 )
10194 .await;
10195 let (module_channel, module_epoch) = route_bind_channel(&bind);
10196 hold.wait().await;
10198
10199 assert!(handler
10201 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10202 .await
10203 .unwrap()
10204 .is_empty());
10205 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10206
10207 hold.wait().await;
10208 assert!(
10209 open_task.await.unwrap().is_empty(),
10210 "route.open answers accepted"
10211 );
10212
10213 match module_rx.try_recv() {
10216 Err(mpsc::error::TryRecvError::Empty) => {}
10217 Ok(outbound) => panic!(
10218 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10219 outbound.frame.header.ty,
10220 outbound.frame.header.channel,
10221 outbound.frame.header.epoch
10222 ),
10223 Err(err) => panic!("module egress closed: {err}"),
10224 }
10225
10226 let router = Router::with_control_handler(Arc::new(handler.clone()));
10229 let request = Frame::build(
10230 FrameType::Request,
10231 Flags::new(true, Priority::Interactive, false),
10232 client_channel,
10233 client_epoch,
10234 201,
10235 b"first request".to_vec(),
10236 )
10237 .unwrap();
10238 router
10239 .route_for_connection(&client_ctx, request)
10240 .await
10241 .unwrap();
10242 let delivered = module_rx.try_recv().unwrap().frame;
10243 assert_eq!(delivered.header.ty, FrameType::Request);
10244 assert_eq!(
10245 (delivered.header.channel, delivered.header.epoch),
10246 (module_channel, module_epoch)
10247 );
10248 assert_eq!(delivered.body, b"first request");
10249 }
10250
10251 #[tokio::test]
10257 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10258 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10259 let client = ConnectionId::new(42);
10260 let (client_ctx, _client_rx) = route_ctx(client);
10261
10262 let (open_task, bind) = relay_route_open(
10263 &handler,
10264 client,
10265 &client_ctx.egress,
10266 &mut module_rx,
10267 300,
10268 "aft",
10269 "released-before-mark",
10270 )
10271 .await;
10272 let (module_channel, module_epoch) = route_bind_channel(&bind);
10273 hold.wait().await;
10274
10275 handler.cleanup_connection(client).unwrap();
10276 assert!(
10277 module_rx.try_recv().is_err(),
10278 "client teardown must not GOODBYE a bind route.open is still queueing"
10279 );
10280
10281 hold.wait().await;
10282 let response = open_task.await.unwrap();
10283 assert_eq!(response.len(), 1);
10284 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10285
10286 let goodbye = module_rx
10287 .try_recv()
10288 .expect("module is told to drop the binding the daemon abandoned")
10289 .frame;
10290 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10291 assert_eq!(
10292 (goodbye.header.channel, goodbye.header.epoch),
10293 (module_channel, module_epoch)
10294 );
10295 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10296 }
10297
10298 #[tokio::test]
10303 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
10304 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10305 let client = ConnectionId::new(43);
10306 let (client_ctx, client_rx) = route_ctx(client);
10307
10308 let (open_task, bind) = relay_route_open(
10309 &handler,
10310 client,
10311 &client_ctx.egress,
10312 &mut module_rx,
10313 400,
10314 "aft",
10315 "gone-client-before-mark",
10316 )
10317 .await;
10318 let (module_channel, module_epoch) = route_bind_channel(&bind);
10319 hold.wait().await;
10320
10321 drop(client_rx);
10322 assert!(client_ctx.egress.is_closed());
10323 handler
10324 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10325 .await
10326 .unwrap();
10327
10328 hold.wait().await;
10329 let response = open_task.await.unwrap();
10330 assert_eq!(response.len(), 1);
10331 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10332
10333 let goodbye = module_rx
10334 .try_recv()
10335 .expect("module is told to drop the binding it acked")
10336 .frame;
10337 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10338 assert_eq!(
10339 (goodbye.header.channel, goodbye.header.epoch),
10340 (module_channel, module_epoch)
10341 );
10342 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10343 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
10344 }
10345
10346 #[test]
10353 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
10354 let handler = ControlHandler::default();
10355 let module_connection = ConnectionId::new(30);
10356 let client_connection = ConnectionId::new(31);
10357
10358 handler
10359 .refuse_to_end_module_connection_for_a_client(
10360 module_connection,
10361 77,
10362 ForwardingError::ConnectionClosing {
10363 connection_id: client_connection,
10364 },
10365 )
10366 .expect("a closing client must never end the module connection");
10367
10368 assert!(matches!(
10369 handler.refuse_to_end_module_connection_for_a_client(
10370 module_connection,
10371 78,
10372 ForwardingError::ConnectionClosing {
10373 connection_id: module_connection,
10374 },
10375 ),
10376 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
10377 connection_id
10378 })) if connection_id == module_connection
10379 ));
10380 assert!(matches!(
10381 handler.refuse_to_end_module_connection_for_a_client(
10382 module_connection,
10383 79,
10384 ForwardingError::Poisoned,
10385 ),
10386 Err(RouterError::Forwarding(ForwardingError::Poisoned))
10387 ));
10388 assert!(matches!(
10389 handler.refuse_to_end_module_connection_for_a_client(
10390 module_connection,
10391 80,
10392 ForwardingError::StaleModuleEndpoint,
10393 ),
10394 Err(RouterError::Forwarding(
10395 ForwardingError::StaleModuleEndpoint
10396 ))
10397 ));
10398 }
10399
10400 #[tokio::test]
10413 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
10414 let registry = Arc::new(Registry::default());
10415 let forwarding = Arc::new(ForwardingTable::default());
10416 let supervisor = SupervisorHandle::new();
10417 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10418 let handler =
10419 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10420 .with_supervisor(supervisor);
10421
10422 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10423 hello_via_sink(
10424 &handler,
10425 &target_ctx,
10426 &mut target_rx,
10427 hello_frame("target", PROTOCOL_VERSION, 1),
10428 )
10429 .await;
10430
10431 let wrong_nonce = handler
10435 .handle_control_frame(
10436 &route_ctx(ConnectionId::new(91)).0,
10437 route_open_frame_with_admission_facts(
10438 20,
10439 "target",
10440 unique_project_root("admission-facts"),
10441 Some(subc_control::ConsumerIdentity {
10442 module_id: "fed".to_string(),
10443 launch_nonce: "not-the-real-nonce".to_string(),
10444 }),
10445 None,
10446 ),
10447 )
10448 .await
10449 .unwrap();
10450 assert_eq!(
10451 parse_error(&wrong_nonce[0])["code"],
10452 "bad_consumer_identity",
10453 "a mismatched launch nonce must be refused, not stamped Reserved"
10454 );
10455
10456 let never_spawned = handler
10460 .handle_control_frame(
10461 &route_ctx(ConnectionId::new(92)).0,
10462 route_open_frame_with_admission_facts(
10463 21,
10464 "target",
10465 unique_project_root("admission-facts"),
10466 Some(subc_control::ConsumerIdentity {
10467 module_id: "never-spawned".to_string(),
10468 launch_nonce: "any-nonce".to_string(),
10469 }),
10470 None,
10471 ),
10472 )
10473 .await
10474 .unwrap();
10475 assert_eq!(
10476 parse_error(&never_spawned[0])["code"],
10477 "bad_consumer_identity",
10478 "an unspawned module_id must be refused rather than accepted for lack of a record"
10479 );
10480 }
10481
10482 #[tokio::test]
10505 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
10506 let registry = Arc::new(Registry::default());
10507 let forwarding = Arc::new(ForwardingTable::default());
10508 let supervisor = SupervisorHandle::new();
10509 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10510 let handler =
10511 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10512 .with_supervisor(supervisor);
10513
10514 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
10515 hello_via_sink(
10516 &handler,
10517 &target_ctx,
10518 &mut target_rx,
10519 hello_frame("target", PROTOCOL_VERSION, 1),
10520 )
10521 .await;
10522
10523 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
10524 let route_handler = handler.clone();
10525 let route_task = tokio::spawn(async move {
10526 route_handler
10527 .handle_control_frame(
10528 &client_ctx,
10529 route_open_frame_with_admission_facts(
10530 30,
10531 "target",
10532 unique_project_root("admission-facts"),
10533 Some(subc_control::ConsumerIdentity {
10534 module_id: "fed".to_string(),
10535 launch_nonce: "fed-nonce".to_string(),
10536 }),
10537 None,
10538 ),
10539 )
10540 .await
10541 .unwrap()
10542 });
10543
10544 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10551 .await
10552 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
10553 .expect("module control channel closed before route.bind");
10554 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10555 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10556 panic!("expected route.bind")
10557 };
10558 assert_eq!(
10559 principal,
10560 Some(Principal::Reserved {
10561 module_id: "fed".to_string()
10562 }),
10563 "a correctly attested consumer must be stamped Reserved for its own id"
10564 );
10565
10566 handler
10567 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10568 .await
10569 .unwrap();
10570 assert!(route_task.await.unwrap().is_empty());
10571 assert!(
10572 matches!(
10573 serde_json::from_slice::<ClientControlResponse>(
10574 &client_rx.recv().await.unwrap().body
10575 )
10576 .unwrap(),
10577 ClientControlResponse::RouteOpen { .. }
10578 ),
10579 "the route must actually open, not merely avoid an error"
10580 );
10581 }
10582
10583 #[tokio::test(start_paused = true)]
10584 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10585 let registry = Arc::new(Registry::default());
10586 let forwarding = Arc::new(ForwardingTable::default());
10587 let supervisor = SupervisorHandle::new();
10588 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10589 let handler =
10590 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10591 .with_supervisor(supervisor);
10592
10593 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10594 hello_via_sink(
10595 &handler,
10596 &target_ctx,
10597 &mut target_rx,
10598 hello_frame("target", PROTOCOL_VERSION, 1),
10599 )
10600 .await;
10601
10602 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10603 let direct_handler = handler.clone();
10604 let direct_open = tokio::spawn(async move {
10605 direct_handler
10606 .handle_control_frame(
10607 &direct_ctx,
10608 route_open_frame(2, "target", unique_project_root("route-census-direct")),
10609 )
10610 .await
10611 .unwrap()
10612 });
10613 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10614 .await
10615 .expect("no direct route.bind within 5s")
10616 .expect("target control channel closed before direct route.bind");
10617 handler
10618 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10619 .await
10620 .unwrap();
10621 assert!(direct_open.await.unwrap().is_empty());
10622 let _ = direct_rx.recv().await.unwrap();
10623
10624 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10625 let reserved_handler = handler.clone();
10626 let reserved_open = tokio::spawn(async move {
10627 reserved_handler
10628 .handle_control_frame(
10629 &reserved_ctx,
10630 route_open_frame_with_admission_facts(
10631 3,
10632 "target",
10633 unique_project_root("admission-facts"),
10634 Some(ConsumerIdentity {
10635 module_id: "fed".to_string(),
10636 launch_nonce: "fed-nonce".to_string(),
10637 }),
10638 None,
10639 ),
10640 )
10641 .await
10642 .unwrap()
10643 });
10644 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10645 .await
10646 .expect("no reserved route.bind within 5s")
10647 .expect("target control channel closed before reserved route.bind");
10648 handler
10649 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10650 .await
10651 .unwrap();
10652 assert!(reserved_open.await.unwrap().is_empty());
10653 let _ = reserved_rx.recv().await.unwrap();
10654
10655 forwarding
10656 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10657 .unwrap();
10658 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
10659 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
10660 module_id: Some("target".to_string()),
10661 })
10662 .unwrap();
10663 let census_frame =
10664 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
10665 let response = handler
10666 .handle_control_frame(&census_ctx, census_frame)
10667 .await
10668 .unwrap()
10669 .pop()
10670 .unwrap();
10671 let actual: Value = serde_json::from_slice(&response.body).unwrap();
10672 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
10673 assert!(matches!(
10674 decoded,
10675 ClientControlResponse::SupervisorRoutes { .. }
10676 ));
10677 let routes = actual["modules"][0]["routes"].as_array().unwrap();
10678 assert_eq!(routes.len(), 2);
10679 assert!(routes.iter().all(|route| route["draining"] == true));
10680 assert!(
10683 routes.iter().all(|route| route["drain_reason"] == "reload"),
10684 "draining routes must name the drain's reason: {routes:?}"
10685 );
10686 assert!(routes.iter().any(|route| {
10687 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
10688 }));
10689 assert!(routes.iter().any(|route| {
10690 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
10691 }));
10692
10693 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10694 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
10695 if std::env::var_os("UPDATE_GOLDEN").is_some() {
10696 std::fs::write(
10697 &golden_path,
10698 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
10699 )
10700 .unwrap();
10701 }
10702 let expected: Value =
10703 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
10704 assert_eq!(actual, expected);
10705 }
10706
10707 async fn query_live_roots(
10708 handler: &ControlHandler,
10709 module_ctx: &RouteCtx,
10710 ) -> ModuleControlResponseToModule {
10711 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
10712 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
10713 let response = handler
10714 .handle_control_frame(module_ctx, frame)
10715 .await
10716 .unwrap()
10717 .pop()
10718 .unwrap();
10719 serde_json::from_slice(&response.body).unwrap()
10720 }
10721
10722 #[tokio::test(start_paused = true)]
10723 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
10724 let registry = Arc::new(Registry::default());
10725 let forwarding = Arc::new(ForwardingTable::default());
10726 let handler = ControlHandler::with_forwarding(registry, forwarding);
10727 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
10728 hello_via_sink(
10729 &handler,
10730 &target_ctx,
10731 &mut target_rx,
10732 hello_frame("target", PROTOCOL_VERSION, 1),
10733 )
10734 .await;
10735 let root = unique_project_root("live-roots-known");
10736 let path = ProjectRootId::from_path_allowing_missing(root.path())
10737 .unwrap()
10738 .as_path()
10739 .to_path_buf();
10740 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
10741 let open_handler = handler.clone();
10742 let opened = tokio::spawn(async move {
10743 open_handler
10744 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
10745 .await
10746 .unwrap()
10747 });
10748 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10749 .await
10750 .unwrap()
10751 .unwrap();
10752 handler
10753 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
10754 .await
10755 .unwrap();
10756 assert!(opened.await.unwrap().is_empty());
10757 let _ = client_rx.recv().await.unwrap();
10758
10759 let root = unique_project_root("live-roots-pending");
10760 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
10761 .unwrap()
10762 .as_path()
10763 .to_path_buf();
10764 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
10765 let open_handler = handler.clone();
10766 let pending = tokio::spawn(async move {
10767 open_handler
10768 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
10769 .await
10770 .unwrap()
10771 });
10772 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10773 .await
10774 .unwrap()
10775 .unwrap();
10776 let actual = query_live_roots(&handler, &target_ctx).await;
10777 let ModuleControlResponseToModule::LiveRoots {
10778 roots,
10779 unknown_root_bindings,
10780 total_bindings,
10781 } = actual
10782 else {
10783 panic!("expected live roots")
10784 };
10785 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
10786 assert_eq!(unknown_root_bindings, 0);
10787 assert_eq!(
10788 roots.len(),
10789 2,
10790 "root-known arm must retain each canonical root"
10791 );
10792 assert_eq!(
10793 total_bindings,
10794 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10795 );
10796 let counts = roots
10797 .iter()
10798 .map(|root| (root.project_root.clone(), root.bound, root.pending))
10799 .collect::<Vec<_>>();
10800 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
10801 expected.sort_by(|a, b| a.0.cmp(&b.0));
10802 assert_eq!(
10803 counts, expected,
10804 "roots must sort by path and count pending separately"
10805 );
10806 handler
10807 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
10808 .await
10809 .unwrap();
10810 assert!(pending.await.unwrap().is_empty());
10811 }
10812
10813 #[tokio::test(start_paused = true)]
10814 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
10815 let forwarding = Arc::new(ForwardingTable::default());
10816 let handler =
10817 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10818 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
10819 hello_via_sink(
10820 &handler,
10821 &target_ctx,
10822 &mut target_rx,
10823 hello_frame("target", PROTOCOL_VERSION, 1),
10824 )
10825 .await;
10826 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
10827 let pending = forwarding
10828 .begin_route_bind_relay_for_test(
10829 client_ctx.connection_id,
10830 client_ctx.egress.clone(),
10831 2,
10832 "target",
10833 )
10834 .unwrap();
10835 forwarding
10836 .complete_pending_relay(
10837 target_ctx.connection_id,
10838 pending.corr,
10839 RouteBindRelayOutcome::Accepted,
10840 )
10841 .unwrap();
10842 let actual = query_live_roots(&handler, &target_ctx).await;
10843 let ModuleControlResponseToModule::LiveRoots {
10844 roots,
10845 unknown_root_bindings,
10846 total_bindings,
10847 } = actual
10848 else {
10849 panic!("expected live roots")
10850 };
10851 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
10852 assert_eq!(
10853 unknown_root_bindings, 1,
10854 "unknown-root arm must not read as no bindings"
10855 );
10856 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
10857 assert_eq!(
10858 total_bindings,
10859 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10860 );
10861 }
10862
10863 #[tokio::test(start_paused = true)]
10871 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
10872 let forwarding = Arc::new(ForwardingTable::default());
10873 let handler =
10874 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10875 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
10876 let replies = handler
10877 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
10878 .await
10879 .unwrap();
10880 let queued_by_hello = module_rx.len();
10881
10882 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
10883 let open_handler = handler.clone();
10884 let open = tokio::spawn(async move {
10885 open_handler
10886 .handle_control_frame(
10887 &client_ctx,
10888 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
10889 )
10890 .await
10891 .unwrap()
10892 });
10893 let mut spins = 0;
10895 while module_rx.len() == queued_by_hello {
10896 spins += 1;
10897 assert!(spins < 10_000, "route.open never queued a route.bind");
10898 tokio::task::yield_now().await;
10899 }
10900
10901 for reply in replies {
10903 module_ctx.egress.send(reply).await.unwrap();
10904 }
10905
10906 let first = module_rx.recv().await.unwrap().frame;
10907 assert_eq!(
10908 first.header.ty,
10909 FrameType::HelloAck,
10910 "the first frame a registering module reads must be its HELLO_ACK"
10911 );
10912 assert_eq!(first.header.corr, 7);
10913 let second = module_rx.recv().await.unwrap().frame;
10914 assert_eq!(second.header.ty, FrameType::Request);
10915 assert!(
10916 matches!(
10917 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
10918 ModuleControlRequest::RouteBind { .. }
10919 ),
10920 "the route.bind follows the ack"
10921 );
10922 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
10923
10924 handler
10925 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
10926 .await
10927 .unwrap();
10928 assert!(open.await.unwrap().is_empty());
10929 let _ = client_rx.recv().await.unwrap();
10930 }
10931
10932 #[tokio::test(start_paused = true)]
10933 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
10934 let handler = ControlHandler::with_forwarding(
10935 Arc::new(Registry::default()),
10936 Arc::new(ForwardingTable::default()),
10937 );
10938 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
10939 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
10940 hello_via_sink(
10941 &handler,
10942 &first_ctx,
10943 &mut first_rx,
10944 hello_frame("first", PROTOCOL_VERSION, 1),
10945 )
10946 .await;
10947 hello_via_sink(
10948 &handler,
10949 &second_ctx,
10950 &mut second_rx,
10951 hello_frame("second", PROTOCOL_VERSION, 2),
10952 )
10953 .await;
10954 let root = unique_project_root("second-only");
10955 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
10956 let cloned = handler.clone();
10957 let open = tokio::spawn(async move {
10958 cloned
10959 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
10960 .await
10961 .unwrap()
10962 });
10963 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
10964 .await
10965 .unwrap()
10966 .unwrap();
10967 let first = query_live_roots(&handler, &first_ctx).await;
10968 let second = query_live_roots(&handler, &second_ctx).await;
10969 assert!(
10970 matches!(
10971 first,
10972 ModuleControlResponseToModule::LiveRoots {
10973 total_bindings: 0,
10974 ..
10975 }
10976 ),
10977 "cross-module scope must not expose another module's roots"
10978 );
10979 assert!(
10980 matches!(
10981 second,
10982 ModuleControlResponseToModule::LiveRoots {
10983 total_bindings: 1,
10984 ..
10985 }
10986 ),
10987 "second module must see its pending route"
10988 );
10989 handler
10990 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
10991 .await
10992 .unwrap();
10993 assert!(open.await.unwrap().is_empty());
10994 }
10995
10996 #[tokio::test(start_paused = true)]
10997 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
10998 let handler = ControlHandler::with_forwarding(
10999 Arc::new(Registry::default()),
11000 Arc::new(ForwardingTable::default()),
11001 );
11002 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
11003 hello_via_sink(
11004 &handler,
11005 &target_ctx,
11006 &mut target_rx,
11007 hello_frame("target", PROTOCOL_VERSION, 1),
11008 )
11009 .await;
11010 let actual = query_live_roots(&handler, &target_ctx).await;
11011 let ModuleControlResponseToModule::LiveRoots {
11012 roots,
11013 unknown_root_bindings,
11014 total_bindings,
11015 } = actual
11016 else {
11017 panic!("expected live roots")
11018 };
11019 assert!(roots.is_empty());
11020 assert_eq!(unknown_root_bindings, 0);
11021 assert_eq!(total_bindings, 0);
11022 assert_eq!(
11023 total_bindings,
11024 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11025 );
11026 }
11027
11028 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11035 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11036 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11037 let text = std::fs::read_to_string(&path)
11038 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11039 let vectors: Vec<(String, Value)> = text
11040 .lines()
11041 .filter(|line| !line.trim().is_empty())
11042 .map(|line| {
11043 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11044 let id = entry["corpus_id"]
11045 .as_str()
11046 .expect("every vector carries a corpus_id")
11047 .to_string();
11048 (id, entry["package"].clone())
11049 })
11050 .collect();
11051 assert_eq!(
11055 vectors.len(),
11056 3,
11057 "vendored fed corpus changed size; re-sync from subc-federation"
11058 );
11059
11060 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11075 let richest = vectors
11076 .iter()
11077 .filter_map(|(_, package)| package.as_object())
11078 .map(|object| {
11079 object
11080 .keys()
11081 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11082 .count()
11083 })
11084 .max()
11085 .unwrap_or(0);
11086 assert!(
11087 richest >= 2,
11088 "vendored corpus no longer carries a package with unmodelled fields, \
11089 so the relay test can no longer distinguish a verbatim relay from a lossy one"
11090 );
11091
11092 vectors
11093 }
11094
11095 #[tokio::test]
11103 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
11104 for (corpus_id, package) in fed_admission_facts_vectors() {
11105 let registry = Arc::new(Registry::default());
11106 let forwarding = Arc::new(ForwardingTable::default());
11107 let supervisor = SupervisorHandle::new();
11108 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11109 let handler =
11110 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11111 .with_supervisor(supervisor)
11112 .with_admission_facts_config(
11113 Some("fed".to_string()),
11114 Some(vec!["target".to_string()]),
11115 );
11116
11117 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11118 hello_via_sink(
11119 &handler,
11120 &target_ctx,
11121 &mut target_rx,
11122 hello_frame("target", PROTOCOL_VERSION, 1),
11123 )
11124 .await;
11125
11126 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11127 let route_handler = handler.clone();
11128 let expected = package.clone();
11129 let route_task = tokio::spawn(async move {
11130 route_handler
11131 .handle_control_frame(
11132 &client_ctx,
11133 route_open_frame_with_admission_facts(
11134 20,
11135 "target",
11136 unique_project_root("admission-facts"),
11137 Some(subc_control::ConsumerIdentity {
11138 module_id: "fed".to_string(),
11139 launch_nonce: "fed-nonce".to_string(),
11140 }),
11141 Some(package),
11142 ),
11143 )
11144 .await
11145 .unwrap()
11146 });
11147
11148 let bind_frame = target_rx.recv().await.unwrap();
11149 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11150 let ModuleControlRequest::RouteBind {
11151 admission_facts, ..
11152 } = bind
11153 else {
11154 panic!("{corpus_id}: expected route.bind")
11155 };
11156 assert_eq!(
11157 admission_facts,
11158 Some(expected),
11159 "{corpus_id}: relay must not add, drop or reshape any field"
11160 );
11161
11162 handler
11163 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11164 .await
11165 .unwrap();
11166 route_task.await.unwrap();
11167 }
11168 }
11169
11170 #[tokio::test]
11171 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11172 let registry = Arc::new(Registry::default());
11173 let forwarding = Arc::new(ForwardingTable::default());
11174 let supervisor = SupervisorHandle::new();
11175 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11176 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11177 let handler =
11178 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11179 .with_supervisor(supervisor)
11180 .with_admission_facts_config(
11181 Some("fed".to_string()),
11182 Some(vec!["target".to_string()]),
11183 );
11184
11185 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11186 hello_via_sink(
11187 &handler,
11188 &target_ctx,
11189 &mut target_rx,
11190 hello_frame("target", PROTOCOL_VERSION, 1),
11191 )
11192 .await;
11193 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11194 hello_via_sink(
11195 &handler,
11196 &other_ctx,
11197 &mut other_rx,
11198 hello_frame("other", PROTOCOL_VERSION, 2),
11199 )
11200 .await;
11201
11202 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11203 let expected_facts = facts.clone();
11204 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11205 let route_handler = handler.clone();
11206 let route_task = tokio::spawn(async move {
11207 route_handler
11208 .handle_control_frame(
11209 &client_ctx,
11210 route_open_frame_with_admission_facts(
11211 10,
11212 "target",
11213 unique_project_root("admission-facts"),
11214 Some(subc_control::ConsumerIdentity {
11215 module_id: "fed".to_string(),
11216 launch_nonce: "fed-nonce".to_string(),
11217 }),
11218 Some(facts.clone()),
11219 ),
11220 )
11221 .await
11222 .unwrap()
11223 });
11224 let bind_frame = target_rx.recv().await.unwrap();
11225 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11226 let ModuleControlRequest::RouteBind {
11227 admission_facts, ..
11228 } = bind
11229 else {
11230 panic!("expected route.bind")
11231 };
11232 assert_eq!(admission_facts, Some(expected_facts));
11233 handler
11234 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11235 .await
11236 .unwrap();
11237 assert!(route_task.await.unwrap().is_empty());
11238 assert!(matches!(
11239 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11240 .unwrap(),
11241 ClientControlResponse::RouteOpen { .. }
11242 ));
11243
11244 let direct = handler
11245 .handle_control_frame(
11246 &route_ctx(ConnectionId::new(73)).0,
11247 route_open_frame_with_admission_facts(
11248 11,
11249 "target",
11250 unique_project_root("admission-facts"),
11251 None,
11252 Some(json!({"x": 1})),
11253 ),
11254 )
11255 .await
11256 .unwrap();
11257 assert_eq!(
11258 parse_error(&direct[0])["code"],
11259 "admission_facts_not_permitted"
11260 );
11261
11262 let different_reserved = handler
11263 .handle_control_frame(
11264 &route_ctx(ConnectionId::new(77)).0,
11265 route_open_frame_with_admission_facts(
11266 15,
11267 "target",
11268 unique_project_root("admission-facts"),
11269 Some(subc_control::ConsumerIdentity {
11270 module_id: "other".to_string(),
11271 launch_nonce: "other-nonce".to_string(),
11272 }),
11273 Some(json!({"x": 1})),
11274 ),
11275 )
11276 .await
11277 .unwrap();
11278 assert_eq!(
11279 parse_error(&different_reserved[0])["code"],
11280 "admission_facts_not_permitted"
11281 );
11282
11283 let other_target = handler
11284 .handle_control_frame(
11285 &route_ctx(ConnectionId::new(74)).0,
11286 route_open_frame_with_admission_facts(
11287 12,
11288 "other",
11289 unique_project_root("admission-facts"),
11290 Some(subc_control::ConsumerIdentity {
11291 module_id: "fed".to_string(),
11292 launch_nonce: "fed-nonce".to_string(),
11293 }),
11294 Some(json!({"x": 1})),
11295 ),
11296 )
11297 .await
11298 .unwrap();
11299 assert_eq!(
11300 parse_error(&other_target[0])["code"],
11301 "admission_facts_target_not_allowed"
11302 );
11303
11304 let nonexistent = handler
11305 .handle_control_frame(
11306 &route_ctx(ConnectionId::new(75)).0,
11307 route_open_frame_with_admission_facts(
11308 13,
11309 "missing",
11310 unique_project_root("admission-facts"),
11311 None,
11312 Some(json!({"x": 1})),
11313 ),
11314 )
11315 .await
11316 .unwrap();
11317 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
11318
11319 let described = handler
11320 .handle_control_frame(
11321 &route_ctx(ConnectionId::new(76)).0,
11322 Frame::build(
11323 FrameType::Request,
11324 control_flags(),
11325 0,
11326 0,
11327 14,
11328 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11329 )
11330 .unwrap(),
11331 )
11332 .await
11333 .unwrap();
11334 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11335 serde_json::from_slice(&described[0].body).unwrap()
11336 else {
11337 panic!("expected server.describe response")
11338 };
11339 assert!(capabilities
11340 .iter()
11341 .any(|cap| cap == "admission_facts_relay_v1"));
11342 }
11343
11344 #[tokio::test]
11345 async fn admission_facts_without_configured_carrier_are_rejected() {
11346 let registry = Arc::new(Registry::default());
11347 let forwarding = Arc::new(ForwardingTable::default());
11348 let handler = ControlHandler::with_forwarding(registry, forwarding);
11349 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
11350 hello_via_sink(
11351 &handler,
11352 &target_ctx,
11353 &mut target_rx,
11354 hello_frame("target", PROTOCOL_VERSION, 1),
11355 )
11356 .await;
11357
11358 let responses = handler
11359 .handle_control_frame(
11360 &route_ctx(ConnectionId::new(79)).0,
11361 route_open_frame_with_admission_facts(
11362 16,
11363 "target",
11364 unique_project_root("admission-facts"),
11365 None,
11366 Some(json!({"x": 1})),
11367 ),
11368 )
11369 .await
11370 .unwrap();
11371 assert_eq!(
11372 parse_error(&responses[0])["code"],
11373 "admission_facts_not_permitted"
11374 );
11375 }
11376
11377 #[tokio::test]
11378 async fn route_open_relays_consumer_capabilities_verbatim() {
11379 let registry = Arc::new(Registry::default());
11380 let forwarding = Arc::new(ForwardingTable::default());
11381 let handler =
11382 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11383 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
11384 hello_via_sink(
11385 &handler,
11386 &module_ctx,
11387 &mut module_rx,
11388 hello_frame("aft", PROTOCOL_VERSION, 7),
11389 )
11390 .await;
11391
11392 let expected = vec!["elicitation".to_string(), "roots".to_string()];
11393 let expected_for_request = expected.clone();
11394 let project_root = unique_project_root("consumer-capabilities-present");
11395 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
11396 let route_handler = handler.clone();
11397 let route_task = tokio::spawn(async move {
11398 route_handler
11399 .handle_control_frame(
11400 &client_ctx,
11401 route_open_frame_with_consumer_capabilities(
11402 401,
11403 "aft",
11404 project_root,
11405 Some(expected_for_request),
11406 ),
11407 )
11408 .await
11409 .unwrap()
11410 });
11411 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11412 .await
11413 .unwrap()
11414 .unwrap();
11415 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11416 let ModuleControlRequest::RouteBind {
11417 consumer_capabilities,
11418 ..
11419 } = bind
11420 else {
11421 panic!("expected route.bind request, got {bind:?}");
11422 };
11423 assert_eq!(consumer_capabilities, Some(expected.clone()));
11424
11425 handler
11426 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11427 .await
11428 .unwrap();
11429 let route_response = route_task.await.unwrap();
11430 assert!(route_response.is_empty());
11431 let published = client_rx.recv().await.unwrap();
11432 assert!(matches!(
11433 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11434 ClientControlResponse::RouteOpen { .. }
11435 ));
11436 }
11437
11438 #[tokio::test]
11439 async fn route_open_without_consumer_capabilities_relays_none() {
11440 let registry = Arc::new(Registry::default());
11441 let forwarding = Arc::new(ForwardingTable::default());
11442 let handler =
11443 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11444 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
11445 hello_via_sink(
11446 &handler,
11447 &module_ctx,
11448 &mut module_rx,
11449 hello_frame("aft", PROTOCOL_VERSION, 7),
11450 )
11451 .await;
11452
11453 let project_root = unique_project_root("consumer-capabilities-absent");
11454 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
11455 let route_handler = handler.clone();
11456 let route_task = tokio::spawn(async move {
11457 route_handler
11458 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
11459 .await
11460 .unwrap()
11461 });
11462 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11463 .await
11464 .unwrap()
11465 .unwrap();
11466 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11467 let ModuleControlRequest::RouteBind {
11468 consumer_capabilities,
11469 ..
11470 } = bind
11471 else {
11472 panic!("expected route.bind request, got {bind:?}");
11473 };
11474 assert_eq!(consumer_capabilities, None);
11475
11476 handler
11477 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11478 .await
11479 .unwrap();
11480 let route_response = route_task.await.unwrap();
11481 assert!(route_response.is_empty());
11482 let published = client_rx.recv().await.unwrap();
11483 assert!(matches!(
11484 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11485 ClientControlResponse::RouteOpen { .. }
11486 ));
11487 }
11488
11489 async fn bind_role_versions_for(
11493 sent: Option<BTreeMap<String, String>>,
11494 connection: u64,
11495 ) -> Option<BTreeMap<String, String>> {
11496 let registry = Arc::new(Registry::default());
11497 let forwarding = Arc::new(ForwardingTable::default());
11498 let handler =
11499 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11500 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
11501 hello_via_sink(
11502 &handler,
11503 &module_ctx,
11504 &mut module_rx,
11505 hello_frame("aft", PROTOCOL_VERSION, 7),
11506 )
11507 .await;
11508 let project_root = unique_project_root("role-versions");
11509 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
11510 let route_handler = handler.clone();
11511 let route_task = tokio::spawn(async move {
11512 route_handler
11513 .handle_control_frame(
11514 &client_ctx,
11515 route_open_frame_with_role_versions(403, "aft", project_root, sent),
11516 )
11517 .await
11518 .unwrap()
11519 });
11520 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11521 .await
11522 .expect("a well-formed route.open reaches the module as a bind")
11523 .unwrap();
11524 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11525 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
11526 panic!("expected route.bind request, got {bind:?}");
11527 };
11528 handler
11529 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11530 .await
11531 .unwrap();
11532 assert!(route_task.await.unwrap().is_empty());
11533 let published = client_rx.recv().await.unwrap();
11534 assert!(matches!(
11535 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11536 ClientControlResponse::RouteOpen { .. }
11537 ));
11538 role_versions
11539 }
11540
11541 #[tokio::test]
11542 async fn route_open_relays_role_versions_verbatim() {
11543 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
11544 assert_eq!(
11545 bind_role_versions_for(Some(sent.clone()), 141).await,
11546 Some(sent)
11547 );
11548 }
11549
11550 #[tokio::test]
11553 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
11554 assert_eq!(bind_role_versions_for(None, 143).await, None);
11555 assert_eq!(
11556 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
11557 None
11558 );
11559 }
11560
11561 #[tokio::test]
11562 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
11563 let registry = Arc::new(Registry::default());
11564 let forwarding = Arc::new(ForwardingTable::default());
11565 let handler =
11566 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11567 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
11568 hello_via_sink(
11569 &handler,
11570 &module_ctx,
11571 &mut module_rx,
11572 hello_frame("aft", PROTOCOL_VERSION, 7),
11573 )
11574 .await;
11575
11576 let nine: BTreeMap<String, String> = (0..9)
11577 .map(|index| (format!("role-{index}"), "v1".to_string()))
11578 .collect();
11579 for (label, malformed) in [
11580 (
11581 "invalid role name",
11582 role_versions(&[("Tool_Provider", "v1")]),
11583 ),
11584 ("invalid version", role_versions(&[("tool-provider", "v0")])),
11585 ("nine entries", nine),
11586 ] {
11587 let responses = tokio::time::timeout(
11590 Duration::from_secs(1),
11591 handler.handle_control_frame(
11592 &route_ctx(ConnectionId::new(148)).0,
11593 route_open_frame_with_role_versions(
11594 404,
11595 "aft",
11596 unique_project_root("role-versions-malformed"),
11597 Some(malformed),
11598 ),
11599 ),
11600 )
11601 .await
11602 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11603 .unwrap();
11604 assert_eq!(responses.len(), 1, "{label}");
11605 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11606 let error = parse_error(&responses[0]);
11607 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11608 assert_eq!(
11609 error["detail"]["field"], "role_versions",
11610 "{label}: {error}"
11611 );
11612 assert!(
11613 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11614 "{label}: a malformed declaration is terminal"
11615 );
11616 assert!(
11617 module_rx.try_recv().is_err(),
11618 "{label}: the module must never see a bind"
11619 );
11620 }
11621 }
11622
11623 #[tokio::test]
11627 async fn route_role_versions_capability_is_advertised() {
11628 let handler = ControlHandler::new(Arc::new(Registry::default()));
11629 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11630 let ack = hello_via_sink(
11631 &handler,
11632 &ctx,
11633 &mut rx,
11634 hello_frame("m", PROTOCOL_VERSION, 1),
11635 )
11636 .await;
11637 let ack = parse_ack(&ack);
11638 assert!(
11639 ack.subc_capabilities
11640 .iter()
11641 .any(|c| c == "route-role-versions/v1"),
11642 "{:?}",
11643 ack.subc_capabilities
11644 );
11645
11646 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11647 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11648 let reply = handler
11649 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11650 .await
11651 .unwrap()
11652 .pop()
11653 .unwrap();
11654 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11655 serde_json::from_slice(&reply.body).unwrap()
11656 else {
11657 panic!("not a server.describe reply");
11658 };
11659 assert!(
11660 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
11661 "{capabilities:?}"
11662 );
11663 }
11664
11665 #[tokio::test]
11669 async fn catalog_events_capability_is_advertised() {
11670 let handler = ControlHandler::new(Arc::new(Registry::default()));
11671 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11672 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11673 let reply = handler
11674 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
11675 .await
11676 .unwrap()
11677 .pop()
11678 .unwrap();
11679 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11680 serde_json::from_slice(&reply.body).unwrap()
11681 else {
11682 panic!("not a server.describe reply");
11683 };
11684 assert!(
11685 capabilities
11686 .iter()
11687 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
11688 "{capabilities:?}"
11689 );
11690 }
11691
11692 #[tokio::test]
11693 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
11694 let handler = ControlHandler::new(Arc::new(Registry::default()));
11695 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
11696 let ack = hello_via_sink(
11697 &handler,
11698 &module,
11699 &mut module_rx,
11700 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
11701 )
11702 .await;
11703 let ack = parse_ack(&ack);
11704 assert!(
11705 ack.subc_capabilities
11706 .iter()
11707 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11708 "{:?}",
11709 ack.subc_capabilities
11710 );
11711 assert!(ack
11712 .subc_capabilities
11713 .iter()
11714 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11715
11716 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
11717 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11718 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11719 let reply = handler
11720 .handle_control_frame(&client, frame)
11721 .await
11722 .unwrap()
11723 .pop()
11724 .unwrap();
11725 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11726 serde_json::from_slice(&reply.body).unwrap()
11727 else {
11728 panic!("not a server.describe reply");
11729 };
11730 assert!(
11731 capabilities
11732 .iter()
11733 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11734 "{capabilities:?}"
11735 );
11736 assert!(capabilities
11737 .iter()
11738 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11739 }
11740
11741 #[tokio::test]
11742 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
11743 let handler = ControlHandler::new(Arc::new(Registry::default()));
11744 let (module, mut rx) = route_ctx(ConnectionId::new(156));
11745 let ack = hello_via_sink(
11746 &handler,
11747 &module,
11748 &mut rx,
11749 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
11750 )
11751 .await;
11752 assert!(parse_ack(&ack)
11753 .subc_capabilities
11754 .iter()
11755 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
11756 assert!(parse_ack(&ack)
11757 .subc_capabilities
11758 .iter()
11759 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
11760 let (client, _rx) = route_ctx(ConnectionId::new(157));
11761 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11762 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11763 let reply = handler
11764 .handle_control_frame(&client, frame)
11765 .await
11766 .unwrap()
11767 .pop()
11768 .unwrap();
11769 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11770 serde_json::from_slice(&reply.body).unwrap()
11771 else {
11772 panic!("server.describe reply")
11773 };
11774 assert!(capabilities
11775 .iter()
11776 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
11777 assert!(capabilities
11778 .iter()
11779 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
11780 }
11781
11782 #[tokio::test]
11783 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
11784 let handler = ControlHandler::new(Arc::new(Registry::default()));
11785 let (module, mut rx) = route_ctx(ConnectionId::new(154));
11786 let ack = hello_via_sink(
11787 &handler,
11788 &module,
11789 &mut rx,
11790 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
11791 )
11792 .await;
11793 assert!(parse_ack(&ack)
11794 .subc_capabilities
11795 .iter()
11796 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11797 let (client, _rx) = route_ctx(ConnectionId::new(155));
11798 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11799 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11800 let reply = handler
11801 .handle_control_frame(&client, frame)
11802 .await
11803 .unwrap()
11804 .pop()
11805 .unwrap();
11806 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11807 serde_json::from_slice(&reply.body).unwrap()
11808 else {
11809 panic!("not a server.describe reply")
11810 };
11811 assert!(capabilities
11812 .iter()
11813 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11814 }
11815
11816 #[tokio::test]
11817 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
11818 let registry = Arc::new(Registry::default());
11819 let forwarding = Arc::new(ForwardingTable::default());
11820 let handler =
11821 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11822 .with_health_probe_timeout(Duration::from_secs(5));
11823 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
11824 hello_via_sink(
11825 &handler,
11826 &module_ctx,
11827 &mut module_rx,
11828 non_routable_hello_frame_with_control_ops(
11829 "mcp",
11830 300,
11831 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11832 ),
11833 )
11834 .await;
11835 assert!(registry
11836 .get_module("mcp")
11837 .unwrap()
11838 .unwrap()
11839 .manifest
11840 .provides
11841 .is_empty());
11842
11843 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
11844 let route_response = handler
11845 .handle_control_frame(
11846 &route_client_ctx,
11847 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
11848 )
11849 .await
11850 .unwrap();
11851 assert_eq!(route_response[0].header.ty, FrameType::Error);
11852 assert_eq!(
11853 parse_error(&route_response[0])["code"],
11854 "target_unavailable"
11855 );
11856 assert!(parse_error(&route_response[0])["message"]
11857 .as_str()
11858 .unwrap()
11859 .contains("does not provide the requested target"));
11860 assert!(module_rx.try_recv().is_err());
11861
11862 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
11863 let health_handler = handler.clone();
11864 let health_task = tokio::spawn(async move {
11865 health_handler
11866 .handle_control_frame(
11867 &health_client_ctx,
11868 supervisor_health_probe_frame(302, "mcp"),
11869 )
11870 .await
11871 .unwrap()
11872 });
11873 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11874 .await
11875 .unwrap()
11876 .unwrap();
11877 assert_eq!(
11878 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11879 ModuleControlRequest::HealthCheck {}
11880 );
11881 handler
11882 .handle_control_frame(
11883 &module_ctx,
11884 health_response(health_frame.header.corr, HealthStatus::Ok),
11885 )
11886 .await
11887 .unwrap();
11888 let health_response = health_task.await.unwrap();
11889 assert_eq!(health_response[0].header.ty, FrameType::Response);
11890 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11891 ClientControlResponse::SupervisorHealthProbe {
11892 module_id, status, ..
11893 } => {
11894 assert_eq!(module_id, "mcp");
11895 assert_eq!(status, HealthStatus::Ok);
11896 }
11897 other => panic!("unexpected health response: {other:?}"),
11898 }
11899
11900 forwarding
11905 .cleanup_connection(module_ctx.connection_id)
11906 .unwrap();
11907 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
11908 let cleanup_response = tokio::time::timeout(
11909 Duration::from_millis(200),
11910 handler.handle_control_frame(
11911 &cleanup_probe_ctx,
11912 supervisor_health_probe_frame(303, "mcp"),
11913 ),
11914 )
11915 .await
11916 .expect("probe should fail immediately when the control lane is gone")
11917 .unwrap();
11918 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
11919 assert_eq!(
11920 parse_error(&cleanup_response[0])["code"],
11921 "target_unavailable"
11922 );
11923 assert!(parse_error(&cleanup_response[0])["message"]
11924 .as_str()
11925 .unwrap()
11926 .contains("no module connection"));
11927
11928 handler
11929 .cleanup_connection(module_ctx.connection_id)
11930 .unwrap();
11931 }
11932
11933 #[tokio::test]
11934 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
11935 let registry = Arc::new(Registry::default());
11936 let supervisor_handle = SupervisorHandle::new();
11937 let supervisor =
11938 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11939 .with_handle(supervisor_handle.clone())
11940 .with_connection_file_path(
11941 std::env::temp_dir()
11942 .join(format!("subc-route-open-warming-{}", std::process::id())),
11943 );
11944 let module = supervisor
11945 .supervise_configured(
11946 ModuleSpec {
11947 module_id: "warming".to_string(),
11948 program: fake_aft_stub_path(),
11949 args: Vec::new(),
11950 env: Vec::new(),
11951 reserved: false,
11952 reserved_prefixes: Vec::new(),
11953 protocol: ModuleProtocol::Subc,
11954 overlap: Default::default(),
11955 },
11956 true,
11957 )
11958 .unwrap();
11959 assert_eq!(module.state().unwrap(), ModuleState::Running);
11960
11961 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11962 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
11963 let response = handler
11964 .handle_control_frame(
11965 &ctx,
11966 route_open_frame(304, "warming", unique_project_root("warming")),
11967 )
11968 .await
11969 .unwrap();
11970 module.stop().await.unwrap();
11971
11972 assert_eq!(response[0].header.ty, FrameType::Error);
11973 let error = parse_error(&response[0]);
11974 assert_eq!(error["code"], "module_warming");
11975 assert!(error["message"]
11976 .as_str()
11977 .unwrap()
11978 .contains("state=running, enabled=true, live=false"));
11979 }
11980
11981 #[test]
11982 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
11983 let handler = ControlHandler::new(Arc::new(Registry::default()));
11984 let capture = EventCapture::default();
11985 let _subscriber =
11986 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11987 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
11988 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
11989 let pending = (0..limit).collect::<Vec<_>>();
11990 let response = handler
11991 .route_open_capacity_refusal(
11992 &ctx,
11993 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
11994 "busy",
11995 pending.len(),
11996 limit,
11997 )
11998 .unwrap();
11999 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12000 let event = capture
12001 .events()
12002 .into_iter()
12003 .find(|event| {
12004 event.target == "control"
12005 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
12006 })
12007 .expect("connection admission refusal event");
12008 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12009 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12010 }
12011
12012 #[test]
12013 fn route_open_target_cap_logs_admission_reason_and_capacity() {
12014 let handler = ControlHandler::new(Arc::new(Registry::default()));
12015 let capture = EventCapture::default();
12016 let _subscriber =
12017 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12018 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
12019 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
12020 let guards = (0..limit)
12021 .map(|_| {
12022 handler
12023 .route_bind_concurrency
12024 .try_admit("busy", limit)
12025 .unwrap()
12026 })
12027 .collect::<Vec<_>>();
12028 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
12029 Err(in_flight) => in_flight,
12030 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
12031 };
12032 let response = handler
12033 .route_open_target_capacity_refusal(
12034 &ctx,
12035 &route_open_frame(397, "busy", unique_project_root("target-cap")),
12036 "busy",
12037 in_flight,
12038 )
12039 .unwrap();
12040 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12041 let event = capture
12042 .events()
12043 .into_iter()
12044 .find(|event| {
12045 event.target == "control"
12046 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
12047 })
12048 .expect("target admission refusal event");
12049 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12050 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12051 drop(guards);
12052 }
12053
12054 #[tokio::test(flavor = "current_thread")]
12055 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
12056 let handler = ControlHandler::new(Arc::new(Registry::default()));
12057 let capture = EventCapture::default();
12058 let _subscriber =
12059 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12060 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
12061 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12062
12063 let _ = handler
12064 .handle_client_control_request(
12065 &ctx,
12066 frame,
12067 ClientControlRequest::SupervisorSetEnabled {
12068 module_id: "broca".to_string(),
12069 enabled: false,
12070 },
12071 &mut None,
12072 )
12073 .await
12074 .unwrap();
12075
12076 let events = capture
12077 .events()
12078 .into_iter()
12079 .filter(|event| {
12080 event.target == "control"
12081 && event.fields.get("message").map(String::as_str)
12082 == Some("supervisor request received")
12083 })
12084 .collect::<Vec<_>>();
12085 assert_eq!(events.len(), 1, "one supervisor request log line");
12086 let event = &events[0];
12087 assert_eq!(event.level, tracing::Level::INFO);
12088 assert_eq!(
12089 event.fields.get("op"),
12090 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
12091 );
12092 assert_eq!(
12093 event.fields.get("module_id"),
12094 Some(&"\"broca\"".to_string())
12095 );
12096 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
12097 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
12098 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
12099 }
12100
12101 #[tokio::test(flavor = "current_thread")]
12102 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
12103 let registry = Arc::new(Registry::default());
12104 let module_connection = ConnectionId::new(303);
12105 registry
12106 .register_with_control_ops(
12107 manifest("aft", PROTOCOL_VERSION),
12108 PROTOCOL_VERSION,
12109 module_connection,
12110 module_baseline_control_ops(),
12111 )
12112 .unwrap();
12113 let handler = ControlHandler::new(registry);
12114 let capture = EventCapture::default();
12115 let _subscriber =
12116 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12117 let (ctx, _rx) = route_ctx(module_connection);
12118 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12119
12120 let _ = handler
12121 .handle_client_control_request(
12122 &ctx,
12123 frame,
12124 ClientControlRequest::SupervisorSetEnabled {
12125 module_id: "broca".to_string(),
12126 enabled: false,
12127 },
12128 &mut None,
12129 )
12130 .await
12131 .unwrap();
12132
12133 let callers = capture
12134 .events()
12135 .into_iter()
12136 .filter(|event| {
12137 event.fields.get("message").map(String::as_str)
12138 == Some("supervisor request received")
12139 })
12140 .map(|event| event.fields.get("caller").cloned())
12141 .collect::<Vec<_>>();
12142 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
12143 }
12144
12145 #[tokio::test(flavor = "current_thread")]
12146 async fn supervisor_list_does_not_log_request_received() {
12147 let handler = ControlHandler::new(Arc::new(Registry::default()));
12148 let capture = EventCapture::default();
12149 let _subscriber =
12150 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12151 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
12152 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12153
12154 handler
12155 .handle_client_control_request(
12156 &ctx,
12157 frame,
12158 ClientControlRequest::SupervisorList {},
12159 &mut None,
12160 )
12161 .await
12162 .unwrap();
12163
12164 assert!(capture.events().into_iter().all(|event| {
12165 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12166 }));
12167 }
12168
12169 #[tokio::test(flavor = "current_thread")]
12170 async fn supervisor_rescan_preview_does_not_log_request_received() {
12171 let handler = ControlHandler::new(Arc::new(Registry::default()));
12172 let capture = EventCapture::default();
12173 let _subscriber =
12174 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12175 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
12176 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12177
12178 handler
12179 .handle_client_control_request(
12180 &ctx,
12181 frame,
12182 ClientControlRequest::SupervisorRescan { preview: true },
12183 &mut None,
12184 )
12185 .await
12186 .unwrap();
12187
12188 assert!(capture.events().into_iter().all(|event| {
12189 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12190 }));
12191 }
12192
12193 struct ApproveOperator;
12194 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
12195 fn prompt(
12196 &self,
12197 _: &str,
12198 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
12199 ) -> crate::operator_confirm::ProviderResult {
12200 crate::operator_confirm::ProviderResult::Approved
12201 }
12202 }
12203
12204 #[tokio::test]
12205 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
12206 assert_eq!(
12207 ControlHandler::default().privileged_operator_prompt,
12208 PrivilegedOperatorPrompt::Log
12209 );
12210 for (initial, changed, running) in [
12211 ("log", "enforce", PrivilegedOperatorPrompt::Log),
12212 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
12213 ] {
12214 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
12215 let path = dir.join("subc.jsonc");
12216 std::fs::write(
12217 &path,
12218 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
12219 )
12220 .unwrap();
12221 let config = crate::daemon_config::load(&path).unwrap().unwrap();
12222 let registry = Arc::new(Registry::default());
12223 let handle = SupervisorHandle::new();
12224 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
12225 .with_handle(handle.clone());
12226 let handler = ControlHandler::new(registry)
12227 .with_supervisor(handle)
12228 .with_privileged_operator_prompt(config.privileged_operator_prompt)
12229 .with_supervisor_rescan(supervisor, &path, config.port);
12230 handler.forwarding.operator_confirms().configure(
12231 Arc::new(ApproveOperator),
12232 crate::operator_confirm::OperatorLimits::default(),
12233 );
12234 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
12235
12236 for (disk_mode, needs_restart) in [
12237 (Some(initial), false),
12238 (Some(changed), true),
12239 (Some(changed), true),
12240 (None, running == PrivilegedOperatorPrompt::Enforce),
12241 (Some(initial), false),
12242 ] {
12243 let mut doc = json!({"version":1});
12244 if let Some(mode) = disk_mode {
12245 doc["privileged_operator_prompt"] = json!(mode);
12246 }
12247 std::fs::write(&path, doc.to_string()).unwrap();
12248 for preview in [true, false] {
12249 let request = ClientControlRequest::SupervisorRescan { preview };
12250 let frame = Frame::build(
12251 FrameType::Request,
12252 control_flags(),
12253 0,
12254 0,
12255 1,
12256 serde_json::to_vec(&request).unwrap(),
12257 )
12258 .unwrap();
12259 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12260 assert_eq!(frames.len(), 1);
12261 let ClientControlResponse::SupervisorRescan { result } =
12262 serde_json::from_slice(&frames[0].body).unwrap()
12263 else {
12264 panic!("expected a rescan response");
12265 };
12266 let expected = if needs_restart {
12267 vec!["privileged_operator_prompt"]
12268 } else {
12269 Vec::new()
12270 };
12271 assert_eq!(result.restart_required, expected);
12272 assert_eq!(handler.privileged_operator_prompt, running);
12273 assert_eq!(
12274 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
12275 running
12276 );
12277 }
12278 }
12279 }
12280 }
12281
12282 #[tokio::test]
12283 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
12284 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
12285 let path = dir.join("subc.jsonc");
12286 let registry = Arc::new(Registry::default());
12287 let handle = SupervisorHandle::new();
12288 let supervisor =
12289 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
12290 let handler = ControlHandler::new(registry)
12291 .with_supervisor(handle)
12292 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
12293 .with_supervisor_rescan(supervisor, &path, None);
12294 handler.forwarding.operator_confirms().configure(
12295 Arc::new(ApproveOperator),
12296 crate::operator_confirm::OperatorLimits::default(),
12297 );
12298 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
12299 for preview in [true, false] {
12300 std::fs::write(
12301 &path,
12302 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
12303 )
12304 .unwrap();
12305 let request = ClientControlRequest::SupervisorRescan { preview };
12306 let frame = Frame::build(
12307 FrameType::Request,
12308 control_flags(),
12309 0,
12310 0,
12311 1,
12312 serde_json::to_vec(&request).unwrap(),
12313 )
12314 .unwrap();
12315 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12316 assert_eq!(frames.len(), 1);
12317 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
12318 assert_eq!(
12319 handler.privileged_operator_prompt,
12320 PrivilegedOperatorPrompt::Enforce
12321 );
12322 }
12323 }
12324
12325 #[tokio::test]
12330 async fn route_open_refusal_names_the_check_that_refused() {
12331 let handler = ControlHandler::new(Arc::new(Registry::default()));
12332 let capture = EventCapture::default();
12333 let _subscriber =
12334 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12335 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12336 let response = handler
12337 .handle_control_frame(
12338 &ctx,
12339 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
12340 )
12341 .await
12342 .unwrap();
12343
12344 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12345 let event = capture
12346 .events()
12347 .into_iter()
12348 .find(|event| {
12349 event.target == "control"
12350 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12351 })
12352 .expect("route.open refusal event");
12353 assert_eq!(
12354 event.fields.get("reason"),
12355 Some(&"\"not_registered\"".to_string())
12356 );
12357 }
12358
12359 #[tokio::test]
12360 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
12361 let registry = Arc::new(Registry::default());
12362 let supervisor_handle = SupervisorHandle::new();
12363 let supervisor =
12364 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12365 .with_handle(supervisor_handle.clone())
12366 .with_connection_file_path(std::env::temp_dir().join(format!(
12367 "subc-route-open-refusal-info-{}",
12368 std::process::id()
12369 )));
12370 let module = supervisor
12371 .supervise_configured(
12372 ModuleSpec {
12373 module_id: "warming".to_string(),
12374 program: fake_aft_stub_path(),
12375 args: Vec::new(),
12376 env: Vec::new(),
12377 reserved: false,
12378 reserved_prefixes: Vec::new(),
12379 protocol: ModuleProtocol::Subc,
12380 overlap: Default::default(),
12381 },
12382 true,
12383 )
12384 .unwrap();
12385 assert_eq!(module.state().unwrap(), ModuleState::Running);
12386
12387 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12388 assert!(handler
12389 .counters()
12390 .snapshot()
12391 .get("route_open_refused_by_code")
12392 .is_none());
12393 let capture = EventCapture::default();
12394 let _subscriber =
12395 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12396 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
12397 let response = handler
12398 .handle_control_frame(
12399 &ctx,
12400 route_open_frame(394, "warming", unique_project_root("refusal-info")),
12401 )
12402 .await
12403 .unwrap();
12404 module.stop().await.unwrap();
12405
12406 assert_eq!(parse_error(&response[0])["code"], "module_warming");
12407 let event = capture
12408 .events()
12409 .into_iter()
12410 .find(|event| {
12411 event.target == "control"
12412 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
12413 })
12414 .expect("route.open refusal event");
12415 assert_eq!(
12416 event.fields.get("module_id"),
12417 Some(&"\"warming\"".to_string())
12418 );
12419 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
12420 assert_eq!(
12421 event.fields.get("reason"),
12422 Some(&"\"supervised_not_registered\"".to_string())
12423 );
12424 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
12425 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
12426 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
12427 assert_eq!(
12428 handler.counters().snapshot()["route_open_refused_by_code"],
12429 json!({ "module_warming": 1 })
12430 );
12431 }
12432
12433 const OUTAGE_START: &str = "route.open refusing module: not serving";
12434 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
12435
12436 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
12437 capture
12438 .events()
12439 .into_iter()
12440 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
12441 .collect()
12442 }
12443
12444 fn supervise_stub(
12445 registry: &Arc<Registry>,
12446 module_id: &str,
12447 enabled: bool,
12448 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
12449 let supervisor_handle = SupervisorHandle::new();
12450 let supervisor =
12451 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
12452 .with_handle(supervisor_handle.clone())
12453 .with_connection_file_path(std::env::temp_dir().join(format!(
12454 "subc-route-outage-{module_id}-{}",
12455 std::process::id()
12456 )));
12457 let module = supervisor
12458 .supervise_configured(
12459 ModuleSpec {
12460 module_id: module_id.to_string(),
12461 program: fake_aft_stub_path(),
12462 args: Vec::new(),
12463 env: Vec::new(),
12464 reserved: false,
12465 reserved_prefixes: Vec::new(),
12466 protocol: ModuleProtocol::Subc,
12467 overlap: Default::default(),
12468 },
12469 enabled,
12470 )
12471 .unwrap();
12472 (supervisor_handle, module)
12473 }
12474
12475 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
12476 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
12477 module_id: module_id.to_string(),
12478 drain_timeout_ms: Some(50),
12479 })
12480 .unwrap();
12481 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
12482 }
12483
12484 #[test]
12488 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
12489 let registry = Arc::new(Registry::default());
12490 let forwarding = Arc::new(ForwardingTable::default());
12491 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12492 let second = ControlHandler::with_forwarding(registry, forwarding);
12493 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
12494 }
12495
12496 #[tokio::test(flavor = "current_thread")]
12500 async fn route_open_unknown_module_refusals_add_no_outage_state() {
12501 let handler = ControlHandler::new(Arc::new(Registry::default()));
12502 let capture = EventCapture::default();
12503 let _subscriber =
12504 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12505 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
12506 for corr in 0..8 {
12507 let response = handler
12508 .handle_control_frame(
12509 &ctx,
12510 route_open_frame(
12511 380 + corr,
12512 &format!("nobody-{corr}"),
12513 unique_project_root("outage-unknown"),
12514 ),
12515 )
12516 .await
12517 .unwrap();
12518 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12519 }
12520
12521 assert_eq!(handler.route_outages.tracked_module_count(), 0);
12522 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
12523 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
12524 }
12525
12526 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12531 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
12532 let registry = Arc::new(Registry::default());
12533 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
12534 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12535 let capture = EventCapture::default();
12536 let _subscriber =
12537 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12538 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
12539 handler.route_outages.record_accepted("outage-restart");
12542
12543 let response = handler
12544 .handle_control_frame(
12545 &ctx,
12546 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
12547 )
12548 .await
12549 .unwrap();
12550 assert_eq!(response[0].header.ty, FrameType::Error);
12551 let starts = outage_lines(&capture, OUTAGE_START);
12552 assert_eq!(starts.len(), 1, "{starts:?}");
12553 assert_eq!(starts[0].level, tracing::Level::WARN);
12554 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
12555 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
12556 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
12557 handler.route_outages.record_accepted("outage-restart");
12558 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
12559
12560 let restart = handler
12561 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
12562 .await
12563 .unwrap();
12564 assert_eq!(
12565 restart[0].header.ty,
12566 FrameType::Response,
12567 "{:?}",
12568 parse_error(&restart[0])
12569 );
12570 handler
12571 .handle_control_frame(
12572 &ctx,
12573 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
12574 )
12575 .await
12576 .unwrap();
12577 module.stop().await.unwrap();
12578
12579 let starts = outage_lines(&capture, OUTAGE_START);
12580 assert_eq!(starts.len(), 2, "{starts:?}");
12581 assert_eq!(starts[1].level, tracing::Level::INFO);
12582 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
12583 }
12584
12585 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12589 async fn failed_operator_restart_leaves_no_operator_mark() {
12590 let registry = Arc::new(Registry::default());
12591 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
12592 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12593 let capture = EventCapture::default();
12594 let _subscriber =
12595 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12596 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
12597 handler.route_outages.record_accepted("outage-disabled");
12598
12599 let restart = handler
12600 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
12601 .await
12602 .unwrap();
12603 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
12604 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
12605
12606 handler
12607 .handle_control_frame(
12608 &ctx,
12609 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
12610 )
12611 .await
12612 .unwrap();
12613 let starts = outage_lines(&capture, OUTAGE_START);
12614 assert_eq!(starts.len(), 1, "{starts:?}");
12615 assert_eq!(starts[0].level, tracing::Level::WARN);
12616 }
12617
12618 #[tokio::test(flavor = "current_thread")]
12619 async fn route_open_unknown_module_escapes_target_module_id() {
12620 let handler = ControlHandler::new(Arc::new(Registry::default()));
12621 let capture = EventCapture::default();
12622 let _subscriber =
12623 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12624 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
12625 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12626 let response = handler
12627 .handle_control_frame(
12628 &ctx,
12629 route_open_frame(
12630 395,
12631 hostile_module_id,
12632 unique_project_root("hostile-target-module-id"),
12633 ),
12634 )
12635 .await
12636 .unwrap();
12637
12638 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12639 let event = capture
12640 .events()
12641 .into_iter()
12642 .find(|event| {
12643 event.target == "control"
12644 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12645 })
12646 .expect("route.open unknown-module refusal event");
12647 let logged = event.fields.get("module_id").expect("module_id field");
12648 assert!(!logged.bytes().any(|byte| byte < 0x20));
12649 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12650 }
12651
12652 #[tokio::test(flavor = "current_thread")]
12653 async fn route_open_module_rejection_uses_daemon_counter_key() {
12654 let registry = Arc::new(Registry::default());
12655 let forwarding = Arc::new(ForwardingTable::default());
12656 let handler =
12657 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12658 let module_connection = ConnectionId::new(95);
12659 let (module_ctx, mut module_rx) = route_ctx(module_connection);
12660 hello_via_sink(
12661 &handler,
12662 &module_ctx,
12663 &mut module_rx,
12664 hello_frame("aft", PROTOCOL_VERSION, 395),
12665 )
12666 .await;
12667
12668 let client_connection = ConnectionId::new(96);
12669 let (client_ctx, _client_rx) = route_ctx(client_connection);
12670 let capture = EventCapture::default();
12671 let _subscriber =
12672 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12673 let (route_task, bind) = relay_route_open(
12674 &handler,
12675 client_connection,
12676 &client_ctx.egress,
12677 &mut module_rx,
12678 396,
12679 "aft",
12680 "hostile-module-code",
12681 )
12682 .await;
12683 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
12684 let rejection = Frame::build(
12685 FrameType::Error,
12686 control_flags(),
12687 0,
12688 0,
12689 bind.header.corr,
12690 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
12691 )
12692 .unwrap();
12693 handler
12694 .handle_control_frame(&module_ctx, rejection)
12695 .await
12696 .unwrap();
12697
12698 let response = route_task.await.unwrap();
12699 assert_eq!(parse_error(&response[0])["code"], hostile_code);
12700 let counters = handler.counters().snapshot();
12701 assert_eq!(
12702 counters["route_open_refused_by_code"],
12703 json!({ "module_rejected": 1 })
12704 );
12705 assert!(counters["route_open_refused_by_code"]
12706 .get(hostile_code)
12707 .is_none());
12708
12709 let event = capture
12710 .events()
12711 .into_iter()
12712 .find(|event| {
12713 event.target == "control"
12714 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
12715 })
12716 .expect("route.open module-rejection refusal event");
12717 let logged = event.fields.get("module_code").expect("module_code field");
12718 assert!(!logged.bytes().any(|byte| byte < 0x20));
12719 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12720 }
12721
12722 #[tokio::test]
12723 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
12724 let registry = Arc::new(Registry::default());
12725 let supervisor_handle = SupervisorHandle::new();
12726 let missing_program = std::env::temp_dir().join(format!(
12727 "subc-route-open-missing-program-{}",
12728 std::process::id()
12729 ));
12730 let supervisor =
12731 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12732 .with_handle(supervisor_handle.clone());
12733 let module = supervisor
12734 .supervise_configured(
12735 ModuleSpec {
12736 module_id: "failed".to_string(),
12737 program: missing_program,
12738 args: Vec::new(),
12739 env: Vec::new(),
12740 reserved: false,
12741 reserved_prefixes: Vec::new(),
12742 protocol: ModuleProtocol::Subc,
12743 overlap: Default::default(),
12744 },
12745 true,
12746 )
12747 .unwrap();
12748 assert_eq!(module.state().unwrap(), ModuleState::Failed);
12749
12750 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12751 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
12752 let response = handler
12753 .handle_control_frame(
12754 &ctx,
12755 route_open_frame(305, "failed", unique_project_root("failed")),
12756 )
12757 .await
12758 .unwrap();
12759
12760 assert_eq!(response[0].header.ty, FrameType::Error);
12761 let error = parse_error(&response[0]);
12762 assert_eq!(error["code"], "target_unavailable");
12763 assert!(error["message"]
12764 .as_str()
12765 .unwrap()
12766 .contains("state=failed, enabled=true, live=false"));
12767 }
12768
12769 #[tokio::test]
12770 async fn route_open_role_mismatch_remains_target_unavailable() {
12771 let registry = Arc::new(Registry::default());
12772 let handler = ControlHandler::new(Arc::clone(®istry));
12773 handler
12774 .handle_control(
12775 ConnectionId::new(41),
12776 non_routable_hello_frame_with_control_ops("health-only", 306, None),
12777 )
12778 .unwrap();
12779
12780 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
12781 let response = handler
12782 .handle_control_frame(
12783 &ctx,
12784 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
12785 )
12786 .await
12787 .unwrap();
12788
12789 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12790 assert!(parse_error(&response[0])["message"]
12791 .as_str()
12792 .unwrap()
12793 .contains("does not provide the requested target"));
12794 }
12795
12796 #[tokio::test]
12797 async fn route_open_inactive_registration_remains_target_unavailable() {
12798 let registry = Arc::new(Registry::default());
12799 let handler = ControlHandler::new(Arc::clone(®istry));
12800 handler
12801 .handle_control(
12802 ConnectionId::new(43),
12803 hello_frame("inactive", PROTOCOL_VERSION, 308),
12804 )
12805 .unwrap();
12806 assert!(registry
12807 .set_module_state_for_test("inactive", ChannelState::Closed)
12808 .unwrap());
12809
12810 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
12811 let response = handler
12812 .handle_control_frame(
12813 &ctx,
12814 route_open_frame(309, "inactive", unique_project_root("inactive")),
12815 )
12816 .await
12817 .unwrap();
12818
12819 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12820 assert!(parse_error(&response[0])["message"]
12821 .as_str()
12822 .unwrap()
12823 .contains("is not active"));
12824 }
12825
12826 #[tokio::test]
12827 async fn late_health_reply_is_recorded_through_the_module_response_path() {
12828 let registry = Arc::new(Registry::default());
12829 let forwarding = Arc::new(ForwardingTable::default());
12830 let supervisor_handle = SupervisorHandle::new();
12831 let supervisor =
12832 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
12833 .with_forwarding(Arc::clone(&forwarding))
12834 .with_handle(supervisor_handle.clone());
12835 let module = supervisor
12836 .supervise_configured(
12837 crate::ModuleSpec {
12838 module_id: "late-health-response".to_string(),
12839 program: PathBuf::from("disabled-module"),
12840 args: Vec::new(),
12841 env: Vec::new(),
12842 reserved: false,
12843 reserved_prefixes: Vec::new(),
12844 protocol: ModuleProtocol::Subc,
12845 overlap: Default::default(),
12846 },
12847 false,
12848 )
12849 .unwrap();
12850 let handler =
12851 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12852 .with_supervisor(supervisor_handle);
12853 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
12854 handler
12855 .handle_control_frame(
12856 &module_ctx,
12857 hello_frame_with_control_ops(
12858 "late-health-response",
12859 PROTOCOL_VERSION,
12860 7,
12861 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12862 ),
12863 )
12864 .await
12865 .unwrap();
12866 let probe_started_at = Instant::now() - Duration::from_millis(80);
12867 let pending = forwarding
12868 .begin_health_probe_rpc_for(
12869 "late-health-response",
12870 MODULE_CONTROL_OP_HEALTH_CHECK,
12871 probe_started_at,
12872 Instant::now() - Duration::from_millis(1),
12873 )
12874 .unwrap();
12875 assert!(forwarding
12876 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
12877 .unwrap());
12878
12879 let responses = handler
12880 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
12881 .await
12882 .unwrap();
12883
12884 assert!(responses.is_empty());
12885 let health = module.status().unwrap().health;
12886 assert_eq!(health.late_answer_count, 1);
12887 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
12888 }
12889
12890 #[tokio::test]
12891 async fn health_probe_timeout_and_module_death_are_typed() {
12892 let registry = Arc::new(Registry::default());
12893 let forwarding = Arc::new(ForwardingTable::default());
12894 let handler =
12895 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12896 .with_health_probe_timeout(Duration::from_millis(50));
12897 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12898 hello_via_sink(
12899 &handler,
12900 &module_ctx,
12901 &mut module_rx,
12902 hello_frame_with_control_ops(
12903 "aft",
12904 PROTOCOL_VERSION,
12905 7,
12906 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12907 ),
12908 )
12909 .await;
12910
12911 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12912 let responses = handler
12913 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
12914 .await
12915 .unwrap();
12916 assert_eq!(responses[0].header.ty, FrameType::Error);
12917 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
12918 let _ = module_rx.try_recv();
12919
12920 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
12921 let health_handler = handler.clone();
12922 let death_task = tokio::spawn(async move {
12923 health_handler
12924 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
12925 .await
12926 .unwrap()
12927 });
12928 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12929 .await
12930 .unwrap()
12931 .unwrap();
12932 handler
12933 .cleanup_connection(module_ctx.connection_id)
12934 .unwrap();
12935 let responses = death_task.await.unwrap();
12936 assert_eq!(responses[0].header.ty, FrameType::Error);
12937 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
12938 }
12939
12940 #[test]
12941 fn hello_requires_exact_protocol_version() {
12942 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
12943 let registry = Arc::new(Registry::default());
12944 let handler = ControlHandler::new(Arc::clone(®istry));
12945 let responses = handler
12946 .handle_control(
12947 ConnectionId::new(connection),
12948 hello_frame("aft", offered, 9),
12949 )
12950 .unwrap();
12951
12952 assert_eq!(responses.len(), 1);
12953 assert_eq!(responses[0].header.ty, FrameType::Error);
12954 let error = parse_error(&responses[0]);
12955 assert_eq!(error["code"], "version_unsupported");
12956 assert!(registry.get_module("aft").unwrap().is_none());
12957 assert_eq!(registry.active_registration_count().unwrap(), 0);
12958 }
12959 }
12960
12961 #[test]
12962 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
12963 let registry = Arc::new(Registry::default());
12964 let forwarding = Arc::new(ForwardingTable::default());
12965 let handler =
12966 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12967 let module_connection = ConnectionId::new(301);
12968 let registration = registry
12969 .register_with_control_ops(
12970 manifest("aft-push", PROTOCOL_VERSION),
12971 PROTOCOL_VERSION,
12972 module_connection,
12973 module_baseline_control_ops(),
12974 )
12975 .unwrap();
12976 let (module_tx, _module_rx) = mpsc::channel(8);
12977 let endpoint = forwarding
12978 .register_module_connection(
12979 module_connection,
12980 "aft-push".to_string(),
12981 PROTOCOL_VERSION,
12982 manifest_concurrency(®istration.manifest),
12983 FrameSink::new(module_tx),
12984 )
12985 .unwrap();
12986
12987 let unknown = Frame::build(
12989 FrameType::Push,
12990 control_flags(),
12991 0,
12992 0,
12993 5,
12994 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
12995 )
12996 .unwrap();
12997 let out = handler.handle_status_update(endpoint, unknown).unwrap();
12998 assert!(
12999 out.is_empty(),
13000 "unknown push op must be ignored, got {out:?}"
13001 );
13002
13003 let malformed = Frame::build(
13005 FrameType::Push,
13006 control_flags(),
13007 0,
13008 0,
13009 6,
13010 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
13011 )
13012 .unwrap();
13013 let out = handler.handle_status_update(endpoint, malformed).unwrap();
13014 assert_eq!(out.len(), 1);
13015 assert_eq!(out[0].header.ty, FrameType::Error);
13016 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
13017 }
13018
13019 #[test]
13020 fn hello_rejected_when_connection_already_owns_client_routes() {
13021 let registry = Arc::new(Registry::default());
13022 let forwarding = Arc::new(ForwardingTable::default());
13023 let handler =
13024 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13025 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
13027 let client_connection = ConnectionId::new(202);
13028
13029 let responses = handler
13032 .handle_control(
13033 client_connection,
13034 hello_frame("aft-second", PROTOCOL_VERSION, 9),
13035 )
13036 .unwrap();
13037 assert_eq!(responses[0].header.ty, FrameType::Error);
13038 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
13039 assert!(registry.get_module("aft-second").unwrap().is_none());
13040 }
13041
13042 #[tokio::test]
13043 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
13044 let registry = Arc::new(Registry::default());
13045 let forwarding = Arc::new(ForwardingTable::default());
13046 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
13047 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
13048 hello_via_sink(
13049 &handler,
13050 &module_ctx,
13051 &mut module_rx,
13052 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
13053 )
13054 .await;
13055 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
13056 let pending = forwarding
13057 .begin_route_bind_relay_for_test(
13058 client_ctx.connection_id,
13059 client_ctx.egress.clone(),
13060 2,
13061 "alpha",
13062 )
13063 .unwrap();
13064 forwarding
13065 .complete_pending_relay(
13066 module_ctx.connection_id,
13067 pending.corr,
13068 RouteBindRelayOutcome::Accepted,
13069 )
13070 .unwrap();
13071 client_rx.try_recv().unwrap();
13072 for module_id in ["beta", "alpha"] {
13073 let replies = handler
13074 .handle_control_frame(
13075 &module_ctx,
13076 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
13077 )
13078 .await
13079 .unwrap();
13080 assert_eq!(replies.len(), 1, "second HELLO must be refused");
13081 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
13082 }
13083 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
13084 assert!(registry.get_module("beta").unwrap().is_none());
13085 assert!(matches!(
13086 forwarding
13087 .lookup_data_route(
13088 client_ctx.connection_id,
13089 pending.client_channel,
13090 pending.client_epoch,
13091 )
13092 .unwrap(),
13093 DataRoute::Client(DataRouteState::Bound(_))
13094 ));
13095 assert!(handler
13096 .hello_launch_nonces
13097 .lock()
13098 .unwrap()
13099 .presented(module_ctx.connection_id, Some("alpha-nonce")));
13100 assert!(module_rx.try_recv().is_err());
13101 }
13102
13103 #[test]
13104 fn reserved_module_hello_requires_matching_launch_nonce() {
13105 let registry = Arc::new(Registry::default());
13106 let supervisor = SupervisorHandle::new();
13107 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
13110 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13111
13112 let no_nonce = handler
13114 .handle_control(
13115 ConnectionId::new(1),
13116 hello_frame("vault", PROTOCOL_VERSION, 1),
13117 )
13118 .unwrap();
13119 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
13120 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
13121 assert!(registry.get_module("vault").unwrap().is_none());
13122
13123 let wrong = handler
13125 .handle_control(
13126 ConnectionId::new(2),
13127 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
13128 )
13129 .unwrap();
13130 assert_eq!(wrong[0].header.ty, FrameType::Error);
13131 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
13132 assert!(registry.get_module("vault").unwrap().is_none());
13133
13134 let ok = handler
13136 .handle_control(
13137 ConnectionId::new(3),
13138 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
13139 )
13140 .unwrap();
13141 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
13142 assert!(registry.get_module("vault").unwrap().is_some());
13143 }
13144
13145 #[test]
13146 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
13147 let registry = Arc::new(Registry::default());
13148 let supervisor = SupervisorHandle::new();
13149 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13150 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13151 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13152
13153 let squat = handler
13154 .handle_control(
13155 ConnectionId::new(1),
13156 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
13157 )
13158 .unwrap();
13159 assert_eq!(squat[0].header.ty, FrameType::Error);
13160 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
13161 assert!(parse_error(&squat[0])["message"]
13162 .as_str()
13163 .unwrap()
13164 .contains("fed:"));
13165
13166 let accepted_peer = handler
13167 .handle_control(
13168 ConnectionId::new(2),
13169 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
13170 )
13171 .unwrap();
13172 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
13173
13174 let accepted_short = handler
13175 .handle_control(
13176 ConnectionId::new(3),
13177 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
13178 )
13179 .unwrap();
13180 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
13181
13182 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
13183 let response = handler
13184 .handle_control(
13185 ConnectionId::new(conn),
13186 hello_frame(module_id, PROTOCOL_VERSION, conn),
13187 )
13188 .unwrap();
13189 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
13190 }
13191 }
13192
13193 #[test]
13194 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
13195 let registry = Arc::new(Registry::default());
13196 let supervisor = SupervisorHandle::new();
13197 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13198 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13199 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
13200 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13201
13202 let owner_nonce = handler
13203 .handle_control(
13204 ConnectionId::new(1),
13205 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
13206 )
13207 .unwrap();
13208 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
13209 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
13210 assert!(registry.get_module("fed:special").unwrap().is_none());
13211
13212 let exact_nonce = handler
13213 .handle_control(
13214 ConnectionId::new(2),
13215 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
13216 )
13217 .unwrap();
13218 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
13219 assert!(registry.get_module("fed:special").unwrap().is_some());
13220 }
13221
13222 #[test]
13223 fn non_reserved_module_ignores_launch_nonce() {
13224 let registry = Arc::new(Registry::default());
13225 let handler = ControlHandler::new(Arc::clone(®istry));
13228 let no_nonce = handler
13229 .handle_control(
13230 ConnectionId::new(1),
13231 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
13232 )
13233 .unwrap();
13234 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
13235 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
13236
13237 let echoed_nonce = handler
13238 .handle_control(
13239 ConnectionId::new(2),
13240 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
13241 )
13242 .unwrap();
13243 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
13244 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
13245 }
13246
13247 #[test]
13248 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
13249 let handler = ControlHandler::default();
13250 let conn = ConnectionId::new(1);
13251 let malformed = Frame::build(
13252 FrameType::Hello,
13253 control_flags(),
13254 0,
13255 0,
13256 3,
13257 b"{not json".to_vec(),
13258 )
13259 .unwrap();
13260
13261 let error = handler.handle_control(conn, malformed).unwrap();
13262 assert_eq!(error[0].header.ty, FrameType::Error);
13263 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
13264
13265 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13266 let pong = handler.handle_control(conn, ping).unwrap();
13267 assert_eq!(pong[0].header.ty, FrameType::Pong);
13268 assert_eq!(pong[0].header.corr, 4);
13269 }
13270
13271 #[test]
13272 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
13273 let registry = Arc::new(Registry::default());
13274 let handler = ControlHandler::new(Arc::clone(®istry));
13275
13276 handler
13277 .handle_control(
13278 ConnectionId::new(1),
13279 hello_frame("aft", PROTOCOL_VERSION, 1),
13280 )
13281 .unwrap();
13282 let duplicate = handler
13283 .handle_control(
13284 ConnectionId::new(2),
13285 hello_frame("aft", PROTOCOL_VERSION, 2),
13286 )
13287 .unwrap();
13288
13289 assert_eq!(duplicate[0].header.ty, FrameType::Error);
13290 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13291 let registration = registry.get_module("aft").unwrap().unwrap();
13292 assert_eq!(registration.connection_id, ConnectionId::new(1));
13293 }
13294
13295 #[test]
13296 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
13297 let registry = Arc::new(Registry::default());
13298 let forwarding = Arc::new(ForwardingTable::default());
13299 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
13300 let handler =
13301 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13302 .with_process_liveness(process_liveness);
13303 let (ctx, route_channel, route_epoch) =
13304 bind_liveness_route(®istry, &forwarding, "aft-dead");
13305 let responses = handler
13306 .handle_route_poll(
13307 &ctx,
13308 route_poll_frame(41, PollKind::Liveness, route_channel),
13309 route_channel,
13310 route_epoch,
13311 PollKind::Liveness,
13312 )
13313 .unwrap();
13314
13315 assert_eq!(responses.len(), 1);
13316 assert_eq!(responses[0].header.ty, FrameType::Response);
13317 assert_route_poll_liveness(&responses[0], false);
13318 }
13319
13320 #[test]
13321 fn liveness_poll_without_process_source_uses_bound_route() {
13322 let registry = Arc::new(Registry::default());
13323 let forwarding = Arc::new(ForwardingTable::default());
13324 let handler =
13325 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13326 let (ctx, route_channel, route_epoch) =
13327 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
13328 let responses = handler
13329 .handle_route_poll(
13330 &ctx,
13331 route_poll_frame(42, PollKind::Liveness, route_channel),
13332 route_channel,
13333 route_epoch,
13334 PollKind::Liveness,
13335 )
13336 .unwrap();
13337
13338 assert_route_poll_liveness(&responses[0], true);
13339 }
13340
13341 #[test]
13342 fn liveness_poll_untracked_process_source_uses_bound_route() {
13343 let registry = Arc::new(Registry::default());
13344 let forwarding = Arc::new(ForwardingTable::default());
13345 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
13346 let handler =
13347 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13348 .with_process_liveness(process_liveness);
13349 let (ctx, route_channel, route_epoch) =
13350 bind_liveness_route(®istry, &forwarding, "aft-untracked");
13351 let responses = handler
13352 .handle_route_poll(
13353 &ctx,
13354 route_poll_frame(43, PollKind::Liveness, route_channel),
13355 route_channel,
13356 route_epoch,
13357 PollKind::Liveness,
13358 )
13359 .unwrap();
13360
13361 assert_route_poll_liveness(&responses[0], true);
13362 }
13363
13364 #[tokio::test]
13365 async fn unknown_op_returns_unknown_control_op() {
13366 let handler = ControlHandler::default();
13367 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
13368 let request = Frame::build(
13369 FrameType::Request,
13370 control_flags(),
13371 0,
13372 0,
13373 55,
13374 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
13375 )
13376 .unwrap();
13377
13378 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13379
13380 assert_eq!(response.len(), 1);
13381 assert_eq!(response[0].header.ty, FrameType::Error);
13382 assert_eq!(response[0].header.corr, 55);
13383 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
13384 }
13385
13386 #[tokio::test]
13387 async fn supervisor_provenance_rejects_unknown_exact_module() {
13388 let handler = ControlHandler::default();
13389 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
13390 let request = Frame::build(
13391 FrameType::Request,
13392 control_flags(),
13393 0,
13394 0,
13395 57,
13396 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
13397 )
13398 .unwrap();
13399
13400 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13401
13402 assert_eq!(response.len(), 1);
13403 assert_eq!(response[0].header.ty, FrameType::Error);
13404 assert_eq!(response[0].header.corr, 57);
13405 let error = parse_error(&response[0]);
13406 assert_eq!(error["code"], "unknown_module");
13407 assert_eq!(error["message"], "module_id 'missing' is not supervised");
13408 }
13409
13410 #[test]
13411 fn provenance_probe_override_keeps_handler_tests_deterministic() {
13412 let expected = subc_control::RunningImageAgreement::Unavailable {
13413 reason: subc_control::RunningImageUnavailableReason::HashFailed,
13414 };
13415 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
13416 assert_eq!(handler.provenance_probe_override, Some(expected));
13417 }
13418
13419 #[test]
13420 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
13421 let verdict = reload_verdict(
13422 std::path::Path::new("/bin/new"),
13423 Some(std::path::Path::new("/bin/old")),
13424 subc_control::RunningImageAgreement::Unavailable {
13425 reason: subc_control::RunningImageUnavailableReason::HashFailed,
13426 },
13427 );
13428 assert!(matches!(
13429 verdict.path,
13430 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
13431 if configured == std::path::Path::new("/bin/new")
13432 && spawned_from == std::path::Path::new("/bin/old")
13433 ));
13434 }
13435
13436 #[test]
13437 fn launch_spec_without_running_digest_is_unavailable_not_match() {
13438 use subc_control::{
13439 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
13440 };
13441 let configured = crate::supervise::ModuleSpec {
13442 module_id: "legacy".into(),
13443 program: "/module".into(),
13444 args: Vec::new(),
13445 env: Vec::new(),
13446 reserved: false,
13447 reserved_prefixes: Vec::new(),
13448 protocol: subc_control::ModuleProtocol::Subc,
13449 overlap: Default::default(),
13450 };
13451 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
13452 assert_eq!(
13453 launch_spec_agreement(&configured, false, Some(&digest)),
13454 Agreement::Unavailable {
13455 reason: Reason::NotRunning
13456 }
13457 );
13458 assert_eq!(
13459 launch_spec_agreement(&configured, true, None),
13460 Agreement::Unavailable {
13461 reason: Reason::SpawnedSpecUnavailable
13462 }
13463 );
13464 assert_eq!(
13465 launch_spec_agreement(&configured, true, Some(&digest)),
13466 Agreement::Match
13467 );
13468 }
13469
13470 #[cfg(windows)]
13471 #[test]
13472 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
13473 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
13474 let configured = dir.join("module.exe");
13475 std::fs::write(&configured, b"executable path fixture").unwrap();
13476 let spawned = std::fs::canonicalize(&configured).unwrap();
13477 assert_ne!(
13478 configured, spawned,
13479 "fixture must exercise different path spellings"
13480 );
13481 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
13482 let lower_case = configured.to_string_lossy().to_lowercase();
13483 for spelling in [
13484 &configured,
13485 Path::new(&forward_slashes),
13486 Path::new(&lower_case),
13487 ] {
13488 let verdict = reload_verdict(
13489 spelling,
13490 Some(&spawned),
13491 subc_control::RunningImageAgreement::Unavailable {
13492 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13493 },
13494 );
13495 assert_eq!(
13496 verdict.path,
13497 ReloadPathAgreement::Match,
13498 "configured {spelling:?}, spawned {spawned:?}"
13499 );
13500 }
13501 }
13502
13503 #[cfg(windows)]
13504 #[test]
13505 fn windows_reload_path_keeps_different_existing_program_pending() {
13506 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
13507 let old = dir.join("old.exe");
13508 let new = dir.join("new.exe");
13509 std::fs::write(&old, b"same executable content").unwrap();
13510 std::fs::copy(&old, &new).unwrap();
13511 let spawned = std::fs::canonicalize(&old).unwrap();
13512 let image = subc_control::RunningImageAgreement::Unavailable {
13513 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13514 };
13515 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
13516 assert!(
13517 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
13518 if configured == new && spawned_from == spawned)
13519 );
13520 let missing = dir.join("missing.exe");
13521 assert!(matches!(
13522 reload_verdict(&missing, Some(&spawned), image).path,
13523 ReloadPathAgreement::Mismatch { .. }
13524 ));
13525 }
13526
13527 #[test]
13528 fn reload_verdict_detects_replaced_image_at_same_path() {
13529 let image = subc_control::RunningImageAgreement::Mismatch {
13530 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
13531 digest: "old".into(),
13532 },
13533 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
13534 digest: "new".into(),
13535 },
13536 };
13537 let verdict = reload_verdict(
13538 std::path::Path::new("/bin/same"),
13539 Some(std::path::Path::new("/bin/same")),
13540 image.clone(),
13541 );
13542 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13543 assert_eq!(verdict.image, image);
13544 }
13545
13546 #[test]
13547 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
13548 let image = subc_control::RunningImageAgreement::Unavailable {
13549 reason: subc_control::RunningImageUnavailableReason::NotRunning,
13550 };
13551 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
13552 assert_eq!(
13553 verdict.path,
13554 subc_control::ReloadPathAgreement::Unavailable {
13555 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
13556 }
13557 );
13558 assert_eq!(verdict.image, image);
13559
13560 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
13561 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13562 };
13563 let verdict = reload_verdict(
13564 std::path::Path::new("/bin/same"),
13565 Some(std::path::Path::new("/bin/same")),
13566 unconfirmed.clone(),
13567 );
13568 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13569 assert_eq!(verdict.image, unconfirmed);
13570 }
13571
13572 #[test]
13573 fn reload_verdict_preserves_each_image_unavailability_reason() {
13574 use subc_control::RunningImageUnavailableReason as Reason;
13575
13576 for reason in [
13577 Reason::NotRunning,
13578 Reason::UnsupportedPlatform,
13579 Reason::RunningExecutableUnreadable,
13580 Reason::SpawnedPathUnreadable,
13581 Reason::HashFailed,
13582 Reason::ProcessIdentityUnconfirmed,
13583 Reason::Unknown("future_probe_reason".to_string()),
13584 ] {
13585 let image = subc_control::RunningImageAgreement::Unavailable {
13586 reason: reason.clone(),
13587 };
13588 let verdict = reload_verdict(
13589 std::path::Path::new("/bin/same"),
13590 Some(std::path::Path::new("/bin/same")),
13591 image.clone(),
13592 );
13593 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13594 assert_eq!(verdict.image, image, "{reason:?}");
13595 }
13596 }
13597
13598 #[tokio::test]
13599 async fn malformed_control_bodies_return_invalid_control_body() {
13600 let handler = ControlHandler::default();
13601 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
13602
13603 for (corr, body) in [
13604 (56, br#"{"route_channel":1}"#.as_slice()),
13605 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
13606 (
13607 58,
13608 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
13609 ),
13610 ] {
13611 let request = Frame::build(
13612 FrameType::Request,
13613 control_flags(),
13614 0,
13615 0,
13616 corr,
13617 body.to_vec(),
13618 )
13619 .unwrap();
13620 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13621
13622 assert_eq!(response.len(), 1);
13623 assert_eq!(response[0].header.ty, FrameType::Error);
13624 assert_eq!(response[0].header.corr, corr);
13625 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
13626 }
13627 }
13628
13629 #[tokio::test]
13630 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
13631 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13632 let registry = Arc::new(Registry::default());
13633 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13634 let router = Router::with_control_handler(Arc::clone(&control));
13635 let connection = router.begin_connection();
13636 let (ctx, mut rx) = route_ctx(connection.id());
13637
13638 router
13639 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
13640 .await
13641 .unwrap();
13642 let response = rx.recv().await.unwrap();
13643 let ack = parse_ack(&response);
13644 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13645 let channel = 1;
13646
13647 let goodbye =
13648 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
13649 router.route_for_connection(&ctx, goodbye).await.unwrap();
13650 assert!(rx.try_recv().is_err());
13651 assert!(registry.get_module("aft").unwrap().is_none());
13652
13653 router
13654 .route_for_connection(&ctx, channel_request(channel, 13))
13655 .await
13656 .unwrap();
13657 let error_frame = rx.recv().await.unwrap();
13658 assert_eq!(error_frame.header.ty, FrameType::Error);
13659 assert_eq!(error_frame.header.channel, channel);
13660 let captured = crate::router::test_log::captured_logs(&logs);
13661 assert_eq!(
13662 captured
13663 .lines()
13664 .filter(|line| {
13665 line.contains("module_id=aft")
13666 && line.contains("reason=explicit_goodbye")
13667 && line.contains("module registration ended")
13668 })
13669 .count(),
13670 1,
13671 "unexpected GOODBYE registry log: {captured}"
13672 );
13673 }
13674
13675 #[tokio::test]
13676 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
13677 let registry = Arc::new(Registry::default());
13678 let handler = ControlHandler::new(registry).with_capability_config(
13679 [("prov".to_string(), true), ("cons".to_string(), true)],
13680 BTreeMap::new(),
13681 );
13682 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
13683 register_capability_manifest(
13684 &handler,
13685 &provider_ctx,
13686 &mut provider_rx,
13687 capability_manifest("prov", &["thing/v1"], &[]),
13688 1,
13689 )
13690 .await;
13691 let mut consumer = capability_manifest("cons", &[], &[]);
13692 consumer.capabilities.as_mut().unwrap().requires.push(
13693 subc_protocol::manifest::CapabilityRequirement {
13694 capability: "thing/v1".to_string(),
13695 need: subc_protocol::manifest::CapabilityNeed::Required,
13696 },
13697 );
13698 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
13699 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
13700 assert_eq!(
13701 handler.capability_evaluator.verdict("cons", "thing/v1"),
13702 Some(CapabilityVerdict::Provided)
13703 );
13704 let (mut client_rx, _) = open_route_for_capability_test(
13705 &handler,
13706 &provider_ctx,
13707 &mut provider_rx,
13708 703,
13709 3,
13710 "prov",
13711 None,
13712 )
13713 .await;
13714 let goodbye =
13715 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13716 handler
13717 .handle_control_frame(&provider_ctx, goodbye)
13718 .await
13719 .unwrap();
13720 assert_eq!(
13721 handler.capability_evaluator.verdict("cons", "thing/v1"),
13722 Some(CapabilityVerdict::NeverProvided)
13723 );
13724 let closed = client_rx
13725 .try_recv()
13726 .expect("GOODBYE pushes route.closed before route GOODBYE");
13727 assert!(
13728 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
13729 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
13730 );
13731 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
13732 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
13733 }
13734
13735 struct RouteClosedFixture {
13736 handler: ControlHandler,
13737 #[cfg(unix)]
13738 supervisor: Supervisor,
13739 module_ctx: RouteCtx,
13740 #[cfg(unix)]
13741 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13742 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13743 route: Arc<crate::forwarding::RouteBinding>,
13744 }
13745
13746 async fn route_closed_fixture() -> RouteClosedFixture {
13747 let registry = Arc::new(Registry::default());
13748 let handler = ControlHandler::new(Arc::clone(®istry));
13749 #[cfg(unix)]
13750 let supervisor = Supervisor::new(registry, RestartPolicy::default())
13751 .with_forwarding(Arc::clone(&handler.forwarding));
13752 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
13753 register_capability_manifest(
13754 &handler,
13755 &module_ctx,
13756 &mut module_rx,
13757 capability_manifest("prov", &[], &[]),
13758 1,
13759 )
13760 .await;
13761 let (client_rx, bind) = open_route_for_capability_test(
13762 &handler,
13763 &module_ctx,
13764 &mut module_rx,
13765 702,
13766 2,
13767 "prov",
13768 None,
13769 )
13770 .await;
13771 let ModuleControlRequest::RouteBind {
13772 route_channel: channel,
13773 epoch: route_epoch,
13774 ..
13775 } = bind
13776 else {
13777 panic!("expected route.bind");
13778 };
13779 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
13780 handler
13781 .forwarding
13782 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
13783 .unwrap()
13784 else {
13785 panic!("the acknowledged route must be bound");
13786 };
13787 RouteClosedFixture {
13788 handler,
13789 #[cfg(unix)]
13790 supervisor,
13791 module_ctx,
13792 #[cfg(unix)]
13793 module_rx,
13794 client_rx,
13795 route,
13796 }
13797 }
13798
13799 impl RouteClosedFixture {
13800 #[cfg(unix)]
13801 async fn cut(&mut self) {
13802 let notices = async {
13803 let mut draining = self.module_rx.recv().await.unwrap();
13804 assert!(matches!(
13805 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
13806 &draining.body
13807 )
13808 .unwrap(),
13809 subc_protocol::session::ModuleControlCommand::Draining {
13810 reason: RouteCloseReason::Restart,
13811 ..
13812 }
13813 ));
13814 draining.flushed.take().unwrap().send(()).unwrap();
13815 let mut closing = self.client_rx.recv().await.unwrap();
13816 assert!(matches!(
13817 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
13818 ClientControlPush::RouteClosing {
13819 reason: RouteCloseReason::Restart,
13820 ..
13821 }
13822 ));
13823 closing.flushed.take().unwrap().send(()).unwrap();
13824 };
13825 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
13826 result.unwrap();
13827 }
13828
13829 fn disconnect(&mut self) -> ClientControlPush {
13830 self.handler
13831 .cleanup_connection(self.module_ctx.connection_id)
13832 .unwrap();
13833 let closed = self.client_rx.try_recv().unwrap();
13834 assert_eq!(closed.header.ty, FrameType::Push);
13835 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
13836 assert!(matches!(
13837 &push,
13838 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
13839 if channels == &[self.route.client_channel]
13840 ));
13841 assert_eq!(
13842 self.client_rx.try_recv().unwrap().header.ty,
13843 FrameType::Goodbye
13844 );
13845 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
13846 push
13847 }
13848 }
13849
13850 #[cfg(unix)]
13851 #[tokio::test(start_paused = true)]
13852 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
13853 let mut fixture = route_closed_fixture().await;
13854 fixture.cut().await;
13855 assert!(matches!(
13856 fixture.disconnect(),
13857 ClientControlPush::RouteClosed {
13858 reason: RouteCloseReason::Restart,
13859 drained: true,
13860 ..
13861 }
13862 ));
13863 }
13864
13865 #[cfg(unix)]
13866 #[tokio::test(start_paused = true)]
13867 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
13868 let mut fixture = route_closed_fixture().await;
13869 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
13870 let started = tokio::time::Instant::now();
13871 fixture.cut().await;
13872 assert!(started.elapsed() >= Duration::from_secs(2));
13873 assert_eq!(fixture.route.flow.in_flight(), 1);
13874 assert!(matches!(
13875 fixture.disconnect(),
13876 ClientControlPush::RouteClosed {
13877 reason: RouteCloseReason::Restart,
13878 drained: false,
13879 ..
13880 }
13881 ));
13882 }
13883
13884 #[tokio::test]
13885 async fn module_connection_crash_reports_crash_and_not_drained() {
13886 let mut fixture = route_closed_fixture().await;
13887 assert!(matches!(
13891 fixture.disconnect(),
13892 ClientControlPush::RouteClosed {
13893 reason: RouteCloseReason::Crash,
13894 drained: false,
13895 ..
13896 }
13897 ));
13898 }
13899
13900 #[tokio::test]
13901 async fn dropping_router_connection_releases_registration() {
13902 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13903 let registry = Arc::new(Registry::default());
13904 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13905 let router = Router::with_control_handler(Arc::clone(&control));
13906 let connection = router.begin_connection();
13907 let connection_id = connection.id();
13908 let (ctx, mut rx) = route_ctx(connection_id);
13909
13910 router
13911 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
13912 .await
13913 .unwrap();
13914 let response = rx.recv().await.unwrap();
13915 let ack = parse_ack(&response);
13916 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13917 assert!(registry.get_module("aft").unwrap().is_some());
13918
13919 drop(connection);
13920
13921 assert!(registry.get_module("aft").unwrap().is_none());
13922 assert_eq!(registry.active_registration_count().unwrap(), 0);
13923
13924 control
13925 .cleanup_connection(ConnectionId::new(u64::MAX))
13926 .unwrap();
13927 let captured = crate::router::test_log::captured_logs(&logs);
13928 println!("captured registry lifecycle logs:\n{captured}");
13929 let events: Vec<_> = captured
13930 .lines()
13931 .filter(|line| {
13932 line.contains("module registered module_id=aft ")
13933 || line.contains("module registration ended")
13934 })
13935 .collect();
13936 assert_eq!(
13937 events.len(),
13938 2,
13939 "unexpected registry lifecycle logs: {captured}"
13940 );
13941 assert!(events[0].contains("module registered module_id=aft "));
13942 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
13943 assert!(events[1].contains(&format!(
13944 "module_id=aft connection_id={}",
13945 connection_id.get()
13946 )));
13947 assert!(events[1].contains("reason=connection_closed"));
13948 assert!(events[1].contains("module registration ended"));
13949 assert_eq!(
13950 captured
13951 .lines()
13952 .filter(|line| line.contains("module registered module_id=aft "))
13953 .count(),
13954 1,
13955 "legacy registration admission line must appear once: {captured}"
13956 );
13957 }
13958
13959 #[tokio::test]
13960 async fn hello_registration_keeps_legacy_module_registered_line_once() {
13961 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13962 let registry = Arc::new(Registry::default());
13963 let control = ControlHandler::new(Arc::clone(®istry));
13964 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
13965 hello_via_sink(
13966 &control,
13967 &ctx,
13968 &mut rx,
13969 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
13970 )
13971 .await;
13972
13973 let captured = crate::router::test_log::captured_logs(&logs);
13974 let admissions: Vec<_> = captured
13975 .lines()
13976 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
13977 .collect();
13978 assert_eq!(
13979 admissions.len(),
13980 1,
13981 "legacy admission line must remain exactly once: {captured}"
13982 );
13983 assert!(admissions[0].contains("routable_provider=true"));
13984 assert!(admissions[0].contains("connection_id=777"));
13985 }
13986
13987 fn capability_manifest(
13988 module_id: &str,
13989 provides: &[&str],
13990 must_never_reach: &[&str],
13991 ) -> ModuleManifest {
13992 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
13993 manifest.capabilities = Some(CapabilityDeclarations {
13994 provides: provides
13995 .iter()
13996 .map(|capability| (*capability).to_string())
13997 .collect(),
13998 requires: Vec::new(),
13999 must_never_reach: must_never_reach
14000 .iter()
14001 .map(|capability| (*capability).to_string())
14002 .collect(),
14003 });
14004 manifest
14005 }
14006
14007 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
14008 Frame::build(
14009 FrameType::Hello,
14010 control_flags(),
14011 0,
14012 0,
14013 corr,
14014 serde_json::to_vec(&ModuleHelloBody {
14015 protocol_ver: manifest.protocol_ver,
14016 manifest,
14017 control_ops: None,
14018 launch_nonce: None,
14019 })
14020 .expect("capability test HELLO serializes"),
14021 )
14022 .expect("capability test HELLO frame builds")
14023 }
14024
14025 fn catalog_update_with_capabilities_frame(
14026 corr: u64,
14027 capabilities: CapabilityDeclarations,
14028 ) -> Frame {
14029 Frame::build(
14030 FrameType::Request,
14031 control_flags(),
14032 0,
14033 0,
14034 corr,
14035 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14036 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
14037 capabilities: Some(capabilities),
14038 ready: None,
14039 })
14040 .expect("capability catalog.update serializes"),
14041 )
14042 .expect("capability catalog.update frame builds")
14043 }
14044
14045 async fn register_capability_manifest(
14046 handler: &ControlHandler,
14047 ctx: &RouteCtx,
14048 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14049 manifest: ModuleManifest,
14050 corr: u64,
14051 ) {
14052 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
14053 }
14054
14055 async fn open_route_for_capability_test(
14056 handler: &ControlHandler,
14057 target_ctx: &RouteCtx,
14058 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14059 client_connection_id: u64,
14060 corr: u64,
14061 target_module_id: &str,
14062 consumer_identity: Option<ConsumerIdentity>,
14063 ) -> (
14064 mpsc::Receiver<crate::router::OutboundFrame>,
14065 ModuleControlRequest,
14066 ) {
14067 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
14068 let route_handler = handler.clone();
14069 let target_module_id = target_module_id.to_string();
14070 let route_task = tokio::spawn(async move {
14071 route_handler
14072 .handle_control_frame(
14073 &client_ctx,
14074 route_open_frame_with_admission_facts(
14075 corr,
14076 &target_module_id,
14077 unique_project_root("admission-facts"),
14078 consumer_identity,
14079 None,
14080 ),
14081 )
14082 .await
14083 .expect("capability test route.open succeeds")
14084 });
14085 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
14086 .await
14087 .expect("capability test route.open must reach route.bind")
14088 .expect("target control receiver stays open");
14089 let bind_request: ModuleControlRequest =
14090 serde_json::from_slice(&bind.body).expect("route.bind decodes");
14091 handler
14092 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
14093 .await
14094 .expect("capability test route.bind ACK succeeds");
14095 assert!(route_task.await.expect("route.open task joins").is_empty());
14096 let opened = client_rx
14097 .recv()
14098 .await
14099 .expect("successful route.open publishes a response");
14100 assert!(matches!(
14101 serde_json::from_slice::<ClientControlResponse>(&opened.body),
14102 Ok(ClientControlResponse::RouteOpen { .. })
14103 ));
14104 (client_rx, bind_request)
14105 }
14106
14107 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
14108 assert_eq!(frame.header.ty, FrameType::Push);
14109 assert_eq!(frame.header.channel, 0);
14110 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
14111 .expect("route.closed control push decodes");
14112 let ClientControlPush::RouteClosed { channels, .. } = &push else {
14113 panic!("expected route.closed");
14114 };
14115 assert_eq!(channels.len(), 1, "exactly one violating route closed");
14116 let channels = channels.clone();
14117 assert_eq!(
14118 push,
14119 ClientControlPush::RouteClosed {
14120 module_id: target_module_id.to_string(),
14121 channels,
14122 reason: RouteCloseReason::CapabilityDenied,
14123 drained: false,
14124 abandoned: 0,
14125 excluded_subscriptions: 0,
14126 terminal: Some(false),
14127 }
14128 );
14129 }
14130
14131 #[tokio::test]
14132 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
14133 let registry = Arc::new(Registry::default());
14134 let forwarding = Arc::new(ForwardingTable::default());
14135 let supervisor = SupervisorHandle::new();
14136 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14137 let handler =
14138 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14139 .with_supervisor(supervisor);
14140 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
14141 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
14142 register_capability_manifest(
14143 &handler,
14144 &target_ctx,
14145 &mut target_rx,
14146 capability_manifest("target", &["credentials-provider/v1"], &[]),
14147 1,
14148 )
14149 .await;
14150 register_capability_manifest(
14151 &handler,
14152 &opener_ctx,
14153 &mut opener_rx,
14154 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14155 2,
14156 )
14157 .await;
14158
14159 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
14160 let replies = handler
14161 .handle_control_frame(
14162 &client_ctx,
14163 route_open_frame_with_admission_facts(
14164 3,
14165 "target",
14166 unique_project_root("admission-facts"),
14167 Some(ConsumerIdentity {
14168 module_id: "opener".to_string(),
14169 launch_nonce: "opener-nonce".to_string(),
14170 }),
14171 None,
14172 ),
14173 )
14174 .await
14175 .expect("denied route.open returns a typed frame");
14176 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14177 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14178 assert!(
14179 target_rx.try_recv().is_err(),
14180 "forbidden route.open must not relay route.bind"
14181 );
14182 }
14183
14184 #[tokio::test]
14185 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
14186 let registry = Arc::new(Registry::default());
14187 let forwarding = Arc::new(ForwardingTable::default());
14188 let supervisor = SupervisorHandle::new();
14189 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14190 let handler =
14191 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14192 .with_supervisor(supervisor);
14193 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
14194 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
14195 register_capability_manifest(
14196 &handler,
14197 &target_ctx,
14198 &mut target_rx,
14199 capability_manifest("target", &["credentials-provider/v1"], &[]),
14200 1,
14201 )
14202 .await;
14203 register_capability_manifest(
14204 &handler,
14205 &old_opener_ctx,
14206 &mut old_opener_rx,
14207 capability_manifest("opener", &[], &[]),
14208 2,
14209 )
14210 .await;
14211 let (mut client_rx, _) = open_route_for_capability_test(
14212 &handler,
14213 &target_ctx,
14214 &mut target_rx,
14215 712,
14216 3,
14217 "target",
14218 Some(ConsumerIdentity {
14219 module_id: "opener".to_string(),
14220 launch_nonce: "opener-nonce".to_string(),
14221 }),
14222 )
14223 .await;
14224 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14225
14226 handler
14227 .cleanup_connection(old_opener_ctx.connection_id)
14228 .expect("old opener registration cleans up");
14229 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
14230 register_capability_manifest(
14231 &handler,
14232 &new_opener_ctx,
14233 &mut new_opener_rx,
14234 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14235 4,
14236 )
14237 .await;
14238
14239 assert_capability_denied_push(
14240 client_rx
14241 .try_recv()
14242 .expect("HELLO deny addition must emit route.closed")
14243 .frame,
14244 "target",
14245 );
14246 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14247 assert!(matches!(
14248 target_rx.try_recv(),
14249 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14250 ));
14251 }
14252
14253 #[tokio::test]
14254 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
14255 let registry = Arc::new(Registry::default());
14256 let forwarding = Arc::new(ForwardingTable::default());
14257 let supervisor = SupervisorHandle::new();
14258 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14259 let handler =
14260 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14261 .with_supervisor(supervisor);
14262 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
14263 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
14264 register_capability_manifest(
14265 &handler,
14266 &target_ctx,
14267 &mut target_rx,
14268 capability_manifest("target", &[], &[]),
14269 1,
14270 )
14271 .await;
14272 register_capability_manifest(
14273 &handler,
14274 &opener_ctx,
14275 &mut opener_rx,
14276 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14277 2,
14278 )
14279 .await;
14280 let (mut client_rx, _) = open_route_for_capability_test(
14281 &handler,
14282 &target_ctx,
14283 &mut target_rx,
14284 722,
14285 3,
14286 "target",
14287 Some(ConsumerIdentity {
14288 module_id: "opener".to_string(),
14289 launch_nonce: "opener-nonce".to_string(),
14290 }),
14291 )
14292 .await;
14293 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14294
14295 let replies = handler
14296 .handle_control_frame(
14297 &target_ctx,
14298 catalog_update_with_capabilities_frame(
14299 4,
14300 CapabilityDeclarations {
14301 provides: vec!["credentials-provider/v1".to_string()],
14302 requires: Vec::new(),
14303 must_never_reach: Vec::new(),
14304 },
14305 ),
14306 )
14307 .await
14308 .expect("claim catalog.update succeeds");
14309 assert!(matches!(
14310 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
14311 Ok(ModuleControlResponseToModule::CatalogUpdate {})
14312 ));
14313 assert_capability_denied_push(
14314 client_rx
14315 .try_recv()
14316 .expect("claim addition must emit route.closed")
14317 .frame,
14318 "target",
14319 );
14320 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14321 assert!(matches!(
14322 target_rx.try_recv(),
14323 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14324 ));
14325 }
14326
14327 #[tokio::test]
14328 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
14329 let registry = Arc::new(Registry::default());
14330 let forwarding = Arc::new(ForwardingTable::default());
14331 let supervisor = SupervisorHandle::new();
14332 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14333 let handler =
14334 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14335 .with_supervisor(supervisor);
14336 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
14337 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
14338 register_capability_manifest(
14339 &handler,
14340 &target_ctx,
14341 &mut target_rx,
14342 capability_manifest("target", &["credentials-provider/v1"], &[]),
14343 1,
14344 )
14345 .await;
14346 register_capability_manifest(
14347 &handler,
14348 &opener_ctx,
14349 &mut opener_rx,
14350 capability_manifest("opener", &[], &[]),
14351 2,
14352 )
14353 .await;
14354 let (mut client_rx, _) = open_route_for_capability_test(
14355 &handler,
14356 &target_ctx,
14357 &mut target_rx,
14358 732,
14359 3,
14360 "target",
14361 Some(ConsumerIdentity {
14362 module_id: "opener".to_string(),
14363 launch_nonce: "opener-nonce".to_string(),
14364 }),
14365 )
14366 .await;
14367 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14368
14369 handler
14370 .handle_control_frame(
14371 &target_ctx,
14372 catalog_update_with_capabilities_frame(
14373 4,
14374 CapabilityDeclarations {
14375 provides: Vec::new(),
14376 requires: Vec::new(),
14377 must_never_reach: Vec::new(),
14378 },
14379 ),
14380 )
14381 .await
14382 .expect("claim removal catalog.update succeeds");
14383 assert_eq!(
14384 forwarding.active_binding_count().unwrap(),
14385 1,
14386 "removing an attested target claim must leave the route census unchanged"
14387 );
14388 assert!(
14389 client_rx.try_recv().is_err(),
14390 "claim removal must not emit route.closed capability_denied"
14391 );
14392 assert!(
14393 target_rx.try_recv().is_err(),
14394 "claim removal must not send the target a route GOODBYE"
14395 );
14396 }
14397
14398 #[tokio::test]
14401 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
14402 let registry = Arc::new(Registry::default());
14403 let forwarding = Arc::new(ForwardingTable::default());
14404 let supervisor = SupervisorHandle::new();
14405 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14406 let handler =
14407 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14408 .with_supervisor(supervisor);
14409 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
14410 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
14411 register_capability_manifest(
14412 &handler,
14413 &target_ctx,
14414 &mut target_rx,
14415 capability_manifest("target", &["credentials-provider/v1"], &[]),
14416 1,
14417 )
14418 .await;
14419 register_capability_manifest(
14420 &handler,
14421 &opener_ctx,
14422 &mut opener_rx,
14423 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14424 2,
14425 )
14426 .await;
14427
14428 let (_client_rx, bind) = open_route_for_capability_test(
14429 &handler,
14430 &target_ctx,
14431 &mut target_rx,
14432 742,
14433 3,
14434 "target",
14435 None,
14436 )
14437 .await;
14438 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14439 panic!("direct scope-honesty route must bind");
14440 };
14441 assert_eq!(principal, Some(Principal::Direct));
14442 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14443 }
14444
14445 #[tokio::test]
14448 async fn must_never_reach_self_route_is_capability_forbidden() {
14449 let registry = Arc::new(Registry::default());
14450 let forwarding = Arc::new(ForwardingTable::default());
14451 let supervisor = SupervisorHandle::new();
14452 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
14453 let handler =
14454 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14455 .with_supervisor(supervisor);
14456 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
14457 register_capability_manifest(
14458 &handler,
14459 &self_ctx,
14460 &mut self_rx,
14461 capability_manifest(
14462 "self-provider",
14463 &["credentials-provider/v1"],
14464 &["credentials-provider/v1"],
14465 ),
14466 1,
14467 )
14468 .await;
14469
14470 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
14471 let replies = handler
14472 .handle_control_frame(
14473 &client_ctx,
14474 route_open_frame_with_admission_facts(
14475 2,
14476 "self-provider",
14477 unique_project_root("admission-facts"),
14478 Some(ConsumerIdentity {
14479 module_id: "self-provider".to_string(),
14480 launch_nonce: "self-nonce".to_string(),
14481 }),
14482 None,
14483 ),
14484 )
14485 .await
14486 .expect("self-route refusal returns a typed frame");
14487 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14488 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14489 assert!(
14490 self_rx.try_recv().is_err(),
14491 "self denial must not relay route.bind"
14492 );
14493 }
14494
14495 #[test]
14496 fn unsupported_channel_zero_frame_returns_error() {
14497 let handler = ControlHandler::default();
14498 let request = Frame::build(
14499 FrameType::Request,
14500 control_flags(),
14501 0,
14502 0,
14503 21,
14504 b"opaque".to_vec(),
14505 )
14506 .unwrap();
14507
14508 let response = handler
14509 .handle_control(ConnectionId::new(1), request)
14510 .unwrap();
14511
14512 assert_eq!(response[0].header.ty, FrameType::Error);
14513 assert_eq!(
14514 parse_error(&response[0])["code"],
14515 "unsupported_control_frame"
14516 );
14517 }
14518
14519 mod swap {
14524 use super::*;
14525
14526 const INCUMBENT: ConnectionId = ConnectionId::new(30);
14527 const CANDIDATE: ConnectionId = ConnectionId::new(40);
14528
14529 struct Swap {
14530 registry: Arc<Registry>,
14531 forwarding: Arc<ForwardingTable>,
14532 handler: ControlHandler,
14533 incumbent_ctx: RouteCtx,
14534 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14535 candidate_ctx: RouteCtx,
14536 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14537 }
14538
14539 async fn swap_with_incumbent() -> Swap {
14540 let registry = Arc::new(Registry::default());
14541 let forwarding = Arc::new(ForwardingTable::default());
14542 let handler =
14543 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14544 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
14545 hello_via_sink(
14546 &handler,
14547 &incumbent_ctx,
14548 &mut incumbent_rx,
14549 hello_frame("aft", PROTOCOL_VERSION, 7),
14550 )
14551 .await;
14552 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
14553 Swap {
14554 registry,
14555 forwarding,
14556 handler,
14557 incumbent_ctx,
14558 incumbent_rx,
14559 candidate_ctx,
14560 candidate_rx,
14561 }
14562 }
14563
14564 fn register_candidate(swap: &Swap, ready: Option<bool>) {
14565 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
14566 candidate_manifest.ready = ready;
14567 let registration = swap
14568 .registry
14569 .register_candidate_with_control_ops(
14570 candidate_manifest,
14571 PROTOCOL_VERSION,
14572 CANDIDATE,
14573 module_baseline_control_ops(),
14574 )
14575 .unwrap();
14576 swap.forwarding
14577 .register_candidate_module_connection(
14578 CANDIDATE,
14579 "aft".to_string(),
14580 PROTOCOL_VERSION,
14581 manifest_concurrency(®istration.manifest),
14582 swap.candidate_ctx.egress.clone(),
14583 )
14584 .unwrap();
14585 }
14586
14587 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
14588 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
14589 swap.registry.promote_candidate("aft").unwrap().unwrap();
14590 cutover.incumbent.unwrap()
14591 }
14592
14593 #[tokio::test]
14602 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
14603 let mut swap = swap_with_incumbent().await;
14604 let hold = Arc::new(std::sync::Barrier::new(2));
14605 let handler = swap
14606 .handler
14607 .clone()
14608 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14609 handler
14610 .supervisor
14611 .open_swap("aft", "candidate-nonce".to_string());
14612 let hello = {
14613 let handler = handler.clone();
14614 let candidate_ctx = swap.candidate_ctx.clone();
14615 std::thread::spawn(move || {
14616 handler
14617 .handle_hello(
14618 candidate_ctx.connection_id,
14619 Some(candidate_ctx.egress.clone()),
14620 hello_frame_with_nonce(
14621 "aft",
14622 PROTOCOL_VERSION,
14623 8,
14624 Some("candidate-nonce"),
14625 ),
14626 )
14627 .unwrap()
14628 })
14629 };
14630
14631 hold.wait();
14632 assert!(swap
14635 .registry
14636 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
14637 .unwrap()
14638 .is_some());
14639 assert!(swap
14640 .registry
14641 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14642 .unwrap()
14643 .is_none());
14644 assert!(swap
14645 .forwarding
14646 .module_endpoint_for_connection(CANDIDATE)
14647 .unwrap()
14648 .is_none());
14649
14650 hold.wait();
14651 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
14652 assert_eq!(
14653 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
14654 FrameType::HelloAck
14655 );
14656 let published = swap
14657 .registry
14658 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14659 .unwrap()
14660 .expect("candidate is published once its HELLO completes");
14661 assert_eq!(published.connection_id, CANDIDATE);
14662 assert!(swap
14663 .forwarding
14664 .module_endpoint_for_connection(CANDIDATE)
14665 .unwrap()
14666 .is_some());
14667 }
14668
14669 fn keyed_total(counters: &Value, key: &str) -> u64 {
14670 counters[key]
14671 .as_object()
14672 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
14673 .unwrap_or(0)
14674 }
14675
14676 #[tokio::test]
14677 async fn replacement_logs_old_end_and_new_admission_once() {
14678 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14679 let mut swap = swap_with_incumbent().await;
14680 swap.handler
14681 .supervisor
14682 .open_swap("aft", "candidate-nonce".to_string());
14683 hello_via_sink(
14684 &swap.handler,
14685 &swap.candidate_ctx,
14686 &mut swap.candidate_rx,
14687 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
14688 )
14689 .await;
14690 cutover(&swap);
14691 swap.handler.cleanup_connection(INCUMBENT).unwrap();
14692
14693 let captured = crate::router::test_log::captured_logs(&logs);
14694 println!("captured replacement registry logs:\n{captured}");
14695 let new_admissions: Vec<_> = captured
14696 .lines()
14697 .filter(|line| {
14698 line.contains("connection_id=40")
14699 && line.contains("swap candidate registered; not routable until cutover")
14700 })
14701 .collect();
14702 assert_eq!(
14703 new_admissions.len(),
14704 1,
14705 "unexpected admission logs: {captured}"
14706 );
14707 assert!(new_admissions[0].contains("module_id=aft"));
14708 assert!(new_admissions[0].contains("connection_id=40"));
14709 assert!(
14710 new_admissions[0].contains("swap candidate registered; not routable until cutover")
14711 );
14712 assert!(new_admissions[0].contains("ready=true"));
14713
14714 let old_admissions: Vec<_> = captured
14715 .lines()
14716 .filter(|line| {
14717 line.contains("module registered module_id=aft ")
14718 && line.contains("connection_id=30")
14719 })
14720 .collect();
14721 assert_eq!(
14722 old_admissions.len(),
14723 1,
14724 "unexpected incumbent admission: {captured}"
14725 );
14726
14727 let old_ends: Vec<_> = captured
14728 .lines()
14729 .filter(|line| {
14730 line.contains("connection_id=30") && line.contains("module registration ended")
14731 })
14732 .collect();
14733 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
14734 assert!(old_ends[0].contains("module_id=aft"));
14735 assert!(old_ends[0].contains("connection_id=30"));
14736 assert!(old_ends[0].contains("reason=replaced"));
14737 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
14738 assert_eq!(
14739 captured
14740 .lines()
14741 .filter(|line| line.contains("module registration promoted"))
14742 .count(),
14743 1,
14744 "unexpected promotion logs: {captured}"
14745 );
14746 }
14747
14748 #[tokio::test]
14754 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
14755 let mut swap = swap_with_incumbent().await;
14756 let handler = swap.handler.clone();
14757
14758 let cotenant = ConnectionId::new(31);
14760 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
14761 let (cotenant_task, cotenant_bind) = relay_route_open(
14762 &handler,
14763 cotenant,
14764 &cotenant_ctx.egress,
14765 &mut swap.incumbent_rx,
14766 100,
14767 "aft",
14768 "swap-cotenant",
14769 )
14770 .await;
14771 handler
14772 .handle_control_frame(
14773 &swap.incumbent_ctx,
14774 route_bind_ack(cotenant_bind.header.corr),
14775 )
14776 .await
14777 .unwrap();
14778 assert!(cotenant_task.await.unwrap().is_empty());
14779 let (cotenant_channel, cotenant_epoch) =
14780 published_route(&cotenant_rx.recv().await.unwrap());
14781
14782 let caller = ConnectionId::new(32);
14784 let (caller_ctx, mut caller_rx) = route_ctx(caller);
14785 let (caller_task, caller_bind) = relay_route_open(
14786 &handler,
14787 caller,
14788 &caller_ctx.egress,
14789 &mut swap.incumbent_rx,
14790 101,
14791 "aft",
14792 "swap-caller",
14793 )
14794 .await;
14795 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
14796
14797 register_candidate(&swap, None);
14798 cutover(&swap);
14799
14800 let ack = handler
14802 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
14803 .await;
14804 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
14805 if module_loop_error.is_some() {
14806 handler.cleanup_connection(INCUMBENT).unwrap();
14809 }
14810
14811 assert!(
14813 cotenant_rx.try_recv().is_err(),
14814 "the co-tenant route on the incumbent was torn down by one late ack: \
14815 {module_loop_error:?}"
14816 );
14817 assert!(matches!(
14818 swap.forwarding
14819 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14820 .unwrap(),
14821 DataRoute::Client(DataRouteState::Bound(_))
14822 ));
14823 assert_eq!(module_loop_error, None);
14824 assert!(swap
14825 .registry
14826 .get_module_by_connection(INCUMBENT)
14827 .unwrap()
14828 .is_some());
14829
14830 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
14832 .await
14833 .expect("the incumbent is told to drop the abandoned binding")
14834 .unwrap()
14835 .frame;
14836 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14837 assert_eq!(goodbye.header.channel, abandoned_channel);
14838 assert_eq!(goodbye.header.epoch, abandoned_epoch);
14839 assert!(swap.incumbent_rx.try_recv().is_err());
14840
14841 let response = caller_task.await.unwrap();
14843 assert_eq!(response.len(), 1);
14844 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14845 assert!(caller_rx.try_recv().is_err());
14846
14847 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
14851 let counters = handler.counters().snapshot();
14852 assert_eq!(
14853 keyed_total(&counters, "route_open_accepted_by_principal"),
14854 1
14855 );
14856 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
14857 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
14858 }
14859
14860 #[tokio::test]
14864 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
14865 let mut swap = swap_with_incumbent().await;
14866 register_candidate(&swap, None);
14867 let incumbent = cutover(&swap);
14868 swap.forwarding
14869 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
14870 .unwrap()
14871 .expect("the incumbent is still registered");
14872
14873 let client = ConnectionId::new(33);
14874 let (client_ctx, mut client_rx) = route_ctx(client);
14875 let route_handler = swap.handler.clone();
14876 let open_ctx = RouteCtx {
14877 connection_id: client,
14878 egress: client_ctx.egress.clone(),
14879 };
14880 let mut route_task = tokio::spawn(async move {
14881 route_handler
14882 .handle_control_frame(
14883 &open_ctx,
14884 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
14885 )
14886 .await
14887 .unwrap()
14888 });
14889 let bind = tokio::select! {
14890 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
14891 response = &mut route_task => {
14892 let response = response.unwrap();
14893 panic!(
14894 "post-cutover route.open was refused instead of relayed to the candidate: {}",
14895 parse_error(&response[0])["code"]
14896 );
14897 }
14898 };
14899 swap.handler
14900 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
14901 .await
14902 .unwrap();
14903 assert!(route_task.await.unwrap().is_empty());
14904 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
14905 match swap
14906 .forwarding
14907 .lookup_data_route(client, channel, epoch)
14908 .unwrap()
14909 {
14910 DataRoute::Client(DataRouteState::Bound(route)) => {
14911 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
14912 }
14913 other => panic!("expected a bound route on the candidate, got {other:?}"),
14914 }
14915 assert!(swap.incumbent_rx.try_recv().is_err());
14916 }
14917
14918 #[tokio::test]
14923 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
14924 let swap = swap_with_incumbent().await;
14925 register_candidate(&swap, Some(false));
14926 let update = Frame::build(
14927 FrameType::Request,
14928 control_flags(),
14929 0,
14930 0,
14931 55,
14932 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14933 provides: manifest("aft", PROTOCOL_VERSION).provides,
14934 capabilities: None,
14935 ready: Some(true),
14936 })
14937 .unwrap(),
14938 )
14939 .unwrap();
14940
14941 let replies = swap
14942 .handler
14943 .handle_control_frame(&swap.candidate_ctx, update)
14944 .await
14945 .unwrap();
14946
14947 assert_eq!(replies.len(), 1);
14948 assert_eq!(
14949 replies[0].header.ty,
14950 FrameType::Response,
14951 "candidate catalog.update was refused: {:?}",
14952 serde_json::from_slice::<Value>(&replies[0].body).ok()
14953 );
14954 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
14955 assert_eq!(
14956 swap.registry
14957 .get_module("aft")
14958 .unwrap()
14959 .unwrap()
14960 .connection_id,
14961 INCUMBENT
14962 );
14963 }
14964 }
14965
14966 mod swap_admission {
14970 use super::*;
14971
14972 const INCUMBENT_NONCE: &str = "incumbent-nonce";
14973 const CANDIDATE_NONCE: &str = "candidate-nonce";
14974
14975 fn handler_with_incumbent(
14976 module_id: &str,
14977 reserved: bool,
14978 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
14979 let registry = Arc::new(Registry::default());
14980 let supervisor = SupervisorHandle::new();
14981 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
14982 if reserved {
14983 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
14984 }
14985 let handler =
14986 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
14987 let incumbent = handler
14988 .handle_control(
14989 ConnectionId::new(1),
14990 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
14991 )
14992 .unwrap();
14993 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
14994 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
14995 (registry, supervisor, handler)
14996 }
14997
14998 #[test]
15004 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
15005 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
15006
15007 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
15008 let replies = handler
15009 .handle_control(
15010 ConnectionId::new(connection),
15011 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
15012 )
15013 .unwrap();
15014 assert_eq!(replies[0].header.ty, FrameType::Error);
15015 assert_eq!(
15016 parse_error(&replies[0])["code"],
15017 "swap_token_invalid",
15018 "nonce {nonce:?}"
15019 );
15020 }
15021 assert!(registry.get_candidate("aft").unwrap().is_none());
15022 assert_eq!(
15023 registry.get_module("aft").unwrap().unwrap().connection_id,
15024 ConnectionId::new(1)
15025 );
15026
15027 let admitted = handler
15030 .handle_control(
15031 ConnectionId::new(4),
15032 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
15033 )
15034 .unwrap();
15035 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
15036 assert_eq!(
15037 registry
15038 .get_candidate("aft")
15039 .unwrap()
15040 .unwrap()
15041 .connection_id,
15042 ConnectionId::new(4)
15043 );
15044 assert_eq!(
15045 registry.get_module("aft").unwrap().unwrap().connection_id,
15046 ConnectionId::new(1),
15047 "the candidate must not take the active slot"
15048 );
15049 let replayed = handler
15050 .handle_control(
15051 ConnectionId::new(5),
15052 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
15053 )
15054 .unwrap();
15055 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
15056
15057 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
15061 let squatter = handler
15062 .handle_control(
15063 ConnectionId::new(6),
15064 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
15065 )
15066 .unwrap();
15067 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
15068 assert!(
15069 registry.get_module("aft").unwrap().is_none(),
15070 "a squatter took the active slot of an id being swapped"
15071 );
15072 }
15073
15074 #[test]
15079 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
15080 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
15081
15082 let replies = handler
15083 .handle_control(
15084 ConnectionId::new(2),
15085 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15086 )
15087 .unwrap();
15088
15089 assert_eq!(
15090 replies[0].header.ty,
15091 FrameType::HelloAck,
15092 "reserved candidate refused: {:?}",
15093 serde_json::from_slice::<Value>(&replies[0].body).ok()
15094 );
15095 assert_eq!(
15096 registry
15097 .get_candidate("vault")
15098 .unwrap()
15099 .unwrap()
15100 .connection_id,
15101 ConnectionId::new(2)
15102 );
15103 }
15104
15105 #[test]
15108 fn without_an_open_swap_the_ordinary_gates_decide() {
15109 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
15110 supervisor.close_swap("vault");
15111
15112 let candidate = handler
15113 .handle_control(
15114 ConnectionId::new(2),
15115 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15116 )
15117 .unwrap();
15118 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
15119 let duplicate = handler
15120 .handle_control(
15121 ConnectionId::new(3),
15122 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
15123 )
15124 .unwrap();
15125 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15126 assert!(registry.get_candidate("vault").unwrap().is_none());
15127 }
15128 }
15129
15130 mod scopes {
15134 mod idle_tests {
15135 use super::*;
15136 include!("scope_idle_tests.rs");
15137 }
15138 mod route_open_timings {
15139 use super::*;
15140 include!("route_open_timing_tests.rs");
15141 }
15142 mod activity {
15143 include!("activity_scope_tests.rs");
15144 }
15145 mod delta {
15146 include!("scope_handler_delta_tests.rs");
15147 }
15148 mod call_modes {
15149 include!("scope_call_mode_tests.rs");
15150 }
15151 #[tokio::test]
15152 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
15153 let rig = rig().await;
15154 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
15155 "s".into(),
15156 1,
15157 "tool".into(),
15158 ))
15159 .unwrap();
15160 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
15161 let (direct, _rx) = wide_ctx(999);
15162 for (ctx, expected) in [
15163 (&rig.owner, FrameType::Response),
15164 (&direct, FrameType::Error),
15165 ] {
15166 let frame =
15167 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
15168 .unwrap();
15169 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
15170 assert_eq!(replies[0].header.ty, expected);
15171 if expected == FrameType::Response {
15172 let reply: subc_control::ScopeServedByReply =
15173 serde_json::from_slice(&replies[0].body).unwrap();
15174 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
15175 } else {
15176 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
15177 }
15178 }
15179 let reply = call(
15180 &rig.handler,
15181 &rig.owner,
15182 &ModuleControlRequestFromModule::ScopeDescribe {
15183 owner: Principal::Reserved {
15184 module_id: OWNER.into(),
15185 },
15186 scope_ref: "s".into(),
15187 },
15188 )
15189 .await;
15190 assert!(matches!(
15191 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
15192 ModuleControlResponseToModule::ScopeDescribe { .. }
15193 ));
15194 }
15195
15196 use subc_protocol::scope::{
15197 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
15198 ScopeStatus,
15199 };
15200
15201 use super::*;
15202
15203 const OWNER: &str = "prefrontal-core";
15204
15205 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
15206 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
15207 }
15208
15209 async fn call(
15210 handler: &ControlHandler,
15211 ctx: &RouteCtx,
15212 request: &ModuleControlRequestFromModule,
15213 ) -> Frame {
15214 let body = serde_json::to_vec(request).unwrap();
15215 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
15216 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
15217 assert_eq!(replies.len(), 1, "{replies:?}");
15218 replies.pop().unwrap()
15219 }
15220
15221 async fn sync(
15222 handler: &ControlHandler,
15223 ctx: &RouteCtx,
15224 generation: u64,
15225 scopes: Vec<ScopeRecord>,
15226 ) -> Result<ModuleControlResponseToModule, String> {
15227 let reply = call(
15228 handler,
15229 ctx,
15230 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
15231 )
15232 .await;
15233 match reply.header.ty {
15234 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
15235 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
15236 }
15237 }
15238
15239 async fn describe(
15240 handler: &ControlHandler,
15241 ctx: &RouteCtx,
15242 owner: &str,
15243 scope_ref: &str,
15244 ) -> ModuleControlResponseToModule {
15245 let reply = call(
15246 handler,
15247 ctx,
15248 &ModuleControlRequestFromModule::ScopeDescribe {
15249 owner: Principal::Reserved {
15250 module_id: owner.to_string(),
15251 },
15252 scope_ref: scope_ref.to_string(),
15253 },
15254 )
15255 .await;
15256 assert_eq!(
15257 reply.header.ty,
15258 FrameType::Response,
15259 "{:?}",
15260 parse_error(&reply)
15261 );
15262 serde_json::from_slice(&reply.body).unwrap()
15263 }
15264
15265 #[tokio::test]
15266 async fn scope_describe_never_echoes_bind_proof_or_hash() {
15267 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15268 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15269 let mut rig = rig().await;
15270 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15271 .await;
15272 let _bound = rig
15273 .bound(
15274 Some(AFT),
15275 PLEXUS,
15276 Some(
15277 rig.selector("s", Some(1))
15278 .with_bind_proof(Some(PROOF.into())),
15279 ),
15280 )
15281 .await;
15282 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
15283 let output = serde_json::to_string(&reply).unwrap();
15284 assert!(output.contains("\"status\":\"live\""));
15285 assert!(!output.contains(PROOF));
15286 assert!(!output.contains(HASH));
15287 }
15288
15289 async fn module(
15291 handler: &ControlHandler,
15292 connection: u64,
15293 module_id: &str,
15294 nonce: Option<&str>,
15295 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15296 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
15297 hello_via_sink(
15298 handler,
15299 &ctx,
15300 &mut rx,
15301 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
15302 )
15303 .await;
15304 (ctx, rx)
15305 }
15306
15307 #[tokio::test]
15310 async fn a_client_connection_cannot_sync_or_describe() {
15311 let handler = ControlHandler::new(Arc::new(Registry::default()));
15312 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
15313 for request in [
15314 ModuleControlRequestFromModule::ScopeSync {
15315 generation: 1,
15316 scopes: vec![head("s", 1)],
15317 },
15318 ModuleControlRequestFromModule::ScopeDescribe {
15319 owner: Principal::Direct,
15320 scope_ref: "s".to_string(),
15321 },
15322 ] {
15323 let reply = call(&handler, &ctx, &request).await;
15324 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
15325 }
15326 assert!(
15327 !handler
15328 .scopes
15329 .read()
15330 .unwrap()
15331 .describe(
15332 &Principal::Reserved {
15333 module_id: OWNER.to_string()
15334 },
15335 "s"
15336 )
15337 .owner_synced
15338 );
15339 }
15340
15341 #[tokio::test]
15344 async fn a_module_without_a_supervised_launch_cannot_sync() {
15345 let handler = ControlHandler::new(Arc::new(Registry::default()));
15346 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
15347 assert_eq!(
15348 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
15349 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15350 );
15351 }
15352
15353 #[tokio::test]
15354 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
15355 let supervisor = SupervisorHandle::new();
15356 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15357 let handler = ControlHandler::new(Arc::new(Registry::default()))
15358 .with_supervisor(supervisor.clone());
15359 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15360 sync(&handler, &incumbent, 1, vec![head("s", 1)])
15361 .await
15362 .expect("the current launch syncs");
15363
15364 supervisor.open_swap(OWNER, "n2".to_string());
15367 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
15368 assert_eq!(
15369 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15370 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15371 );
15372 sync(&handler, &incumbent, 2, vec![head("s", 1)])
15373 .await
15374 .expect("the serving owner syncs during the swap");
15375
15376 supervisor.close_swap(OWNER);
15379 assert_eq!(
15380 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15381 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15382 );
15383 sync(&handler, &incumbent, 3, vec![head("s", 1)])
15384 .await
15385 .expect("the serving owner syncs after the rollback");
15386 handler.cleanup_connection(candidate.connection_id).unwrap();
15387
15388 supervisor.open_swap(OWNER, "n3".to_string());
15393 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
15394 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
15395 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
15396 .await
15397 .expect("the promoted launch takes authority");
15398 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15399 panic!("unexpected reply {reply:?}");
15400 };
15401 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
15402 assert_eq!(
15403 sync(&handler, &incumbent, 4, Vec::new()).await,
15404 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15405 );
15406 }
15407
15408 #[tokio::test]
15411 async fn closing_the_authority_connection_frees_sync_authority() {
15412 let supervisor = SupervisorHandle::new();
15413 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15414 let handler = ControlHandler::new(Arc::new(Registry::default()))
15415 .with_supervisor(supervisor.clone());
15416 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15417 sync(&handler, &first, 10, vec![head("s", 1)])
15418 .await
15419 .unwrap();
15420 handler.cleanup_connection(first.connection_id).unwrap();
15421
15422 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
15423 sync(&handler, &second, 1, vec![head("s", 1)])
15424 .await
15425 .expect("the next connection takes the released authority");
15426 }
15427
15428 #[tokio::test]
15429 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
15430 let supervisor = SupervisorHandle::new();
15431 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15432 let handler =
15433 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
15434 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
15435 sync(&handler, &first, 10, vec![head("s", 1)])
15436 .await
15437 .unwrap();
15438 handler
15439 .handle_control_frame(
15440 &first,
15441 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
15442 )
15443 .await
15444 .unwrap();
15445 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
15446 sync(&handler, &second, 1, vec![head("s", 1)])
15447 .await
15448 .expect("GOODBYE releases authority even if the old socket remains open");
15449 }
15450
15451 #[tokio::test]
15452 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
15453 let registry = Arc::new(Registry::default());
15454 let supervisor_handle = SupervisorHandle::new();
15455 let supervisor =
15456 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15457 .with_handle(supervisor_handle.clone())
15458 .with_daemon_incarnation("incarnation-7".to_string());
15459 supervisor
15462 .supervise_configured(
15463 ModuleSpec {
15464 module_id: OWNER.to_string(),
15465 program: PathBuf::from("/nonexistent/prefrontal-core"),
15466 args: Vec::new(),
15467 env: Vec::new(),
15468 reserved: false,
15469 reserved_prefixes: Vec::new(),
15470 protocol: ModuleProtocol::Subc,
15471 overlap: Default::default(),
15472 },
15473 false,
15474 )
15475 .unwrap();
15476 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
15477 let handler =
15478 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15479 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
15480
15481 let ModuleControlResponseToModule::ScopeDescribe {
15483 status,
15484 daemon_incarnation,
15485 owner_synced,
15486 owner_configured,
15487 scope,
15488 ..
15489 } = describe(&handler, &reader, OWNER, "s").await
15490 else {
15491 panic!("not a describe reply");
15492 };
15493 assert_eq!(status, ScopeStatus::NotLive);
15494 assert_eq!(daemon_incarnation, "incarnation-7");
15495 assert!(!owner_synced);
15496 assert!(owner_configured);
15497 assert!(scope.is_none());
15498
15499 let ModuleControlResponseToModule::ScopeDescribe {
15502 status,
15503 owner_configured,
15504 ..
15505 } = describe(&handler, &reader, "ghost", "s").await
15506 else {
15507 panic!("not a describe reply");
15508 };
15509 assert_eq!(status, ScopeStatus::NotLive);
15510 assert!(!owner_configured);
15511
15512 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
15514 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
15515 let ModuleControlResponseToModule::ScopeDescribe {
15516 status,
15517 scope_epoch,
15518 owner_synced,
15519 scope,
15520 ..
15521 } = describe(&handler, &reader, OWNER, "s").await
15522 else {
15523 panic!("not a describe reply");
15524 };
15525 assert_eq!(status, ScopeStatus::Live);
15526 assert_eq!(scope_epoch, Some(4));
15527 assert!(owner_synced);
15528 let stamp = scope.expect("a live scope carries its stamp");
15529 assert!(
15530 stamp.owner_authorized,
15531 "prefrontal-core is the default authority"
15532 );
15533 assert_eq!(stamp.kind, ScopeKind::Head);
15534 }
15535
15536 #[tokio::test]
15537 async fn scope_authority_owners_decides_owner_authorized() {
15538 let supervisor = SupervisorHandle::new();
15539 supervisor.set_spawn_nonce("broca", "b1".to_string());
15540 let handler = ControlHandler::new(Arc::new(Registry::default()))
15541 .with_supervisor(supervisor)
15542 .with_scope_authority_owners(vec!["broca".to_string()]);
15543 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
15544 let mut gated = head("s", 1);
15545 gated.attributes.agent_id = Some("agent".to_string());
15546 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
15547 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
15548 describe(&handler, &broca, "broca", "s").await
15549 else {
15550 panic!("not a describe reply");
15551 };
15552 assert!(scope.unwrap().owner_authorized);
15553 }
15554
15555 #[tokio::test]
15559 async fn scope_ops_and_the_scopes_capability_are_advertised() {
15560 let handler = ControlHandler::new(Arc::new(Registry::default()));
15561 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
15562 let ack = hello_via_sink(
15563 &handler,
15564 &ctx,
15565 &mut rx,
15566 hello_frame("m", PROTOCOL_VERSION, 1),
15567 )
15568 .await;
15569 let ack = parse_ack(&ack);
15570 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
15571 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
15572 }
15573 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
15574
15575 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
15576 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15577 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15578 let reply = handler
15579 .handle_control_frame(&client, frame)
15580 .await
15581 .unwrap()
15582 .pop()
15583 .unwrap();
15584 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15585 serde_json::from_slice(&reply.body).unwrap()
15586 else {
15587 panic!("not a server.describe reply");
15588 };
15589 assert!(
15590 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
15591 "{capabilities:?}"
15592 );
15593 }
15594
15595 const PLEXUS: &str = "plexus";
15598 const OTHER: &str = "other";
15599 const AFT: &str = "aft";
15600 const BROCA: &str = "broca";
15601 const MAGIC: &str = "magic-context";
15602
15603 fn nonce(module_id: &str) -> String {
15604 format!("nonce-{module_id}")
15605 }
15606
15607 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15608 let (tx, rx) = mpsc::channel(64);
15609 (
15610 RouteCtx {
15611 connection_id: ConnectionId::new(connection),
15612 egress: FrameSink::new(tx),
15613 },
15614 rx,
15615 )
15616 }
15617
15618 struct Rig {
15622 handler: ControlHandler,
15623 forwarding: Arc<ForwardingTable>,
15624 owner: RouteCtx,
15625 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15626 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
15627 generation: u64,
15628 next_connection: u64,
15629 _supervisor: Supervisor,
15630 }
15631
15632 async fn rig() -> Rig {
15633 rig_with_flow_support(true).await
15634 }
15635
15636 async fn rig_with_flow_support(flow_support: bool) -> Rig {
15637 rig_with_scope_capabilities(flow_support, false).await
15638 }
15639
15640 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
15641 let mut provides = Vec::new();
15642 if flow_support {
15643 provides.push("flow-scopes/v1");
15644 }
15645 if run_support {
15646 provides.push("agent-run-scopes/v1");
15647 }
15648 rig_with_target_capabilities(&provides).await
15649 }
15650
15651 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
15652 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
15653 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15654 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15657 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
15658 Frame::build(
15659 FrameType::Hello,
15660 control_flags(),
15661 0,
15662 0,
15663 connection,
15664 serde_json::to_vec(&body).unwrap(),
15665 )
15666 .unwrap()
15667 }
15668
15669 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
15670 let registry = Arc::new(Registry::default());
15671 let forwarding = Arc::new(ForwardingTable::default());
15672 let supervisor_handle = SupervisorHandle::new();
15673 let supervisor =
15674 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15675 .with_handle(supervisor_handle.clone());
15676 supervisor
15677 .supervise_configured(
15678 ModuleSpec {
15679 module_id: OWNER.to_string(),
15680 program: PathBuf::from("/nonexistent/prefrontal-core"),
15681 args: Vec::new(),
15682 env: Vec::new(),
15683 reserved: false,
15684 reserved_prefixes: Vec::new(),
15685 protocol: ModuleProtocol::Subc,
15686 overlap: Default::default(),
15687 },
15688 false,
15689 )
15690 .unwrap();
15691 for module_id in [OWNER, AFT, BROCA, MAGIC] {
15692 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
15693 }
15694 let handler =
15695 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15696 .with_supervisor(supervisor_handle);
15697 let (owner, mut owner_rx) = wide_ctx(1);
15698 hello_via_sink(
15699 &handler,
15700 &owner,
15701 &mut owner_rx,
15702 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
15703 )
15704 .await;
15705 let mut modules = BTreeMap::new();
15706 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
15707 let (ctx, mut rx) = wide_ctx(connection);
15708 let hello = hello_with_capabilities(module_id, connection, provides);
15709 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
15710 modules.insert(module_id.to_string(), (ctx, rx));
15711 }
15712 Rig {
15713 handler,
15714 forwarding,
15715 owner,
15716 _owner_rx: owner_rx,
15717 modules,
15718 generation: 0,
15719 next_connection: 100,
15720 _supervisor: supervisor,
15721 }
15722 }
15723
15724 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
15725 ScopeCarrier::new(Principal::Reserved {
15726 module_id: module_id.to_string(),
15727 })
15728 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
15729 }
15730
15731 fn session(scope_epoch: u64) -> ScopeRecord {
15734 let mut record = head("s", scope_epoch);
15735 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
15736 record.attributes.agent_id = Some("agent-1".to_string());
15737 record.attributes.delegates = true;
15738 record
15739 }
15740
15741 impl Rig {
15742 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
15743 self.generation += 1;
15744 sync(&self.handler, &self.owner, self.generation, scopes)
15745 .await
15746 .expect("the owner's sync is accepted");
15747 }
15748
15749 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15750 ScopeSelector::new(
15751 Principal::Reserved {
15752 module_id: OWNER.to_string(),
15753 },
15754 scope_ref.to_string(),
15755 scope_epoch,
15756 )
15757 }
15758
15759 fn open_frame(
15760 &mut self,
15761 opener: Option<&str>,
15762 target: &str,
15763 scope: Option<ScopeSelector>,
15764 ) -> (
15765 RouteCtx,
15766 mpsc::Receiver<crate::router::OutboundFrame>,
15767 Frame,
15768 ) {
15769 self.next_connection += 1;
15770 let (ctx, rx) = wide_ctx(self.next_connection);
15771 let root = unique_project_root("scoped-open");
15772 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15773 target: RouteTarget::ToolProvider {
15774 module_id: target.to_string(),
15775 },
15776 identity: BindIdentity::new(
15777 root.path().to_path_buf(),
15778 "unit".to_string(),
15779 "session".to_string(),
15780 ),
15781 consumer_identity: opener.map(|module_id| ConsumerIdentity {
15782 module_id: module_id.to_string(),
15783 launch_nonce: nonce(module_id),
15784 }),
15785 consumer_capabilities: None,
15786 role_versions: None,
15787 admission_facts: None,
15788 scope,
15789 })
15790 .unwrap();
15791 let frame = Frame::build(
15792 FrameType::Request,
15793 control_flags(),
15794 0,
15795 0,
15796 self.next_connection,
15797 body,
15798 )
15799 .unwrap();
15800 (ctx, rx, frame)
15801 }
15802
15803 async fn refused(
15805 &mut self,
15806 opener: Option<&str>,
15807 target: &str,
15808 scope: Option<ScopeSelector>,
15809 ) -> String {
15810 self.refusal_body(opener, target, scope).await["code"]
15811 .as_str()
15812 .unwrap()
15813 .to_string()
15814 }
15815
15816 async fn refusal_body(
15817 &mut self,
15818 opener: Option<&str>,
15819 target: &str,
15820 scope: Option<ScopeSelector>,
15821 ) -> Value {
15822 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
15823 let replies = tokio::time::timeout(
15824 Duration::from_secs(2),
15825 self.handler.handle_control_frame(&ctx, frame),
15826 )
15827 .await
15828 .expect("the open must be refused before waiting for a bind ack")
15829 .unwrap();
15830 assert_eq!(replies.len(), 1, "{replies:?}");
15831 assert_eq!(replies[0].header.ty, FrameType::Error);
15832 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15833 assert!(
15834 module_rx.try_recv().is_err(),
15835 "a refused open relays nothing"
15836 );
15837 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
15838 parse_error(&replies[0])
15839 }
15840
15841 async fn relayed(
15843 &mut self,
15844 opener: Option<&str>,
15845 target: &str,
15846 scope: Option<ScopeSelector>,
15847 ) -> Relayed {
15848 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
15849 let handler = self.handler.clone();
15850 let task_ctx = ctx.clone();
15851 let task = tokio::spawn(async move {
15852 handler
15853 .handle_control_frame(&task_ctx, frame)
15854 .await
15855 .unwrap()
15856 });
15857 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15858 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
15859 .await
15860 .expect("the target receives the relayed route.bind")
15861 .unwrap()
15862 .frame;
15863 Relayed {
15864 target: target.to_string(),
15865 client: ctx,
15866 client_rx: rx,
15867 task,
15868 bind,
15869 }
15870 }
15871
15872 async fn ack(&self, relayed: &Relayed) {
15873 let (module, _) = &self.modules[&relayed.target];
15874 self.handler
15875 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
15876 .await
15877 .unwrap();
15878 }
15879
15880 async fn bound(
15882 &mut self,
15883 opener: Option<&str>,
15884 target: &str,
15885 scope: Option<ScopeSelector>,
15886 ) -> Bound {
15887 let relayed = self.relayed(opener, target, scope).await;
15888 self.ack(&relayed).await;
15889 let Relayed {
15890 target,
15891 client,
15892 mut client_rx,
15893 task,
15894 bind,
15895 } = relayed;
15896 assert!(
15897 task.await.unwrap().is_empty(),
15898 "the open is answered by commit"
15899 );
15900 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
15901 Bound {
15902 target,
15903 client,
15904 client_rx,
15905 channel,
15906 epoch,
15907 bind,
15908 }
15909 }
15910
15911 fn live(&self, route: &Bound) -> bool {
15912 matches!(
15913 self.forwarding
15914 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15915 .unwrap(),
15916 DataRoute::Client(DataRouteState::Bound(_))
15917 )
15918 }
15919 }
15920
15921 struct Relayed {
15922 target: String,
15923 client: RouteCtx,
15924 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15925 task: tokio::task::JoinHandle<Vec<Frame>>,
15926 bind: Frame,
15927 }
15928
15929 struct Bound {
15930 target: String,
15931 client: RouteCtx,
15932 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15933 channel: u16,
15934 epoch: u32,
15935 bind: Frame,
15936 }
15937
15938 impl Bound {
15939 fn closed_reason(&mut self) -> RouteCloseReason {
15942 let mut reason = None;
15943 let mut goodbye = false;
15944 while let Ok(outbound) = self.client_rx.try_recv() {
15945 let frame = outbound.frame;
15946 match frame.header.ty {
15947 FrameType::Goodbye => {
15948 assert_eq!(
15949 (frame.header.channel, frame.header.epoch),
15950 (self.channel, self.epoch)
15951 );
15952 goodbye = true;
15953 }
15954 FrameType::Push => {
15955 let ClientControlPush::RouteClosed {
15956 reason: r,
15957 module_id,
15958 ..
15959 } = serde_json::from_slice(&frame.body).unwrap()
15960 else {
15961 panic!("unexpected push");
15962 };
15963 assert_eq!(module_id, self.target);
15964 reason = Some(r);
15965 }
15966 other => panic!("unexpected frame {other:?}"),
15967 }
15968 }
15969 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
15970 reason.expect("the client is told why the route closed")
15971 }
15972
15973 fn untouched(&mut self) -> bool {
15974 self.client_rx.try_recv().is_err()
15975 }
15976
15977 fn stamp(&self) -> Option<ScopeStamp> {
15978 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
15979 ModuleControlRequest::RouteBind { scope, .. } => scope,
15980 other => panic!("expected a route.bind, got {other:?}"),
15981 }
15982 }
15983 }
15984
15985 #[tokio::test]
15986 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
15987 ) {
15988 let mut rig = rig().await;
15989 let mut record = session(1);
15990 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
15991 record.child_owners = vec![Principal::Reserved {
15992 module_id: MAGIC.to_string(),
15993 }];
15994 rig.sync(vec![record]).await;
15995 let scope = || Some(rig_selector("s", Some(1)));
15996
15997 rig.bound(Some(OWNER), PLEXUS, scope()).await;
16000 rig.bound(Some(AFT), OTHER, scope()).await;
16001 rig.bound(Some(BROCA), PLEXUS, scope()).await;
16002
16003 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
16007 assert_eq!(
16008 rig.refused(opener, target, scope()).await,
16009 error_codes::SCOPE_NOT_CARRIER,
16010 "{opener:?} -> {target}"
16011 );
16012 }
16013 }
16014
16015 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16016 ScopeSelector::new(
16017 Principal::Reserved {
16018 module_id: OWNER.to_string(),
16019 },
16020 scope_ref.to_string(),
16021 scope_epoch,
16022 )
16023 }
16024
16025 #[tokio::test]
16026 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
16027 let mut rig = rig().await;
16028 rig.sync(vec![session(1)]).await;
16029 for opener in [OWNER, AFT] {
16030 assert_eq!(
16031 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
16032 .await,
16033 error_codes::SCOPE_EPOCH_REQUIRED,
16034 "{opener}"
16035 );
16036 }
16037 }
16038
16039 #[tokio::test]
16040 async fn admission_separates_not_synced_not_live_and_ended() {
16041 let mut rig = rig().await;
16042 let code = rig
16044 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16045 .await;
16046 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
16047 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
16048
16049 let ghost = ScopeSelector::new(
16051 Principal::Reserved {
16052 module_id: "ghost".to_string(),
16053 },
16054 "s".to_string(),
16055 Some(1),
16056 );
16057 assert_eq!(
16058 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
16059 error_codes::SCOPE_NOT_LIVE
16060 );
16061
16062 rig.sync(vec![session(2)]).await;
16063 assert_eq!(
16064 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
16065 .await,
16066 error_codes::SCOPE_NOT_LIVE
16067 );
16068 for epoch in [1, 3] {
16069 assert_eq!(
16070 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
16071 .await,
16072 error_codes::SCOPE_ENDED,
16073 "epoch {epoch}"
16074 );
16075 }
16076 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
16078 .await;
16079 }
16080
16081 #[tokio::test]
16082 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
16083 let mut rig = rig().await;
16084 rig.sync(vec![session(1)]).await;
16085 let route = rig
16086 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16087 .await;
16088 let stamp = route.stamp().expect("a scoped bind carries the stamp");
16089 assert_eq!(stamp.scope_ref, "s");
16090 assert_eq!(stamp.scope_epoch, 1);
16091 assert_eq!(stamp.kind, ScopeKind::Head);
16092 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
16093 assert!(stamp.attributes.delegates);
16094 assert!(stamp.owner_authorized);
16095 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16096 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
16097
16098 let (broca, mut broca_rx) = wide_ctx(50);
16101 hello_via_sink(
16102 &rig.handler,
16103 &broca,
16104 &mut broca_rx,
16105 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
16106 )
16107 .await;
16108 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
16109 .await
16110 .unwrap();
16111 let own = ScopeSelector::new(
16112 Principal::Reserved {
16113 module_id: BROCA.to_string(),
16114 },
16115 "b".to_string(),
16116 Some(1),
16117 );
16118 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
16119 assert!(!route.stamp().unwrap().owner_authorized);
16120 }
16121
16122 #[tokio::test]
16123 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
16124 let mut rig = rig().await;
16125 let mut record = head("s", 1);
16126 record.carriers = vec![carrier(AFT, None)];
16127 let flow_id = "Flow:run-7/step_2!~";
16128 record.attributes.flow_id = Some(flow_id.to_string());
16129 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
16130 .await
16131 .unwrap();
16132 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16133 panic!("not a sync reply");
16134 };
16135 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
16136 let route = rig
16137 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16138 .await;
16139 assert!(rig.live(&route), "the stamped bind committed");
16140 let stamp = route.stamp().expect("a flow scope carries a stamp");
16141 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
16142 assert_eq!(stamp.attributes.agent_id, None);
16143 assert!(!stamp.attributes.delegates);
16144 assert!(stamp.owner_authorized);
16145 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16146 assert_eq!(unscoped.stamp(), None);
16147 }
16148
16149 #[tokio::test]
16150 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
16151 let mut rig = rig_with_flow_support(false).await;
16152 let mut record = session(1);
16153 record.attributes.flow_id = Some("flow:7".to_string());
16154 rig.sync(vec![record]).await;
16155 for opener in [OWNER, AFT] {
16156 let body = rig
16157 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16158 .await;
16159 assert_eq!(body["code"], "target_flow_unsupported");
16160 let message = body["message"].as_str().unwrap();
16161 for required in [PLEXUS, "flow-scopes/v1"] {
16162 assert!(message.contains(required), "{message}");
16163 }
16164 }
16165 }
16166
16167 #[tokio::test]
16168 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
16169 let mut rig = rig_with_flow_support(true).await;
16170 let mut record = session(1);
16171 record.attributes.flow_id = Some("flow:7".to_string());
16172 rig.sync(vec![record]).await;
16173 for opener in [OWNER, AFT] {
16174 let route = rig
16175 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16176 .await;
16177 assert!(rig.live(&route));
16178 assert_eq!(
16179 route.stamp().unwrap().attributes.flow_id.as_deref(),
16180 Some("flow:7")
16181 );
16182 }
16183 }
16184
16185 #[tokio::test]
16186 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
16187 use std::future::Future;
16188
16189 let mut rig = rig_with_flow_support(true).await;
16190 let mut record = session(1);
16191 record.attributes.flow_id = Some("flow:7".to_string());
16192 rig.sync(vec![record]).await;
16193 let (client, mut client_rx, frame) =
16194 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
16195 for _ in 0..64 {
16198 client.egress.try_send(route_bind_ack(1)).unwrap();
16199 }
16200 let handler = rig.handler.clone();
16201 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16202 std::future::poll_fn(|cx| {
16203 assert!(open.as_mut().poll(cx).is_pending());
16204 std::task::Poll::Ready(())
16205 })
16206 .await;
16207 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16208
16209 let old_connection = rig.modules[PLEXUS].0.connection_id;
16210 rig.handler.cleanup_connection(old_connection).unwrap();
16211 let (replacement, mut replacement_rx) = wide_ctx(200);
16212 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
16213 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16214 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16215 let hello = Frame::build(
16216 FrameType::Hello,
16217 control_flags(),
16218 0,
16219 0,
16220 200,
16221 serde_json::to_vec(&body).unwrap(),
16222 )
16223 .unwrap();
16224 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
16225
16226 client_rx.try_recv().unwrap();
16227 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16228 .await
16229 .expect("the replacement is refused without waiting for a bind ack")
16230 .unwrap();
16231 assert_eq!(replies.len(), 1);
16232 let body = parse_error(&replies[0]);
16233 assert_eq!(body["code"], "target_flow_unsupported");
16234 for required in [PLEXUS, "flow-scopes/v1"] {
16235 assert!(body["message"].as_str().unwrap().contains(required));
16236 }
16237 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
16238 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16239 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16240 assert!(rig
16241 .handler
16242 .registry
16243 .get_module_by_connection(replacement.connection_id)
16244 .unwrap()
16245 .is_some());
16246 }
16247
16248 #[tokio::test]
16249 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
16250 use std::future::Future;
16251 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16252 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16253 const NEXT_HASH: &str =
16254 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16255 let mut rig = rig().await;
16256 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
16257 record.carriers.push(ScopeCarrier::new(Principal::Direct));
16258 rig.sync(vec![record.clone()]).await;
16259 let (client, mut rx, frame) = rig.open_frame(
16260 None,
16261 PLEXUS,
16262 Some(
16263 rig.selector("s", Some(1))
16264 .with_bind_proof(Some(PROOF.into())),
16265 ),
16266 );
16267 for _ in 0..64 {
16270 client.egress.try_send(route_bind_ack(1)).unwrap();
16271 }
16272 let handler = rig.handler.clone();
16273 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16274 std::future::poll_fn(|cx| {
16275 assert!(open.as_mut().poll(cx).is_pending());
16276 std::task::Poll::Ready(())
16277 })
16278 .await;
16279 record.bind_proof_sha256 = Some(NEXT_HASH.into());
16280 rig.sync(vec![record]).await;
16281 rx.try_recv().unwrap();
16282 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16283 .await
16284 .unwrap()
16285 .unwrap();
16286 assert_eq!(
16287 parse_error(&replies[0])["code"],
16288 error_codes::SCOPE_BIND_PROOF_MISMATCH
16289 );
16290 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
16291 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16292 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16293 }
16294
16295 #[tokio::test]
16296 async fn scope_bind_proof_reserved_openers_ignore_proof() {
16297 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16298 let mut rig = rig().await;
16299 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16300 .await;
16301 for opener in [OWNER, AFT] {
16302 for proof in [None, Some("malformed+proof=")] {
16303 let route = rig
16304 .bound(
16305 Some(opener),
16306 PLEXUS,
16307 Some(
16308 rig.selector("s", Some(1))
16309 .with_bind_proof(proof.map(str::to_string)),
16310 ),
16311 )
16312 .await;
16313 assert!(rig.live(&route));
16314 }
16315 }
16316 }
16317
16318 #[tokio::test]
16319 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
16320 use std::future::Future;
16321 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16322 const NEXT_HASH: &str =
16323 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16324 let mut rig = rig().await;
16325 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16326 .await;
16327 let (client, mut rx, frame) =
16328 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
16329 for _ in 0..64 {
16330 client.egress.try_send(route_bind_ack(1)).unwrap();
16331 }
16332 let handler = rig.handler.clone();
16333 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16334 std::future::poll_fn(|cx| {
16335 assert!(open.as_mut().poll(cx).is_pending());
16336 std::task::Poll::Ready(())
16337 })
16338 .await;
16339 rig.sync(vec![
16340 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
16341 ])
16342 .await;
16343 rx.try_recv().unwrap();
16344 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16345 let bind = std::future::poll_fn(|cx| {
16346 assert!(open.as_mut().poll(cx).is_pending());
16347 module_rx.poll_recv(cx)
16348 })
16349 .await
16350 .unwrap()
16351 .frame;
16352 handler
16353 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
16354 .await
16355 .unwrap();
16356 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16357 .await
16358 .unwrap()
16359 .unwrap();
16360 assert_eq!(replies.len(), 1);
16361 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
16364 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16365 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
16366 let route = rig
16367 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
16368 .await;
16369 assert!(rig.live(&route));
16370 }
16371
16372 #[tokio::test]
16373 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
16374 {
16375 let mut rig = rig_with_flow_support(false).await;
16376 rig.sync(vec![session(1)]).await;
16377 let route = rig
16378 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16379 .await;
16380 assert!(rig.live(&route));
16381 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
16382 let mut record = session(1);
16383 record.attributes.flow_id = Some("flow:7".to_string());
16384 rig.sync(vec![record]).await;
16385 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
16386 assert!(rig.live(&unscoped));
16387 assert_eq!(unscoped.stamp(), None);
16388 }
16389
16390 #[tokio::test]
16391 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
16392 let mut rig = rig().await;
16393 let mut record = session(1);
16394 record.attributes.flow_id = Some("flow:7".to_string());
16395 rig.sync(vec![record.clone()]).await;
16396 let before = rig.forwarding.published_scope_tag(OWNER, "s");
16397 let mut owner_route = rig
16398 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16399 .await;
16400 let mut carrier_route = rig
16401 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
16402 .await;
16403 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16404 rig.sync(vec![record.clone()]).await;
16405 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16406 assert!(rig.live(&owner_route) && owner_route.untouched());
16407 assert!(rig.live(&carrier_route) && carrier_route.untouched());
16408
16409 record.attributes.flow_id = Some("flow:8".to_string());
16410 rig.sync(vec![record]).await;
16411 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
16412 let before = before.unwrap();
16413 assert_eq!(after.scope_epoch, before.scope_epoch);
16414 assert!(after.version > before.version);
16415 for route in [&mut owner_route, &mut carrier_route] {
16416 assert!(!rig.live(route));
16417 assert_eq!(
16418 route.closed_reason(),
16419 RouteCloseReason::ScopeDelegationChanged
16420 );
16421 }
16422 assert!(rig.live(&unscoped) && unscoped.untouched());
16423 for target in [PLEXUS, OTHER] {
16426 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
16427 let goodbye = module_rx
16428 .try_recv()
16429 .expect("the provider sees the drain")
16430 .frame;
16431 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16432 assert!(module_rx.try_recv().is_err());
16433 }
16434 let rebound = rig
16435 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16436 .await;
16437 assert_eq!(
16438 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
16439 Some("flow:8")
16440 );
16441 }
16442
16443 #[tokio::test]
16448 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
16449 let mut rig = rig().await;
16450 rig.sync(vec![session(1)]).await;
16451 let mut cotenant = rig
16452 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16453 .await;
16454
16455 let mut changed = session(1);
16456 changed.child_owners.push(Principal::Reserved {
16457 module_id: MAGIC.to_string(),
16458 });
16459 let mut ended = None;
16460 for (code, next) in [
16461 (error_codes::SCOPE_CHANGED, vec![changed]),
16462 (error_codes::SCOPE_ENDED, Vec::new()),
16463 ] {
16464 let relayed = rig
16465 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16466 .await;
16467 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
16468 ended = Some(next.is_empty());
16469 rig.sync(next).await;
16470 rig.ack(&relayed).await;
16471 let replies = relayed.task.await.unwrap();
16472 assert_eq!(replies.len(), 1, "{replies:?}");
16473 assert_eq!(parse_error(&replies[0])["code"], code);
16474 assert_eq!(
16475 subc_protocol::error_codes::is_retryable_route_open(code),
16476 code == error_codes::SCOPE_CHANGED
16477 );
16478 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16479 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16481 let mut goodbyes = Vec::new();
16484 while let Ok(outbound) = plexus_rx.try_recv() {
16485 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
16486 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
16487 }
16488 assert!(
16489 goodbyes.contains(&(bind_channel, bind_epoch)),
16490 "{goodbyes:?}"
16491 );
16492 assert!(rig
16493 .handler
16494 .registry
16495 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
16496 .unwrap()
16497 .is_some());
16498 }
16499 assert_eq!(ended, Some(true));
16500 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
16503 }
16504
16505 #[tokio::test]
16508 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
16509 struct Case {
16510 name: &'static str,
16511 change: fn(&mut ScopeRecord),
16512 closed: [Option<RouteCloseReason>; 4],
16515 }
16516 use RouteCloseReason::*;
16517 let cases = [
16518 Case {
16519 name: "a carrier entry removed",
16520 change: |r| {
16521 r.carriers.retain(|c| {
16522 c.principal
16523 != Principal::Reserved {
16524 module_id: AFT.to_string(),
16525 }
16526 })
16527 },
16528 closed: [None, Some(ScopeCarrierRemoved), None, None],
16529 },
16530 Case {
16531 name: "a target removed from a carrier",
16532 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
16533 closed: [None, None, None, Some(ScopeCarrierRemoved)],
16534 },
16535 Case {
16536 name: "a bare carrier narrowed to targets",
16537 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
16538 closed: [None, Some(ScopeCarrierRemoved), None, None],
16539 },
16540 Case {
16541 name: "delegates turned off",
16542 change: |r| r.attributes.delegates = false,
16543 closed: [Some(ScopeDelegationChanged); 4],
16544 },
16545 Case {
16546 name: "agent_id changed",
16547 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
16548 closed: [Some(ScopeDelegationChanged); 4],
16549 },
16550 Case {
16551 name: "a carrier added, child owners changed, the record re-sent",
16552 change: |r| {
16553 r.carriers.push(carrier(MAGIC, None));
16554 r.child_owners.push(Principal::Reserved {
16555 module_id: MAGIC.to_string(),
16556 });
16557 },
16558 closed: [None; 4],
16559 },
16560 Case {
16561 name: "a target added",
16562 change: |r| {
16563 r.carriers[1]
16564 .targets
16565 .as_mut()
16566 .unwrap()
16567 .push("third".to_string())
16568 },
16569 closed: [None; 4],
16570 },
16571 Case {
16572 name: "delegates turned on",
16573 change: |r| r.attributes.delegates = true,
16574 closed: [None; 4],
16575 },
16576 ];
16577 for case in cases {
16578 let mut rig = rig().await;
16579 rig.sync(vec![session(1)]).await;
16580 let scope = || Some(rig_selector("s", Some(1)));
16581 let mut routes = [
16582 rig.bound(Some(OWNER), PLEXUS, scope()).await,
16583 rig.bound(Some(AFT), PLEXUS, scope()).await,
16584 rig.bound(Some(BROCA), PLEXUS, scope()).await,
16585 rig.bound(Some(BROCA), OTHER, scope()).await,
16586 ];
16587 let mut record = session(1);
16588 (case.change)(&mut record);
16589 rig.sync(vec![record]).await;
16590 for (index, expected) in case.closed.iter().enumerate() {
16591 let route = &mut routes[index];
16592 match expected {
16593 Some(reason) => {
16594 assert!(!rig.live(route), "{}: route {index} still live", case.name);
16595 assert_eq!(
16596 route.closed_reason(),
16597 *reason,
16598 "{}: route {index}",
16599 case.name
16600 );
16601 }
16602 None => {
16603 assert!(rig.live(route), "{}: route {index} closed", case.name);
16604 assert!(
16605 route.untouched(),
16606 "{}: route {index} was told something",
16607 case.name
16608 );
16609 }
16610 }
16611 }
16612 }
16613 }
16614
16615 #[tokio::test]
16616 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
16617 for next in [Vec::new(), vec![session(2)]] {
16619 let mut rig = rig().await;
16620 rig.sync(vec![session(1)]).await;
16621 let mut route = rig
16622 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16623 .await;
16624 rig.sync(next).await;
16625 assert!(!rig.live(&route));
16626 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
16627 }
16628
16629 let mut rig = rig().await;
16632 let mut child = session(1);
16633 child.scope_ref = "child".to_string();
16634 child.kind = ScopeKind::Worker;
16635 child.parent = Some(ScopeParent::new(
16636 Principal::Reserved {
16637 module_id: OWNER.to_string(),
16638 },
16639 "s".to_string(),
16640 1,
16641 ));
16642 rig.sync(vec![session(1), child.clone()]).await;
16643 let mut child_route = rig
16644 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
16645 .await;
16646 assert_eq!(
16647 child_route.stamp().unwrap().parent_state,
16648 Some(ParentState::Linked)
16649 );
16650 rig.sync(vec![child]).await;
16651 assert!(!rig.live(&child_route));
16652 assert_eq!(
16653 child_route.closed_reason(),
16654 RouteCloseReason::ScopeParentEnded
16655 );
16656 }
16657
16658 #[tokio::test]
16659 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
16660 ) {
16661 let mut rig = rig().await;
16662 rig.sync(vec![session(1)]).await;
16663 let mut route = rig
16664 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16665 .await;
16666 let before = rig.forwarding.published_scope_tag(OWNER, "s");
16667 rig.sync(vec![session(1)]).await;
16668 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16669 assert!(rig.live(&route) && route.untouched());
16670
16671 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
16675 .forwarding
16676 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16677 .unwrap()
16678 else {
16679 panic!("the route is bound");
16680 };
16681 binding.flow.acquire_tagged(9, false).await.unwrap();
16682 let mut widened = session(1);
16683 widened.carriers.push(carrier(MAGIC, None));
16684 rig.sync(vec![widened]).await;
16685 assert!(rig.live(&route) && route.untouched());
16686 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16687 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
16688 assert_eq!(binding.flow.in_flight(), 1);
16691 binding
16692 .flow
16693 .acquire_tagged(10, false)
16694 .await
16695 .expect("the flow is still open");
16696 }
16697
16698 #[tokio::test]
16701 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
16702 let mut rig = rig().await;
16703 rig.sync(vec![session(1)]).await;
16704 let mut on_incumbent = rig
16705 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16706 .await;
16707
16708 let (candidate, _candidate_rx) = wide_ctx(9);
16711 let registration = rig
16712 .handler
16713 .registry
16714 .register_candidate_with_control_ops(
16715 manifest(PLEXUS, PROTOCOL_VERSION),
16716 PROTOCOL_VERSION,
16717 candidate.connection_id,
16718 module_baseline_control_ops(),
16719 )
16720 .unwrap();
16721 rig.forwarding
16722 .register_candidate_module_connection(
16723 candidate.connection_id,
16724 PLEXUS.to_string(),
16725 PROTOCOL_VERSION,
16726 manifest_concurrency(®istration.manifest),
16727 candidate.egress.clone(),
16728 )
16729 .unwrap();
16730 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
16731 rig.handler
16732 .registry
16733 .promote_candidate(PLEXUS)
16734 .unwrap()
16735 .unwrap();
16736 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
16737
16738 rig.sync(Vec::new()).await;
16739 assert!(!rig.live(&on_incumbent));
16740 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
16741 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16742 let goodbye = incumbent_rx
16743 .try_recv()
16744 .expect("the superseded endpoint is told")
16745 .frame;
16746 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16747 }
16748 }
16749}
16750
16751#[cfg(test)]
16752mod concurrency_default_exposure_tests {
16753 use super::*;
16754
16755 fn hello_body(role_json: &str) -> Vec<u8> {
16756 format!(
16757 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
16758 )
16759 .into_bytes()
16760 }
16761
16762 fn manifest_from(body: &[u8]) -> ModuleManifest {
16763 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
16764 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
16765 .expect("manifest parses")
16766 }
16767
16768 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
16769
16770 #[test]
16771 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
16772 let body = hello_body(&format!(
16773 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
16774 ));
16775 let manifest = manifest_from(&body);
16776 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16779 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
16780 }
16781
16782 #[test]
16783 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
16784 let body = hello_body(&format!(
16785 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
16786 ));
16787 let manifest = manifest_from(&body);
16788 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16789 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16790 }
16791
16792 #[test]
16793 fn non_management_roles_are_never_reported() {
16794 let body = hello_body(
16795 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
16796 );
16797 let manifest = manifest_from(&body);
16798 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16799 }
16800}