1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
26 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
27 },
28 scope::{
29 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
30 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
31 SCOPE_SYNC_OP,
32 },
33 session::{
34 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
35 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
36 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
37 ROLE_VERSIONS_FIELD,
38 },
39 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
40 Priority, RouteTarget, PROTOCOL_VERSION,
41};
42use tokio::time::{timeout_at, Instant};
43use tracing::{debug, info, warn};
44
45use crate::{
46 caller_identity::CallerIdentity,
47 capability_requirements::{
48 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
49 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
50 RuntimeModule,
51 },
52 child_roster::DaemonShutdownFlag,
53 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
54 forwarding::{
55 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
56 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
57 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
58 },
59 observability::{
60 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
61 },
62 privileged_prompt::{prompt_text, PrivilegedOperation},
63 provenance::{
64 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
65 },
66 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
67 router::{RouteCtx, RouterError},
68 scopes::{HelloLaunchNonces, ScopeTable},
69 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
70 stderr_tail::{CaptureState, TailEntry},
71 supervise::{
72 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
73 SupervisorHandle, SwapHelloAdmission,
74 },
75 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
76};
77
78pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
84
85const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
86const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
87const CAP_PING_PONG: &str = "ping_pong_v1";
88const CAP_SESSION_ATTACH: &str = "session_attach_v1";
89const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
90
91const SUBC_CONTROL_OPS: &[&str] = &[
92 ops::ACTIVITY_RECORD,
93 ops::ACTIVITY_QUERY,
94 ops::ACTIVITY_SUBSCRIBE,
95 ops::SERVER_DESCRIBE,
96 ops::CATALOG_LIST,
97 ops::ROUTE_OPEN,
98 ops::ROUTE_POLL,
99 ops::ROUTE_CLOSING,
100 ops::ROUTE_CLOSED,
101 ops::SUPERVISOR_LIST,
102 ops::SUPERVISOR_RESTART,
103 ops::SUPERVISOR_SWAP,
104 ops::SUPERVISOR_RELOAD,
105 ops::SUPERVISOR_RESCAN,
106 ops::SUPERVISOR_RELEASE_RESERVED,
107 ops::SUPERVISOR_SET_ENABLED,
108 ops::SUPERVISOR_HEALTH_PROBE,
109 ops::SUPERVISOR_HEALTH,
110 ops::SUPERVISOR_STDERR_TAIL,
111 ops::SUPERVISOR_TERMINALS,
112 ops::SUPERVISOR_ROUTES,
113 ops::SUPERVISOR_PROVENANCE,
114 ops::SUPERVISOR_SPAWN_SNAPSHOT,
115 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
116];
117
118const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
125
126mod rescan_checked;
127use rescan_checked::{RescanMode, RescanPlan};
128
129const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
130 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
131 "supervisor.live_roots",
132 SCOPE_SYNC_OP,
133 SCOPE_APPLY_OP,
134 SCOPE_DESCRIBE_OP,
135 "operator.confirm",
136];
137
138const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
142
143const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
144
145pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
153
154pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
169
170pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
188
189const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
190const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
191
192fn reload_verdict(
193 configured: &Path,
194 spawned_from: Option<&Path>,
195 image: subc_control::RunningImageAgreement,
196) -> PendingReloadVerdict {
197 let path = match spawned_from {
198 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
199 ReloadPathAgreement::Match
200 }
201 Some(spawned_from) => ReloadPathAgreement::Mismatch {
202 configured: configured.to_path_buf(),
203 spawned_from: spawned_from.to_path_buf(),
204 },
205 None => ReloadPathAgreement::Unavailable {
206 reason: if matches!(
207 image,
208 subc_control::RunningImageAgreement::Unavailable {
209 reason: subc_control::RunningImageUnavailableReason::NotRunning
210 }
211 ) {
212 ReloadPathUnavailableReason::NotRunning
213 } else {
214 ReloadPathUnavailableReason::SpawnedPathUnavailable
215 },
216 },
217 };
218 PendingReloadVerdict {
219 path,
220 image,
221 spec: None,
222 }
223}
224
225fn launch_spec_agreement(
226 configured: &crate::supervise::ModuleSpec,
227 running: bool,
228 spawned: Option<&crate::supervise::LaunchSpecDigest>,
229) -> subc_control::LaunchSpecAgreement {
230 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
231 if !running {
232 return Agreement::Unavailable {
233 reason: Reason::NotRunning,
234 };
235 }
236 match spawned {
237 None => Agreement::Unavailable {
238 reason: Reason::SpawnedSpecUnavailable,
239 },
240 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
241 Agreement::Match
242 }
243 Some(_) => Agreement::Mismatch,
244 }
245}
246
247fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
248 if configured == spawned_from {
249 return true;
250 }
251 #[cfg(windows)]
252 {
253 match (
259 std::fs::canonicalize(configured),
260 std::fs::canonicalize(spawned_from),
261 ) {
262 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
263 _ => false,
264 }
265 }
266 #[cfg(not(windows))]
267 {
268 false
269 }
270}
271
272#[derive(Clone)]
273struct DaemonProvenanceFacts {
274 build: DaemonBuildProvenance,
275 pid: Option<u32>,
276 started_at_ms: Option<u64>,
277 start_clock: Option<crate::clock::StartClock>,
278 executable_path: Option<PathBuf>,
279 executable_identity: Option<SpawnedFileIdentity>,
280 process_start_time: Option<u64>,
281 probe: ExecutableIdentityProbe,
282}
283
284impl Default for DaemonProvenanceFacts {
285 fn default() -> Self {
286 Self {
287 build: DaemonBuildProvenance {
288 build_git_sha: None,
289 build_lock_digest: None,
290 },
291 pid: None,
292 started_at_ms: None,
293 start_clock: None,
294 executable_path: None,
295 executable_identity: None,
296 process_start_time: None,
297 probe: ExecutableIdentityProbe::default(),
298 }
299 }
300}
301
302#[derive(Debug, Clone)]
303struct SupervisorRescanContext {
304 supervisor: Supervisor,
305 config_path: PathBuf,
306 configured_port: Option<u16>,
307 storage_config: Option<crate::daemon_config::StorageConfig>,
308 admission_facts_carrier_module_id: Option<String>,
309 admission_facts_targets: Option<Vec<String>>,
310 scope_authority_owners: Vec<String>,
311 privileged_operator_prompt: PrivilegedOperatorPrompt,
312 generation: Arc<std::sync::atomic::AtomicU64>,
318}
319
320const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
341 "reloading",
342 "supervisor_not_live",
343 "registration_not_active",
344 "no_forwarding_connection",
345 "relay_send_failed",
346];
347
348#[derive(Clone)]
350pub struct ControlHandler {
351 registry: Arc<Registry>,
352 activity: Option<crate::activity::ActivityLedger>,
353 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
354 forwarding: Arc<ForwardingTable>,
355 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
356 supervisor: SupervisorHandle,
357 subc_capabilities: Arc<[String]>,
358 route_bind_relay_timeout: Duration,
362 route_bind_relay_timeouts: BTreeMap<String, Duration>,
366 route_bind_breakers: RouteBindBreakers,
369 route_bind_concurrency: RouteBindConcurrency,
372 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
377 route_bind_breaker_threshold: u32,
379 route_bind_breaker_cooldown: Duration,
381 health_probe_timeout: Duration,
382 storage_config: Option<crate::daemon_config::StorageConfig>,
385 machine_id: Option<crate::machine_id::MachineId>,
389 admission_facts_carrier_module_id: Option<String>,
390 admission_facts_targets: Option<Vec<String>>,
391 scopes: Arc<RwLock<ScopeTable>>,
395 scope_deadline_added: Arc<tokio::sync::Notify>,
399 scope_authority_owners: Vec<String>,
402 privileged_operator_prompt: PrivilegedOperatorPrompt,
404 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
407 #[cfg(test)]
408 caller_lookup: Option<TestCallerLookup>,
409 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
412 rescan: Option<SupervisorRescanContext>,
413 connected_clients: ConnectedClients,
414 counters: DaemonCounters,
415 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
416 daemon_shutdown: DaemonShutdownFlag,
421 daemon_provenance: DaemonProvenanceFacts,
422 #[cfg(test)]
423 control_dispatch_delay: Option<Duration>,
424 #[cfg(test)]
425 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
426 #[cfg(test)]
430 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
431 #[cfg(test)]
437 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
438}
439
440impl fmt::Debug for ControlHandler {
441 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
442 f.debug_struct("ControlHandler")
443 .field("registry", &self.registry)
444 .field("forwarding", &self.forwarding)
445 .field("process_liveness", &self.process_liveness.is_some())
446 .field("supervisor", &self.supervisor)
447 .field("subc_capabilities", &self.subc_capabilities)
448 .finish()
449 }
450}
451
452struct RouteOpenRequest {
453 target: RouteTarget,
454 identity: BindIdentity,
455 consumer_identity: Option<ConsumerIdentity>,
456 consumer_capabilities: Option<Vec<String>>,
457 role_versions: Option<BTreeMap<String, String>>,
458 admission_facts: Option<serde_json::Value>,
459 scope: Option<ScopeSelector>,
460}
461
462#[derive(Clone, Copy)]
463enum RouteOpenPhase {
464 Admission,
465 ProjectRoot,
466 EgressReservation,
467 BindSend,
468 BindAckWait,
469}
470
471struct RouteOpenTimings {
475 module_id: String,
476 principal: String,
477 admission: Duration,
478 project_root: Duration,
479 egress_reservation: Duration,
480 bind_send: Duration,
481 bind_ack_wait: Duration,
482 phase: Option<RouteOpenPhase>,
483 phase_started_at: Instant,
484}
485
486impl RouteOpenTimings {
487 fn new() -> Self {
488 Self {
489 module_id: String::new(),
490 principal: "unresolved".to_string(),
492 admission: Duration::ZERO,
493 project_root: Duration::ZERO,
494 egress_reservation: Duration::ZERO,
495 bind_send: Duration::ZERO,
496 bind_ack_wait: Duration::ZERO,
497 phase: Some(RouteOpenPhase::Admission),
498 phase_started_at: Instant::now(),
499 }
500 }
501
502 fn finish(&mut self) {
503 let elapsed = self.phase_started_at.elapsed();
504 match self.phase.take() {
505 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
506 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
507 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
508 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
509 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
510 None => {}
511 }
512 }
513
514 fn enter(&mut self, phase: RouteOpenPhase) {
515 self.finish();
516 self.phase = Some(phase);
517 self.phase_started_at = Instant::now();
518 }
519}
520
521struct RouteBindReservationGuard {
522 forwarding: Arc<ForwardingTable>,
523 endpoint: ModuleEndpointId,
524 relay_corr: u64,
525 armed: bool,
526}
527
528struct ModuleControlRpcGuard {
529 forwarding: Arc<ForwardingTable>,
530 endpoint: ModuleEndpointId,
531 corr: u64,
532 armed: bool,
533}
534
535impl ModuleControlRpcGuard {
536 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
537 Self {
538 forwarding,
539 endpoint,
540 corr,
541 armed: true,
542 }
543 }
544
545 fn disarm(&mut self) {
546 self.armed = false;
547 }
548}
549
550impl Drop for ModuleControlRpcGuard {
551 fn drop(&mut self) {
552 if self.armed {
553 let _ = self
554 .forwarding
555 .cancel_module_control_rpc(self.endpoint, self.corr);
556 }
557 }
558}
559
560impl RouteBindReservationGuard {
561 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
562 Self {
563 forwarding,
564 endpoint,
565 relay_corr,
566 armed: true,
567 }
568 }
569
570 fn release_and_disarm(&mut self) {
571 if !self.armed {
572 return;
573 }
574 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
575 self.endpoint,
576 self.relay_corr,
577 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
578 ) {
579 send_goodbye_target_best_effort(
580 &self.forwarding.counters(),
581 &target,
582 "canceled route.bind",
583 );
584 }
585 self.armed = false;
586 }
587
588 fn disarm(&mut self) {
589 self.armed = false;
590 }
591}
592
593impl Drop for RouteBindReservationGuard {
594 fn drop(&mut self) {
595 self.release_and_disarm();
596 }
597}
598
599#[derive(Debug, Clone, Default)]
629pub(crate) struct RouteBindBreakers {
630 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
631}
632
633#[derive(Debug, Clone, Default)]
634pub(crate) struct RouteBindConcurrency {
635 modules: Arc<Mutex<HashMap<String, usize>>>,
636}
637
638struct RouteBindConcurrencyGuard {
639 concurrency: RouteBindConcurrency,
640 module_id: String,
641}
642
643impl RouteBindConcurrency {
644 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
647 let mut modules = self
648 .modules
649 .lock()
650 .expect("route.bind concurrency mutex poisoned");
651 let in_flight = modules.entry(module_id.to_string()).or_default();
652 if *in_flight >= limit {
653 return Err(*in_flight);
654 }
655 *in_flight += 1;
656 Ok(RouteBindConcurrencyGuard {
657 concurrency: self.clone(),
658 module_id: module_id.to_string(),
659 })
660 }
661}
662
663impl Drop for RouteBindConcurrencyGuard {
664 fn drop(&mut self) {
665 let mut modules = self
666 .concurrency
667 .modules
668 .lock()
669 .expect("route.bind concurrency mutex poisoned");
670 let remove = {
671 let in_flight = modules
672 .get_mut(&self.module_id)
673 .expect("admitted route.bind has a concurrency entry");
674 *in_flight -= 1;
675 *in_flight == 0
676 };
677 if remove {
678 modules.remove(&self.module_id);
679 }
680 }
681}
682
683#[derive(Debug, Default)]
684struct ModuleBreakerState {
685 consecutive_timeouts: u32,
687 cooldown_until: Option<Instant>,
690 probe_in_flight: Option<Arc<()>>,
695}
696
697enum RouteBindAdmission<'a> {
699 Admitted {
700 guard: RouteBindBreakerGuard<'a>,
701 probe: bool,
704 },
705 Refused {
706 consecutive_timeouts: u32,
707 retry_in: Duration,
711 probe_in_flight: bool,
712 },
713}
714
715struct RouteBindBreakerGuard<'a> {
722 breakers: RouteBindBreakers,
723 module_id: &'a str,
724 probe_token: Option<Arc<()>>,
725 settled: bool,
726}
727
728impl RouteBindBreakerGuard<'_> {
729 fn record_accepted(&mut self) -> bool {
733 self.settled = true;
734 self.breakers.record_accepted(self.module_id)
735 }
736
737 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
740 self.settled = true;
741 self.breakers.record_timeout(
742 self.module_id,
743 self.probe_token.as_ref(),
744 threshold,
745 cooldown,
746 )
747 }
748
749 fn record_inconclusive(&mut self) {
758 self.settled = true;
759 self.breakers
760 .record_inconclusive(self.module_id, self.probe_token.as_ref());
761 }
762}
763
764impl Drop for RouteBindBreakerGuard<'_> {
765 fn drop(&mut self) {
766 if !self.settled {
767 self.breakers
768 .record_inconclusive(self.module_id, self.probe_token.as_ref());
769 }
770 }
771}
772
773struct BreakerOpened {
777 consecutive_timeouts: u32,
778 reopened_after_probe: bool,
781}
782
783impl RouteBindBreakers {
784 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
785 self.modules
786 .lock()
787 .expect("route.bind breaker mutex poisoned")
788 }
789
790 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
793 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
794 probe: probe_token.is_some(),
795 guard: RouteBindBreakerGuard {
796 breakers: self.clone(),
797 module_id,
798 probe_token,
799 settled: false,
800 },
801 };
802
803 let mut modules = self.lock();
804 let Some(state) = modules.get_mut(module_id) else {
805 return admitted(None);
806 };
807 let Some(cooldown_until) = state.cooldown_until else {
808 return admitted(None);
809 };
810 if state.probe_in_flight.is_some() {
811 return RouteBindAdmission::Refused {
812 consecutive_timeouts: state.consecutive_timeouts,
813 retry_in: Duration::ZERO,
814 probe_in_flight: true,
815 };
816 }
817 let now = Instant::now();
818 if now < cooldown_until {
819 return RouteBindAdmission::Refused {
820 consecutive_timeouts: state.consecutive_timeouts,
821 retry_in: cooldown_until - now,
822 probe_in_flight: false,
823 };
824 }
825 let token = Arc::new(());
826 state.probe_in_flight = Some(Arc::clone(&token));
827 admitted(Some(token))
828 }
829
830 fn record_accepted(&self, module_id: &str) -> bool {
831 self.lock()
832 .remove(module_id)
833 .is_some_and(|state| state.cooldown_until.is_some())
834 }
835
836 fn record_timeout(
837 &self,
838 module_id: &str,
839 probe_token: Option<&Arc<()>>,
840 threshold: u32,
841 cooldown: Duration,
842 ) -> Option<BreakerOpened> {
843 let mut modules = self.lock();
844 let state = modules.entry(module_id.to_string()).or_default();
845 let was_open = state.cooldown_until.is_some();
846 let was_probe = Self::owns_probe(state, probe_token);
847 if was_probe {
848 state.probe_in_flight = None;
849 }
850 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
851 if state.consecutive_timeouts < threshold {
852 return None;
853 }
854 state.cooldown_until = Some(Instant::now() + cooldown);
855 Some(BreakerOpened {
856 consecutive_timeouts: state.consecutive_timeouts,
857 reopened_after_probe: was_open && was_probe,
858 })
859 }
860
861 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
862 state
866 .probe_in_flight
867 .as_ref()
868 .zip(token)
869 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
870 }
871
872 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
873 if let Some(state) = self.lock().get_mut(module_id) {
874 if Self::owns_probe(state, probe_token) {
875 state.probe_in_flight = None;
876 }
877 }
878 }
879
880 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
895 self.lock()
896 .remove(module_id)
897 .map(|state| state.consecutive_timeouts)
898 .filter(|discarded| *discarded > 0)
899 }
900
901 fn open_snapshot(&self) -> Option<serde_json::Value> {
908 let now = Instant::now();
909 let modules = self.lock();
910 let open = modules
911 .iter()
912 .filter_map(|(module_id, state)| {
913 let cooldown_until = state.cooldown_until?;
914 Some((
915 module_id.clone(),
916 serde_json::json!({
917 "consecutive_timeouts": state.consecutive_timeouts,
918 "cooldown_remaining_ms":
919 cooldown_until.saturating_duration_since(now).as_millis() as u64,
920 "probe_in_flight": state.probe_in_flight.is_some(),
921 }),
922 ))
923 })
924 .collect::<serde_json::Map<String, serde_json::Value>>();
925 (!open.is_empty()).then_some(serde_json::Value::Object(open))
926 }
927}
928
929impl ControlHandler {
930 pub fn new(registry: Arc<Registry>) -> Self {
931 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
932 }
933
934 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
935 let counters = forwarding.counters();
936 let route_bind_breakers = forwarding.route_bind_breakers();
940 let route_bind_concurrency = forwarding.route_bind_concurrency();
941 let route_outages = forwarding.route_outages();
942 Self {
943 registry,
944 activity: None,
945 activity_streams: Arc::new(Mutex::new(HashMap::new())),
946 forwarding,
947 process_liveness: None,
948 supervisor: SupervisorHandle::new(),
949 subc_capabilities: Arc::from([
950 CAP_MANIFEST_REGISTRATION.to_string(),
951 CAP_CHANNEL_LIFECYCLE.to_string(),
952 CAP_PING_PONG.to_string(),
953 CAP_SESSION_ATTACH.to_string(),
954 CAP_ADMISSION_FACTS_RELAY.to_string(),
955 CAP_SCOPES_V1.to_string(),
956 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
957 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
958 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
959 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
960 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
961 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
962 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
963 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
964 ]),
965 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
966 route_bind_relay_timeouts: BTreeMap::new(),
967 route_bind_breakers,
968 route_bind_concurrency,
969 route_outages,
970 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
971 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
972 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
973 storage_config: None,
974 machine_id: None,
975 admission_facts_carrier_module_id: None,
976 admission_facts_targets: None,
977 scopes: Arc::new(RwLock::new(ScopeTable::new(
978 crate::daemon_config::default_scope_authority_owners(),
979 ))),
980 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
981 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
982 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
983 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
984 #[cfg(test)]
985 caller_lookup: None,
986 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
987 rescan: None,
988 connected_clients: ConnectedClients::new(),
989 counters,
990 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
991 daemon_shutdown: DaemonShutdownFlag::default(),
992 daemon_provenance: DaemonProvenanceFacts::default(),
993 #[cfg(test)]
994 control_dispatch_delay: None,
995 #[cfg(test)]
996 provenance_probe_override: None,
997 #[cfg(test)]
998 route_bind_relay_sent_hold: None,
999 #[cfg(test)]
1000 hello_registry_to_forwarding_hold: None,
1001 }
1002 }
1003
1004 pub fn with_storage_config(
1007 mut self,
1008 storage_config: Option<crate::daemon_config::StorageConfig>,
1009 ) -> Self {
1010 self.storage_config = storage_config;
1011 self
1012 }
1013
1014 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1017 self.activity = Some(crate::activity::ActivityLedger::open(
1018 run_dir.join("activity.jsonl"),
1019 incarnation,
1020 ));
1021 self.supervisor.set_activity(self.activity.clone());
1022 self
1023 }
1024
1025 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1028 self.machine_id = machine_id;
1029 self
1030 }
1031
1032 pub fn with_admission_facts_config(
1036 mut self,
1037 carrier_module_id: Option<String>,
1038 targets: Option<Vec<String>>,
1039 ) -> Self {
1040 self.admission_facts_carrier_module_id = carrier_module_id;
1041 self.admission_facts_targets = targets;
1042 self
1043 }
1044
1045 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1049 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1050 self.scope_authority_owners = owners;
1051 self
1052 }
1053
1054 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1056 self.privileged_operator_prompt = mode;
1057 self
1058 }
1059
1060 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1063 self.route_bind_relay_timeout = timeout;
1064 self
1065 }
1066
1067 pub fn with_route_bind_relay_timeouts(
1073 mut self,
1074 timeouts: impl IntoIterator<Item = (String, Duration)>,
1075 ) -> Self {
1076 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1077 self
1078 }
1079
1080 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1086 self.route_bind_relay_timeouts
1087 .get(module_id)
1088 .copied()
1089 .unwrap_or(self.route_bind_relay_timeout)
1090 }
1091
1092 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1100 self.route_bind_breaker_threshold = threshold.max(1);
1101 self.route_bind_breaker_cooldown = cooldown;
1102 self
1103 }
1104
1105 #[cfg(test)]
1106 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1107 self.health_probe_timeout = timeout;
1108 self
1109 }
1110
1111 #[cfg(test)]
1112 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1113 self.control_dispatch_delay = Some(delay);
1114 self
1115 }
1116
1117 #[cfg(test)]
1121 pub(crate) fn with_route_bind_relay_sent_hold(
1122 mut self,
1123 hold: Arc<tokio::sync::Barrier>,
1124 ) -> Self {
1125 self.route_bind_relay_sent_hold = Some(hold);
1126 self
1127 }
1128
1129 #[cfg(test)]
1134 pub(crate) fn with_hello_registry_to_forwarding_hold(
1135 mut self,
1136 hold: Arc<std::sync::Barrier>,
1137 ) -> Self {
1138 self.hello_registry_to_forwarding_hold = Some(hold);
1139 self
1140 }
1141
1142 #[cfg(test)]
1143 fn hold_between_registry_and_forwarding(&self) {
1144 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1145 hold.wait();
1146 hold.wait();
1147 }
1148 }
1149
1150 pub fn with_process_liveness(
1151 mut self,
1152 process_liveness: Arc<dyn ModuleProcessLiveness>,
1153 ) -> Self {
1154 self.process_liveness = Some(process_liveness);
1155 self
1156 }
1157
1158 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1159 self.supervisor = supervisor;
1160 self.supervisor.set_activity(self.activity.clone());
1161 self
1162 }
1163
1164 pub fn with_daemon_provenance(
1165 mut self,
1166 pid: u32,
1167 started_at_ms: u64,
1168 executable_path: Option<PathBuf>,
1169 build_git_sha: Option<String>,
1170 build_lock_digest: Option<String>,
1171 ) -> Self {
1172 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1173 let process_start_time = process_start_time(pid);
1174 self.daemon_provenance = DaemonProvenanceFacts {
1175 build: DaemonBuildProvenance {
1176 build_git_sha,
1177 build_lock_digest,
1178 },
1179 pid: Some(pid),
1180 started_at_ms: Some(started_at_ms),
1181 start_clock: None,
1182 executable_path,
1183 executable_identity,
1184 process_start_time,
1185 probe: ExecutableIdentityProbe::default(),
1186 };
1187 self
1188 }
1189
1190 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1191 self.daemon_provenance.start_clock = Some(clock);
1192 self
1193 }
1194
1195 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1197 self.scopes
1198 .write()
1199 .unwrap_or_else(|poisoned| poisoned.into_inner())
1200 .set_wall_clock(Arc::new(clock));
1201 self
1202 }
1203
1204 #[cfg(test)]
1205 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1206 self.provenance_probe_override = Some(result);
1207 self
1208 }
1209
1210 pub fn with_capability_config(
1214 self,
1215 modules: impl IntoIterator<Item = (String, bool)>,
1216 reserved_capabilities: BTreeMap<String, String>,
1217 ) -> Self {
1218 self.capability_evaluator
1219 .configure(modules, reserved_capabilities);
1220 self
1221 }
1222
1223 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1227 self.daemon_shutdown = flag;
1228 self
1229 }
1230
1231 pub fn with_supervisor_rescan(
1232 mut self,
1233 supervisor: Supervisor,
1234 config_path: impl Into<PathBuf>,
1235 configured_port: Option<u16>,
1236 ) -> Self {
1237 self.rescan = Some(SupervisorRescanContext {
1238 supervisor,
1239 config_path: config_path.into(),
1240 configured_port,
1241 storage_config: self.storage_config.clone(),
1242 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1243 admission_facts_targets: self.admission_facts_targets.clone(),
1244 scope_authority_owners: self.scope_authority_owners.clone(),
1245 privileged_operator_prompt: self.privileged_operator_prompt,
1246 generation: Arc::default(),
1247 });
1248 self
1249 }
1250
1251 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1252 self.connected_clients = connected_clients;
1253 self
1254 }
1255
1256 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1257 Arc::clone(&self.forwarding)
1258 }
1259
1260 pub(crate) fn counters(&self) -> DaemonCounters {
1261 self.counters.clone()
1262 }
1263
1264 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1267 tokio::spawn(async move {
1268 loop {
1269 self.capability_evaluator
1270 .wait_for_change_or_deadline()
1271 .await;
1272 self.refresh_capability_requirements();
1273 }
1274 });
1275 }
1276
1277 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1283 tokio::spawn(async move {
1284 loop {
1285 let has_deadlines = self
1286 .scopes
1287 .read()
1288 .unwrap_or_else(|poisoned| poisoned.into_inner())
1289 .has_deadlines();
1290 if !has_deadlines {
1291 self.scope_deadline_added.notified().await;
1292 continue;
1293 }
1294 tokio::time::sleep(Duration::from_secs(1)).await;
1295 if let Err(error) = self.sweep_expired_scopes() {
1296 warn!(%error, "scope expiry sweep failed");
1297 }
1298 }
1299 });
1300 }
1301
1302 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1303 let mut table = self
1304 .scopes
1305 .write()
1306 .unwrap_or_else(|poisoned| poisoned.into_inner());
1307 let swept = table.sweep_expired();
1308 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1309 drop(table);
1310 self.close_scope_drained_routes(drained);
1311 Ok(())
1312 }
1313
1314 fn publish_scope_changes(
1315 &self,
1316 changes: &[crate::scopes::ScopeTagChange],
1317 expired: &[crate::scopes::ScopeExpired],
1318 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1319 if changes.is_empty() {
1320 return Ok(Vec::new());
1321 }
1322 let drained = self
1323 .forwarding
1324 .publish_scope_changes(changes)
1325 .map_err(RouterError::Forwarding)?;
1326 let mut counts = HashMap::new();
1327 for route in &drained {
1328 *counts
1329 .entry((
1330 route.scope.owner.as_str(),
1331 route.scope.scope_ref.as_str(),
1332 route.scope.tag.scope_epoch,
1333 ))
1334 .or_insert(0usize) += 1;
1335 }
1336 for scope in expired {
1337 let routes_closed = counts
1338 .get(&(
1339 scope.owner.as_str(),
1340 scope.scope_ref.as_str(),
1341 scope.scope_epoch,
1342 ))
1343 .copied()
1344 .unwrap_or(0);
1345 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1346 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1347 routes_closed, "scope expired");
1348 }
1349 Ok(drained)
1350 }
1351
1352 fn runtime_capability_snapshot(
1353 &self,
1354 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1355 let runtime = self
1356 .supervisor
1357 .list()
1358 .into_iter()
1359 .map(|module| {
1360 let status = module.status().map_err(|err| {
1361 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1362 })?;
1363 Ok(RuntimeModule {
1364 module_id: status.module_id,
1365 state: status.state,
1366 enabled: status.enabled,
1367 })
1368 })
1369 .collect::<Result<Vec<_>, RouterError>>()?;
1370 let registrations = self.registry.list_registered_modules().map_err(|err| {
1376 RouterError::backend(
1377 0,
1378 0,
1379 format!("failed to list capability registrations: {err}"),
1380 )
1381 })?;
1382 let registrations = registrations
1383 .into_iter()
1384 .map(|registration| RegisteredModule {
1385 module_id: registration.manifest.module_id,
1386 module_version: registration.manifest.module_version,
1387 capabilities: registration.manifest.capabilities,
1388 })
1389 .collect();
1390 Ok((runtime, registrations))
1391 }
1392
1393 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1400 let cached_registration = RegisteredModule {
1401 module_id: registration.manifest.module_id.clone(),
1402 module_version: registration.manifest.module_version.clone(),
1403 capabilities: registration.manifest.capabilities.clone(),
1404 };
1405 if self.capability_evaluator.record_hello(&cached_registration) {
1406 warn!(
1407 module_id = %cached_registration.module_id,
1408 "capability claims drifted from the cached manifest"
1409 );
1410 }
1411 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1412 self.enforce_capability_denies();
1413 }
1414 self.refresh_capability_requirements();
1415 }
1416
1417 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1421 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1422 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1423 self.supervisor.set_swap_promotion_observer(observer);
1424 }
1425
1426 pub fn refresh_capability_requirements(&self) {
1427 match self.runtime_capability_snapshot() {
1428 Ok((runtime, registrations)) => {
1429 log_requirement_events(
1430 self.capability_evaluator
1431 .evaluate_now(&runtime, ®istrations),
1432 &self.daemon_shutdown,
1433 );
1434 }
1435 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1436 }
1437 }
1438
1439 fn enforce_capability_denies(&self) {
1443 let registrations = match self.registry.list_registered_modules() {
1444 Ok(snapshot) => snapshot,
1445 Err(err) => {
1446 warn!(error = %err, "failed to read registrations for capability deny census");
1447 return;
1448 }
1449 };
1450 let manifests = registrations
1451 .into_iter()
1452 .map(|registration| {
1453 (
1454 registration.manifest.module_id.clone(),
1455 registration.manifest,
1456 )
1457 })
1458 .collect::<BTreeMap<_, _>>();
1459 let census = match self.forwarding.route_census(None) {
1460 Ok(census) => census,
1461 Err(err) => {
1462 warn!(error = %err, "failed to read route census for capability deny enforcement");
1463 return;
1464 }
1465 };
1466
1467 for (target_module_id, routes) in census {
1468 let Some(target_manifest) = manifests.get(&target_module_id) else {
1469 continue;
1470 };
1471 let mut closed_routes = Vec::new();
1472 let mut module_goodbyes = Vec::new();
1473 for route in routes {
1474 let Principal::Reserved {
1475 module_id: opening_module_id,
1476 } = &route.principal
1477 else {
1478 continue;
1479 };
1480 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1481 continue;
1482 };
1483 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1484 continue;
1485 };
1486
1487 match self.forwarding.release_client_route(
1488 route.goodbye_target.connection_id,
1489 route.goodbye_target.channel,
1490 route.goodbye_target.epoch,
1491 ) {
1492 Ok(RouteRelease::Removed(module_goodbye)) => {
1493 warn!(
1494 opening_module_id,
1495 target_module_id,
1496 capability,
1497 "force-closing route because an attested capability deny edge now matches"
1498 );
1499 closed_routes.push(route);
1500 module_goodbyes.push(module_goodbye);
1501 }
1502 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1503 Err(err) => warn!(
1504 opening_module_id,
1505 target_module_id,
1506 capability,
1507 error = %err,
1508 "failed to force-close capability-denied route"
1509 ),
1510 }
1511 }
1512
1513 if closed_routes.is_empty() {
1514 continue;
1515 }
1516 send_route_control_pushes(
1517 &self.forwarding,
1518 closed_routes,
1519 ClientControlPush::RouteClosed {
1520 module_id: target_module_id,
1521 channels: Vec::new(),
1522 reason: RouteCloseReason::CapabilityDenied,
1523 drained: false,
1524 abandoned: 0,
1525 excluded_subscriptions: 0,
1526 terminal: Some(false),
1527 },
1528 );
1529 self.emit_route_goodbyes(module_goodbyes);
1530 }
1531 }
1532
1533 fn not_ready_reason(
1539 &self,
1540 registration: &crate::registry::ModuleRegistration,
1541 ) -> Option<NotReadyReason> {
1542 if !registration.ready {
1543 return Some(NotReadyReason {
1544 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1545 capability: None,
1546 });
1547 }
1548 self.first_unprovided_required_capability(registration)
1549 .map(|capability| NotReadyReason {
1550 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1551 capability: Some(capability),
1552 })
1553 }
1554
1555 fn first_unprovided_required_capability(
1573 &self,
1574 registration: &crate::registry::ModuleRegistration,
1575 ) -> Option<String> {
1576 let required = registration
1577 .manifest
1578 .capabilities
1579 .iter()
1580 .flat_map(|declarations| declarations.requires.iter())
1581 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1582 .map(|requirement| requirement.capability.as_str())
1583 .collect::<BTreeSet<_>>();
1584 if required.is_empty() {
1585 return None;
1586 }
1587 let module_id = registration.manifest.module_id.as_str();
1588 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1589 if required
1590 .iter()
1591 .any(|capability| verdict(capability).is_none())
1592 {
1593 self.refresh_capability_requirements();
1594 }
1595 required
1596 .into_iter()
1597 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1598 .map(str::to_string)
1599 }
1600
1601 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1602 self.capability_evaluator
1603 .statuses()
1604 .into_iter()
1605 .map(capability_requirement_status)
1606 .collect()
1607 }
1608
1609 fn deregister_connection(
1616 &self,
1617 connection_id: ConnectionId,
1618 reason: RegistrationEndReason,
1619 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1620 self.registry
1621 .deregister_connection_with_reason(connection_id, reason)
1622 }
1623
1624 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1625 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1626 return None;
1627 }
1628 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1629 parse_client_control_request(&frame.body)
1630 else {
1631 return None;
1632 };
1633 Some(target_module_id(&target).to_string())
1634 }
1635
1636 pub(crate) fn route_open_capacity_refusal(
1637 &self,
1638 ctx: &RouteCtx,
1639 frame: &Frame,
1640 target_module_id: &str,
1641 in_flight: usize,
1642 limit: usize,
1643 ) -> Result<Frame, RouterError> {
1644 self.route_open_admission_refusal_frame(
1645 ctx,
1646 frame,
1647 target_module_id,
1648 "open_admission_full",
1649 (in_flight, limit),
1650 format!(
1651 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1652 ),
1653 )
1654 }
1655
1656 fn route_open_target_capacity_refusal(
1657 &self,
1658 ctx: &RouteCtx,
1659 frame: &Frame,
1660 target_module_id: &str,
1661 in_flight: usize,
1662 ) -> Result<Frame, RouterError> {
1663 self.route_open_admission_refusal_frame(
1664 ctx,
1665 frame,
1666 target_module_id,
1667 "target_binds_full",
1668 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1669 format!(
1670 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1671 ),
1672 )
1673 }
1674
1675 fn route_open_admission_refusal_frame(
1683 &self,
1684 ctx: &RouteCtx,
1685 frame: &Frame,
1686 target_module_id: &str,
1687 reason: &'static str,
1688 (in_flight, limit): (usize, usize),
1689 message: impl Into<String>,
1690 ) -> Result<Frame, RouterError> {
1691 let code = error_codes::TARGET_UNAVAILABLE;
1692 self.counters.increment_route_open_refused(code);
1693 info!(
1694 target: "control",
1695 code,
1696 reason,
1697 module_id = ?target_module_id,
1698 connection_id = ctx.connection_id.get(),
1699 in_flight,
1700 limit,
1701 "route.open refused"
1702 );
1703 control_error_frame(frame, code, message.into())
1704 }
1705
1706 #[cfg(test)]
1712 pub fn handle_control(
1713 &self,
1714 connection_id: ConnectionId,
1715 frame: Frame,
1716 ) -> Result<Vec<Frame>, RouterError> {
1717 match frame.header.ty {
1718 FrameType::Ping => Ok(vec![pong(&frame)?]),
1719 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1720 FrameType::Goodbye => self.handle_goodbye(connection_id),
1721 ty => Ok(vec![control_error_frame(
1722 &frame,
1723 "unsupported_control_frame",
1724 format!("unsupported channel-0 frame {ty:?}"),
1725 )?]),
1726 }
1727 }
1728
1729 pub async fn handle_control_frame(
1730 &self,
1731 ctx: &RouteCtx,
1732 frame: Frame,
1733 ) -> Result<Vec<Frame>, RouterError> {
1734 self.handle_control_frame_timed(ctx, frame, None).await
1735 }
1736
1737 pub(crate) fn record_caller_endpoints(
1738 &self,
1739 connection: ConnectionId,
1740 endpoints: Option<(SocketAddr, SocketAddr)>,
1741 ) {
1742 let mut callers = self
1743 .caller_endpoints
1744 .lock()
1745 .unwrap_or_else(|p| p.into_inner());
1746 callers.remove(&connection);
1747 if let Some(endpoints) = endpoints {
1748 callers.insert(connection, endpoints);
1749 }
1750 }
1751
1752 #[cfg(test)]
1753 pub(crate) fn with_caller_lookup_for_test(
1754 mut self,
1755 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1756 ) -> Self {
1757 self.caller_lookup = Some(Arc::new(lookup));
1758 self
1759 }
1760
1761 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1762 let endpoints = self
1763 .caller_endpoints
1764 .lock()
1765 .unwrap_or_else(|p| p.into_inner())
1766 .get(&connection)
1767 .copied();
1768 #[cfg(test)]
1769 let lookup = self.caller_lookup.clone();
1770 tokio::task::spawn_blocking(move || {
1771 #[cfg(test)]
1772 if let Some(lookup) = lookup {
1773 return lookup(endpoints);
1774 }
1775 endpoints
1776 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1777 .unwrap_or_default()
1778 })
1779 .await
1780 .unwrap_or_default()
1781 }
1782
1783 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1784 frame.header.channel == 0
1785 && frame.header.ty == FrameType::Request
1786 && self
1787 .forwarding
1788 .module_endpoint_for_connection(ctx.connection_id)
1789 .is_ok_and(|endpoint| endpoint.is_none())
1790 && parse_client_control_request(&frame.body)
1791 .ok()
1792 .is_some_and(|request| privileged_operation(&request).is_some())
1793 }
1794
1795 pub(crate) async fn handle_control_frame_timed(
1796 &self,
1797 ctx: &RouteCtx,
1798 frame: Frame,
1799 dispatch_started_at: Option<StdInstant>,
1800 ) -> Result<Vec<Frame>, RouterError> {
1801 match frame.header.ty {
1802 FrameType::Ping => Ok(vec![pong(&frame)?]),
1803 FrameType::Hello => {
1804 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1805 }
1806 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1807 FrameType::Cancel => {
1808 if self
1811 .forwarding
1812 .operator_confirms()
1813 .cancel(ctx.connection_id, frame.header.corr)
1814 || self
1815 .supervisor
1816 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1817 || self.cancel_activity_subscription(ctx, &frame)?
1818 {
1819 Ok(Vec::new())
1820 } else {
1821 Ok(vec![control_error_frame(
1822 &frame,
1823 "unknown_subscription",
1824 "no subscription has this correlation id",
1825 )?])
1826 }
1827 }
1828 FrameType::Request => {
1829 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1832 if op
1833 .as_ref()
1834 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1835 {
1836 return self.handle_activity_record(ctx, frame).await;
1837 }
1838 if op.as_ref().is_some_and(|probe| {
1839 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1840 }) {
1841 return self.handle_activity_read(ctx, frame).await;
1842 }
1843 if op
1844 .as_ref()
1845 .is_some_and(|probe| probe.op == "operator.confirm")
1846 {
1847 return self.handle_operator_confirm(ctx, frame);
1848 }
1849 if op
1850 .as_ref()
1851 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1852 {
1853 return self.handle_scope_served_by(ctx.connection_id, frame);
1854 }
1855 if self
1856 .forwarding
1857 .module_endpoint_for_connection(ctx.connection_id)
1858 .map_err(RouterError::Forwarding)?
1859 .is_some()
1860 {
1861 if !is_known_module_request_op(&frame.body) {
1862 return Ok(vec![control_error_frame(
1863 &frame,
1864 "unsupported_control_frame",
1865 "module-originated channel-0 REQUEST is not supported",
1866 )?]);
1867 }
1868 let request = match parse_module_control_request_from_module(&frame.body) {
1869 Ok(request) => request,
1870 Err((err, ControlRequestBodyError::UnknownOp)) => {
1871 return Ok(vec![control_error_frame(
1872 &frame,
1873 "unsupported_control_frame",
1874 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1875 )?])
1876 }
1877 Err((err, ControlRequestBodyError::InvalidBody)) => {
1878 return Ok(vec![control_error_frame(
1879 &frame,
1880 "invalid_control_body",
1881 format!("malformed module control body: {err}"),
1882 )?])
1883 }
1884 };
1885 let op = module_control_request_op(&request);
1886 let corr = frame.header.corr;
1887 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1888 let result =
1889 self.handle_module_control_request(ctx.connection_id, frame, request);
1890 log_slow_control_dispatch(
1891 dispatch_started_at,
1892 op,
1893 ctx.connection_id,
1894 corr,
1895 None,
1896 );
1897 return result;
1898 }
1899
1900 if is_known_module_request_op(&frame.body) {
1901 return Ok(vec![control_error_frame(
1902 &frame,
1903 "not_registered",
1904 "catalog.update requires an active module registration owned by this connection",
1905 )?]);
1906 }
1907
1908 let request = match parse_client_control_request(&frame.body) {
1909 Ok(request) => request,
1910 Err((err, ControlRequestBodyError::UnknownOp)) => {
1911 return Ok(vec![control_error_frame(
1912 &frame,
1913 "unknown_control_op",
1914 format!("unknown client control op: {err}"),
1915 )?])
1916 }
1917 Err((err, ControlRequestBodyError::InvalidBody)) => {
1918 return Ok(vec![control_error_frame(
1919 &frame,
1920 "invalid_control_body",
1921 format!("malformed client control body: {err}"),
1922 )?])
1923 }
1924 };
1925 let op = client_control_request_op(&request);
1926 let corr = frame.header.corr;
1927 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1928 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1929 #[cfg(test)]
1930 if let Some(delay) = self.control_dispatch_delay {
1931 tokio::time::sleep(delay).await;
1932 }
1933 let result = self
1934 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1935 .await;
1936 if let Some(timings) = route_open_timings.as_mut() {
1937 timings.finish();
1938 }
1939 log_slow_control_dispatch(
1940 dispatch_started_at,
1941 op,
1942 ctx.connection_id,
1943 corr,
1944 route_open_timings.as_ref(),
1945 );
1946 result
1947 }
1948 FrameType::Push => {
1949 let Some(endpoint) = self
1950 .forwarding
1951 .module_endpoint_for_connection(ctx.connection_id)
1952 .map_err(RouterError::Forwarding)?
1953 else {
1954 return Ok(vec![control_error_frame(
1955 &frame,
1956 "unsupported_control_frame",
1957 "client-originated channel-0 PUSH is not supported",
1958 )?]);
1959 };
1960 self.handle_status_update(endpoint, frame)
1961 }
1962 FrameType::Response | FrameType::Error
1963 if self
1964 .forwarding
1965 .module_endpoint_for_connection(ctx.connection_id)
1966 .map_err(RouterError::Forwarding)?
1967 .is_some() =>
1968 {
1969 self.handle_module_relay_response(ctx.connection_id, frame)
1970 }
1971 ty => Ok(vec![control_error_frame(
1972 &frame,
1973 "unsupported_control_frame",
1974 format!("unsupported channel-0 frame {ty:?}"),
1975 )?]),
1976 }
1977 }
1978
1979 pub fn cleanup_connection(
1980 &self,
1981 connection_id: ConnectionId,
1982 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1983 self.caller_endpoints
1984 .lock()
1985 .unwrap_or_else(|p| p.into_inner())
1986 .remove(&connection_id);
1987 self.forwarding
1988 .operator_confirms()
1989 .connection_closed(connection_id);
1990 self.cleanup_connection_with_end_reason(
1991 connection_id,
1992 RegistrationEndReason::ConnectionClosed,
1993 )
1994 }
1995
1996 fn cleanup_connection_with_end_reason(
1997 &self,
1998 connection_id: ConnectionId,
1999 requested_reason: RegistrationEndReason,
2000 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2001 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2002 self.registry
2003 .get_module_by_connection(connection_id)?
2004 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2005 .and_then(|module| module.registration_end_reason().ok().flatten())
2006 .unwrap_or(requested_reason)
2007 } else {
2008 requested_reason
2009 };
2010 let crash_closed = self
2011 .registry
2012 .get_module_by_connection(connection_id)?
2013 .and_then(|registration| {
2014 self.forwarding
2015 .module_endpoint_for_connection(connection_id)
2016 .ok()
2017 .flatten()
2018 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2019 .map(|routes| (registration.manifest.module_id, routes))
2020 });
2021 let crash_closed = crash_closed.map(|(module_id, routes)| {
2022 let terminal = match self.supervisor.get(&module_id) {
2023 None => false,
2024 Some(module) => match module.will_recover_after_connection_loss() {
2025 Ok(will_recover) => !will_recover,
2026 Err(err) => {
2027 warn!(
2028 %module_id,
2029 error = %err,
2030 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2031 );
2032 false
2033 }
2034 },
2035 };
2036 let (reason, drained) = match self
2040 .forwarding
2041 .daemon_drain_result_for_connection(connection_id)
2042 {
2043 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2044 Ok(None) => (RouteCloseReason::Crash, false),
2045 Err(err) => {
2046 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2047 (RouteCloseReason::Crash, false)
2048 }
2049 };
2050 (module_id, routes, reason, drained, terminal)
2051 });
2052 let registrations = self.deregister_connection(connection_id, end_reason);
2053 let cleanup = if crash_closed.is_some() {
2054 self.forwarding.cleanup_connection_counted(connection_id)
2055 } else {
2056 self.forwarding
2062 .cleanup_connection(connection_id)
2063 .map(|released| crate::forwarding::ConnectionCleanup {
2064 released,
2065 abandoned_relays: 0,
2066 })
2067 };
2068 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2073 let abandoned = cleanup
2074 .as_ref()
2075 .map_or(0, |cleanup| cleanup.abandoned_relays);
2076 send_route_control_pushes(
2077 &self.forwarding,
2078 routes,
2079 ClientControlPush::RouteClosed {
2080 module_id,
2081 channels: Vec::new(),
2082 reason,
2083 drained,
2084 abandoned,
2085 excluded_subscriptions: 0,
2086 terminal: Some(terminal),
2087 },
2088 );
2089 }
2090 if let Ok(cleanup) = cleanup {
2091 self.emit_route_goodbyes(cleanup.released);
2092 }
2093 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2097 crate::supervise::notify_registration_release();
2098 self.capability_evaluator.wake_deadline_loop();
2099 self.refresh_capability_requirements();
2100 }
2101 self.supervisor.remove_spawn_subscribers(connection_id);
2102 self.activity_streams
2103 .lock()
2104 .unwrap_or_else(|p| p.into_inner())
2105 .retain(|(id, _), task| {
2106 if *id == connection_id {
2107 task.abort();
2108 false
2109 } else {
2110 true
2111 }
2112 });
2113 self.hello_launch_nonces
2116 .lock()
2117 .unwrap_or_else(|poisoned| poisoned.into_inner())
2118 .forget(connection_id);
2119 self.scopes
2120 .write()
2121 .unwrap_or_else(|poisoned| poisoned.into_inner())
2122 .release_connection(connection_id);
2123 registrations
2124 }
2125
2126 pub(crate) fn handle_route_goodbye(
2127 &self,
2128 connection_id: ConnectionId,
2129 route_channel: u16,
2130 route_epoch: u32,
2131 ) -> Result<bool, RouterError> {
2132 debug!(
2133 connection_id = connection_id.get(),
2134 route_channel, route_epoch, "handling route GOODBYE"
2135 );
2136 let RouteRelease::Removed(released_route) = self
2137 .forwarding
2138 .release_client_route(connection_id, route_channel, route_epoch)
2139 .map_err(RouterError::Forwarding)?
2140 else {
2141 return Ok(false);
2142 };
2143 self.emit_route_goodbyes(vec![released_route]);
2144 Ok(true)
2145 }
2146
2147 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2148 for released in released_routes {
2149 let frame = match Frame::build_with_version(
2150 released.negotiated_ver,
2151 FrameType::Goodbye,
2152 control_flags(),
2153 released.channel,
2154 released.epoch,
2155 0,
2156 Vec::new(),
2157 ) {
2158 Ok(frame) => frame,
2159 Err(err) => {
2160 warn!(
2161 route_channel = released.channel,
2162 error = %err,
2163 "failed to build route GOODBYE frame"
2164 );
2165 continue;
2166 }
2167 };
2168 if !released.close_on_delivery_failure() {
2169 crate::forwarding::send_module_route_goodbye(
2170 &self.counters,
2171 &released.sink,
2172 frame,
2173 released.module_id.as_deref(),
2174 "client route released",
2175 );
2176 continue;
2177 }
2178 if let Err(err) = released.sink.try_send(frame) {
2179 warn!(
2180 target_connection_id = released.connection_id.get(),
2181 route_channel = released.channel,
2182 error = %err,
2183 "route GOODBYE was not delivered to client; closing target connection"
2184 );
2185 if self
2186 .forwarding
2187 .escalate_client_delivery_failure(
2188 released.connection_id,
2189 released.channel,
2190 released.epoch,
2191 CloseReason::new(
2192 "route_goodbye_delivery_failed",
2193 format!(
2194 "failed to enqueue route GOODBYE for channel {}: {err}",
2195 released.channel
2196 ),
2197 ),
2198 crate::forwarding::UndeliveredFrame {
2199 module_id: released.module_id.as_deref(),
2200 sink: &released.sink,
2201 },
2202 )
2203 .unwrap_or(false)
2204 {
2205 self.counters.increment_goodbye_relay_client_failed();
2206 }
2207 }
2208 }
2209 }
2210
2211 fn send_abandoned_route_bind_goodbye(
2223 &self,
2224 module_sink: &crate::FrameSink,
2225 negotiated_ver: u8,
2226 module_channel: u16,
2227 module_epoch: u32,
2228 ) {
2229 let frame = match Frame::build_with_version(
2230 negotiated_ver,
2231 FrameType::Goodbye,
2232 control_flags(),
2233 module_channel,
2234 module_epoch,
2235 0,
2236 Vec::new(),
2237 ) {
2238 Ok(frame) => frame,
2239 Err(err) => {
2240 warn!(
2241 route_channel = module_channel,
2242 error = %err,
2243 "failed to build GOODBYE for abandoned route.bind"
2244 );
2245 return;
2246 }
2247 };
2248 crate::forwarding::send_module_route_goodbye(
2249 &self.counters,
2250 module_sink,
2251 frame,
2252 None,
2253 "abandoned route.bind",
2254 );
2255 }
2256
2257 fn handle_hello(
2258 &self,
2259 connection_id: ConnectionId,
2260 sink: Option<crate::FrameSink>,
2261 frame: Frame,
2262 ) -> Result<Vec<Frame>, RouterError> {
2263 debug!(
2264 connection_id = connection_id.get(),
2265 corr = frame.header.corr,
2266 "handling HELLO"
2267 );
2268 if self
2272 .registry
2273 .get_module_by_connection(connection_id)
2274 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2275 .is_some()
2276 {
2277 return Ok(vec![control_error_frame(
2278 &frame,
2279 "invalid_hello",
2280 "connection is already registered as a module",
2281 )?]);
2282 }
2283 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2284 Ok(value) => value,
2285 Err(err) => {
2286 return Ok(vec![control_error_frame(
2287 &frame,
2288 "invalid_hello",
2289 format!("malformed HELLO body: {err}"),
2290 )?])
2291 }
2292 };
2293 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2294 return Ok(vec![control_error_frame(
2295 &frame,
2296 "invalid_capability_grammar",
2297 err.to_string(),
2298 )?]);
2299 }
2300 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2301 return Ok(vec![control_error_frame(
2302 &frame,
2303 "invalid_manifest",
2304 err.to_string(),
2305 )?]);
2306 }
2307 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2308 return Ok(vec![control_error_frame(
2309 &frame,
2310 "invalid_event_declaration",
2311 err.to_string(),
2312 )?]);
2313 }
2314 if let Some(provenance) = hello_value
2315 .get("manifest")
2316 .and_then(|manifest| manifest.get("provenance"))
2317 {
2318 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2319 return Ok(vec![control_error_frame(
2320 &frame,
2321 "invalid_manifest",
2322 format!("malformed manifest provenance: {err}"),
2323 )?]);
2324 }
2325 }
2326 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2327 Ok(hello) => hello,
2328 Err(err) => {
2329 return Ok(vec![control_error_frame(
2330 &frame,
2331 "invalid_hello",
2332 format!("malformed HELLO body: {err}"),
2333 )?])
2334 }
2335 };
2336
2337 if hello.protocol_ver != hello.manifest.protocol_ver {
2338 return Ok(vec![control_error_frame(
2339 &frame,
2340 "invalid_manifest",
2341 format!(
2342 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2343 hello.protocol_ver, hello.manifest.protocol_ver
2344 ),
2345 )?]);
2346 }
2347
2348 if hello.manifest.module_id.trim().is_empty() {
2349 return Ok(vec![control_error_frame(
2350 &frame,
2351 "invalid_manifest",
2352 "manifest module_id must not be empty",
2353 )?]);
2354 }
2355
2356 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2357 Ok(negotiated_ver) => negotiated_ver,
2358 Err(message) => {
2359 return Ok(vec![control_error_frame(
2360 &frame,
2361 "version_unsupported",
2362 message,
2363 )?])
2364 }
2365 };
2366
2367 let swap_admission = self
2375 .supervisor
2376 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2377 if swap_admission == SwapHelloAdmission::Refused {
2378 warn!(
2379 module_id = %hello.manifest.module_id,
2380 connection_id = connection_id.get(),
2381 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2382 );
2383 return Ok(vec![control_error_frame(
2384 &frame,
2385 "swap_token_invalid",
2386 format!(
2387 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2388 hello.manifest.module_id
2389 ),
2390 )?]);
2391 }
2392 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2393
2394 if let Some(rejection) = (!swap_candidate)
2402 .then(|| {
2403 self.supervisor.reserved_hello_rejection(
2404 &hello.manifest.module_id,
2405 hello.launch_nonce.as_deref(),
2406 )
2407 })
2408 .flatten()
2409 {
2410 let message = match rejection {
2411 ReservedHelloRejection::Exact { module_id } => format!(
2412 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2413 ),
2414 ReservedHelloRejection::Prefix {
2415 prefix,
2416 owner_module_id,
2417 } => format!(
2418 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2419 hello.manifest.module_id
2420 ),
2421 };
2422 return Ok(vec![control_error_frame(
2423 &frame,
2424 "reserved_module",
2425 message,
2426 )?]);
2427 }
2428
2429 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2430 &hello.manifest.module_id,
2431 hello.manifest.capabilities.as_ref(),
2432 );
2433 if let Some(refusal) = reserved_capability_refusals.first() {
2434 let capability = refusal.capability.clone();
2435 let bound_module = refusal.claimants[0].clone();
2436 log_duplicate_claim_events(reserved_capability_refusals);
2437 return Ok(vec![control_error_frame(
2438 &frame,
2439 "reserved_capability",
2440 format!(
2441 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2442 capability, bound_module, hello.manifest.module_id
2443 ),
2444 )?]);
2445 }
2446
2447 if self
2450 .forwarding
2451 .connection_has_client_routes(connection_id)
2452 .map_err(RouterError::Forwarding)?
2453 {
2454 return Ok(vec![control_error_frame(
2455 &frame,
2456 "invalid_hello",
2457 "connection has open client routes and cannot also register as a module",
2458 )?]);
2459 }
2460
2461 self.hello_launch_nonces
2466 .lock()
2467 .unwrap_or_else(|poisoned| poisoned.into_inner())
2468 .record(connection_id, hello.launch_nonce.as_deref());
2469 let control_ops = effective_module_control_ops(hello.control_ops);
2470 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2473 if swap_candidate {
2474 return self.register_swap_candidate(
2475 connection_id,
2476 sink,
2477 &frame,
2478 hello.manifest,
2479 negotiated_ver,
2480 control_ops,
2481 hello_ack,
2482 );
2483 }
2484 let registration = match self.registry.register_unpublished_with_control_ops(
2491 hello.manifest,
2492 negotiated_ver,
2493 connection_id,
2494 control_ops,
2495 ) {
2496 Ok(registration) => registration,
2497 Err(RegistryError::DuplicateModuleId { module_id }) => {
2498 return Ok(vec![control_error_frame(
2499 &frame,
2500 "duplicate_module_id",
2501 format!(
2502 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2503 ),
2504 )?])
2505 }
2506 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2507 return Ok(vec![control_error_frame(
2508 &frame,
2509 "invalid_module_id",
2510 err.to_string(),
2511 )?])
2512 }
2513 Err(err) => {
2514 return Ok(vec![control_error_frame(
2515 &frame,
2516 "registry_error",
2517 err.to_string(),
2518 )?])
2519 }
2520 };
2521 #[cfg(test)]
2522 self.hold_between_registry_and_forwarding();
2523
2524 let reply = if let Some(sink) = sink {
2525 let concurrency = manifest_concurrency(®istration.manifest);
2541 if let Err(err) = self.forwarding.register_module_connection_acked(
2542 connection_id,
2543 registration.manifest.module_id.clone(),
2544 negotiated_ver,
2545 concurrency,
2546 sink,
2547 hello_ack,
2548 ) {
2549 if matches!(
2554 self.deregister_connection(
2555 connection_id,
2556 RegistrationEndReason::RegistrationFailed,
2557 ),
2558 Ok(r) if !r.is_empty()
2559 ) {
2560 crate::supervise::notify_registration_release();
2561 }
2562 return Ok(vec![control_error_frame(
2563 &frame,
2564 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2565 "invalid_hello"
2566 } else {
2567 forwarding_error_code(&err)
2568 },
2569 err.to_string(),
2570 )?]);
2571 }
2572 Vec::new()
2573 } else {
2574 vec![hello_ack]
2577 };
2578
2579 if !self
2584 .registry
2585 .publish_registration(connection_id)
2586 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2587 {
2588 debug!(
2589 module_id = %registration.manifest.module_id,
2590 connection_id = connection_id.get(),
2591 "registration was removed before it could be published"
2592 );
2593 }
2594
2595 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2604 info!(
2605 module_id = %registration.manifest.module_id,
2606 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2607 );
2608 }
2609
2610 self.apply_registration_capabilities(®istration);
2611
2612 self.supervisor
2613 .record_activity_registration(®istration.manifest.module_id);
2614
2615 info!(
2616 module_id = %registration.manifest.module_id,
2617 module_version = %registration.manifest.module_version,
2618 negotiated_ver,
2619 routable_provider = manifest_provides_routable_role(®istration.manifest),
2620 connection_id = connection_id.get(),
2621 "module registered"
2622 );
2623
2624 Ok(reply)
2625 }
2626
2627 #[allow(clippy::too_many_arguments)]
2644 fn register_swap_candidate(
2645 &self,
2646 connection_id: ConnectionId,
2647 sink: Option<crate::FrameSink>,
2648 frame: &Frame,
2649 manifest: ModuleManifest,
2650 negotiated_ver: u8,
2651 control_ops: Vec<String>,
2652 hello_ack: Frame,
2653 ) -> Result<Vec<Frame>, RouterError> {
2654 let module_id = manifest.module_id.clone();
2655 let registration = match self.registry.register_unpublished_candidate_with_control_ops(
2656 manifest,
2657 negotiated_ver,
2658 connection_id,
2659 control_ops,
2660 ) {
2661 Ok(registration) => registration,
2662 Err(RegistryError::DuplicateModuleId { module_id }) => {
2663 return Ok(vec![control_error_frame(
2664 frame,
2665 "duplicate_module_id",
2666 format!(
2667 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2668 ),
2669 )?])
2670 }
2671 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2672 return Ok(vec![control_error_frame(
2673 frame,
2674 "invalid_module_id",
2675 err.to_string(),
2676 )?])
2677 }
2678 Err(err) => {
2679 return Ok(vec![control_error_frame(
2680 frame,
2681 "registry_error",
2682 err.to_string(),
2683 )?])
2684 }
2685 };
2686 #[cfg(test)]
2687 self.hold_between_registry_and_forwarding();
2688 let reply = if let Some(sink) = sink {
2689 let concurrency = manifest_concurrency(®istration.manifest);
2693 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2694 connection_id,
2695 module_id.clone(),
2696 negotiated_ver,
2697 concurrency,
2698 sink,
2699 hello_ack,
2700 ) {
2701 if matches!(
2702 self.deregister_connection(
2703 connection_id,
2704 RegistrationEndReason::RegistrationFailed,
2705 ),
2706 Ok(r) if !r.is_empty()
2707 ) {
2708 crate::supervise::notify_registration_release();
2709 }
2710 return Ok(vec![control_error_frame(
2711 frame,
2712 forwarding_error_code(&err),
2713 err.to_string(),
2714 )?]);
2715 }
2716 Vec::new()
2717 } else {
2718 vec![hello_ack]
2719 };
2720 if !self
2721 .registry
2722 .publish_registration(connection_id)
2723 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2724 {
2725 debug!(
2726 module_id = %module_id,
2727 connection_id = connection_id.get(),
2728 "swap candidate registration was removed before it could be published"
2729 );
2730 }
2731 self.supervisor.mark_swap_candidate_admitted(&module_id);
2732 info!(
2733 module_id = %module_id,
2734 module_version = %registration.manifest.module_version,
2735 negotiated_ver,
2736 ready = registration.ready,
2737 connection_id = connection_id.get(),
2738 "swap candidate registered; not routable until cutover"
2739 );
2740 Ok(reply)
2741 }
2742
2743 fn build_hello_ack(
2744 &self,
2745 frame: &Frame,
2746 negotiated_ver: u8,
2747 module_id: &str,
2748 ) -> Result<Frame, RouterError> {
2749 let ack = ModuleHelloAckBody {
2750 negotiated_ver,
2751 subc_ops: module_subc_ops(),
2752 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2753 storage: self
2754 .storage_config
2755 .as_ref()
2756 .map(|cfg| cfg.descriptor_for(module_id)),
2757 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2758 };
2759 let body = serde_json::to_vec(&ack).map_err(|err| {
2760 RouterError::backend(
2761 0,
2762 frame.header.corr,
2763 format!("failed to encode HELLO_ACK: {err}"),
2764 )
2765 })?;
2766
2767 Frame::build_with_version(
2768 negotiated_ver,
2769 FrameType::HelloAck,
2770 control_flags(),
2771 0,
2772 0,
2773 frame.header.corr,
2774 body,
2775 )
2776 .map_err(RouterError::FrameBuild)
2777 }
2778
2779 async fn handle_client_control_request(
2780 &self,
2781 ctx: &RouteCtx,
2782 frame: Frame,
2783 request: ClientControlRequest,
2784 route_open_timings: &mut Option<RouteOpenTimings>,
2785 ) -> Result<Vec<Frame>, RouterError> {
2786 if let Some(operation) = privileged_operation(&request) {
2787 let caller = self.resolve_caller(ctx.connection_id).await;
2788 let op = client_control_request_op(&request);
2789 let text = prompt_text(operation, &caller);
2790 match self.privileged_operator_prompt {
2791 PrivilegedOperatorPrompt::Log => {
2792 info!(target: "subc_daemon::privileged_prompt", operation = op,
2793 target = ?operation, caller = %caller.description(), summary = %text,
2794 "privileged request would prompt the operator");
2795 }
2796 PrivilegedOperatorPrompt::Enforce => {
2797 let key =
2798 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2799 let outcome = self
2800 .forwarding
2801 .operator_confirms()
2802 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2803 .await;
2804 if outcome.code != "confirmed" {
2805 return Ok(vec![outcome.frame(&frame)]);
2806 }
2807 }
2808 }
2809 }
2810 match request {
2811 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2812 ClientControlRequest::CatalogList { module_id } => {
2813 self.handle_catalog_list(frame, module_id)
2814 }
2815 ClientControlRequest::RouteOpen {
2816 target,
2817 identity,
2818 consumer_identity,
2819 consumer_capabilities,
2820 role_versions,
2821 admission_facts,
2822 scope,
2823 } => {
2824 self.handle_route_open(
2825 ctx,
2826 frame,
2827 RouteOpenRequest {
2828 target,
2829 identity,
2830 consumer_identity,
2831 consumer_capabilities,
2832 role_versions,
2833 admission_facts,
2834 scope,
2835 },
2836 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2837 )
2838 .await
2839 }
2840 ClientControlRequest::RoutePoll {
2841 route_channel,
2842 route_epoch,
2843 kind,
2844 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2845 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2846 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2847 self.handle_supervisor_spawn_snapshot(frame)
2848 }
2849 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2850 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2851 }
2852 ClientControlRequest::SupervisorRestart {
2853 module_id,
2854 drain_timeout_ms,
2855 } => {
2856 self.log_supervisor_request_received(
2857 ctx,
2858 frame.header.corr,
2859 ops::SUPERVISOR_RESTART,
2860 Some(&module_id),
2861 None,
2862 )?;
2863 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2864 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2865 .await
2866 }
2867 ClientControlRequest::SupervisorSwap {
2868 module_id,
2869 ready_timeout_ms,
2870 } => {
2871 self.log_supervisor_request_received(
2872 ctx,
2873 frame.header.corr,
2874 ops::SUPERVISOR_SWAP,
2875 Some(&module_id),
2876 None,
2877 )?;
2878 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2879 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2880 .await
2881 }
2882 ClientControlRequest::SupervisorReload { module_id } => {
2883 self.log_supervisor_request_received(
2884 ctx,
2885 frame.header.corr,
2886 ops::SUPERVISOR_RELOAD,
2887 Some(&module_id),
2888 None,
2889 )?;
2890 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2891 self.handle_supervisor_reload(frame, module_id, requester)
2892 .await
2893 }
2894 ClientControlRequest::SupervisorRescan { preview } => {
2895 if !preview {
2896 self.log_supervisor_request_received(
2897 ctx,
2898 frame.header.corr,
2899 ops::SUPERVISOR_RESCAN,
2900 None,
2901 None,
2902 )?;
2903 }
2904 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2905 self.handle_supervisor_rescan(
2906 frame,
2907 if preview {
2908 RescanMode::Preview
2909 } else {
2910 RescanMode::Execute
2911 },
2912 requester,
2913 )
2914 .await
2915 }
2916 ClientControlRequest::SupervisorRescanChecked {
2917 preview,
2918 candidate,
2919 expect,
2920 } => {
2921 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2922 .await
2923 }
2924 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2925 self.log_supervisor_request_received(
2926 ctx,
2927 frame.header.corr,
2928 ops::SUPERVISOR_RELEASE_RESERVED,
2929 Some(&module_id),
2930 None,
2931 )?;
2932 self.handle_supervisor_release_reserved(frame, module_id)
2933 .await
2934 }
2935 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2936 self.log_supervisor_request_received(
2937 ctx,
2938 frame.header.corr,
2939 ops::SUPERVISOR_SET_ENABLED,
2940 Some(&module_id),
2941 Some(enabled),
2942 )?;
2943 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2944 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2945 .await
2946 }
2947 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2948 self.handle_supervisor_health_probe(frame, module_id).await
2949 }
2950 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2951 ClientControlRequest::SupervisorRoutes { module_id } => {
2952 self.handle_supervisor_routes(frame, module_id)
2953 }
2954 ClientControlRequest::SupervisorProvenance { module_id } => {
2955 self.handle_supervisor_provenance(frame, module_id).await
2956 }
2957 ClientControlRequest::SupervisorStderrTail {
2958 module_id,
2959 max_lines,
2960 max_bytes,
2961 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2962 ClientControlRequest::SupervisorTerminals { module_id } => {
2963 self.handle_supervisor_terminals(frame, module_id).await
2964 }
2965 }
2966 }
2967
2968 fn log_supervisor_request_received(
2969 &self,
2970 ctx: &RouteCtx,
2971 corr: u64,
2972 op: &'static str,
2973 module_id: Option<&str>,
2974 enabled: Option<bool>,
2975 ) -> Result<(), RouterError> {
2976 let caller = self.supervisor_requester(ctx, corr)?;
2977 let caller = principal_label(&caller);
2978 info!(
2981 target: "control",
2982 op,
2983 module_id,
2984 enabled,
2985 connection_id = ctx.connection_id.get(),
2986 caller = %caller,
2987 "supervisor request received"
2988 );
2989 Ok(())
2990 }
2991
2992 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2993 Ok(self
2994 .registry
2995 .get_module_by_connection(ctx.connection_id)
2996 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2997 .map(|registration| Principal::Reserved {
2998 module_id: registration.manifest.module_id,
2999 })
3000 .unwrap_or(Principal::Direct))
3001 }
3002
3003 fn handle_module_control_request(
3004 &self,
3005 connection_id: ConnectionId,
3006 frame: Frame,
3007 request: ModuleControlRequestFromModule,
3008 ) -> Result<Vec<Frame>, RouterError> {
3009 match request {
3010 ModuleControlRequestFromModule::CatalogUpdate {
3011 provides,
3012 capabilities,
3013 ready,
3014 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3015 ModuleControlRequestFromModule::LiveRoots {} => {
3016 let registered = self
3017 .registry
3018 .get_module_by_connection(connection_id)
3019 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3020 let Some(registration) = registered else {
3021 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3022 };
3023 let response = self
3024 .forwarding
3025 .live_roots(®istration.manifest.module_id)
3026 .map_err(RouterError::Forwarding)?;
3027 Ok(vec![control_response_body_frame(
3028 &frame,
3029 &response,
3030 "ModuleControlResponseToModule::LiveRoots",
3031 )?])
3032 }
3033 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3034 self.handle_scope_sync(connection_id, frame, generation, scopes)
3035 }
3036 ModuleControlRequestFromModule::ScopeApply {
3037 generation,
3038 upsert,
3039 end,
3040 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3041 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3042 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3043 }
3044 }
3045 }
3046
3047 fn handle_scope_served_by(
3048 &self,
3049 connection_id: ConnectionId,
3050 frame: Frame,
3051 ) -> Result<Vec<Frame>, RouterError> {
3052 let registered = self
3053 .registry
3054 .get_module_by_connection(connection_id)
3055 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3056 let Some(registered) = registered else {
3057 return Ok(vec![control_error_frame(
3058 &frame,
3059 "not_registered",
3060 "scope.served_by requires an active module registration owned by this connection",
3061 )?]);
3062 };
3063 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3064 {
3065 Ok(request) => request,
3066 Err(err) => {
3067 return Ok(vec![control_error_frame(
3068 &frame,
3069 "invalid_control_body",
3070 format!("malformed scope.served_by body: {err}"),
3071 )?])
3072 }
3073 };
3074 let answer = self
3075 .scopes
3076 .read()
3077 .unwrap_or_else(|poisoned| poisoned.into_inner())
3078 .served_by(
3079 ®istered.manifest.module_id,
3080 &request.scope_ref,
3081 request.scope_epoch,
3082 &request.module_id,
3083 );
3084 let reply = subc_control::ScopeServedByReply::new(
3085 answer,
3086 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3087 );
3088 Ok(vec![control_response_body_frame(
3089 &frame,
3090 &reply,
3091 "ScopeServedByReply",
3092 )?])
3093 }
3094
3095 fn handle_operator_confirm(
3096 &self,
3097 ctx: &RouteCtx,
3098 frame: Frame,
3099 ) -> Result<Vec<Frame>, RouterError> {
3100 use crate::operator_confirm::{audit, Outcome};
3101 let registration = self
3102 .registry
3103 .get_module_by_connection(ctx.connection_id)
3104 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3105 let Some(registration) = registration else {
3106 let outcome = Outcome::refusal("not_registered");
3107 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3108 return Ok(vec![outcome.frame(&frame)]);
3109 };
3110 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3111 Ok(request) => request,
3112 Err(_) => {
3113 let outcome = Outcome::refusal("invalid_control_body");
3114 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3115 return Ok(vec![outcome.frame(&frame)]);
3116 }
3117 };
3118 let module_id = registration.manifest.module_id;
3119 let nonce = self
3123 .hello_launch_nonces
3124 .lock()
3125 .unwrap_or_else(|p| p.into_inner())
3126 .nonce(ctx.connection_id)
3127 .map(str::to_owned);
3128 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3129 self.supervisor
3130 .spawned_consumer_authorized(&module_id, nonce)
3131 });
3132 let confirms = self.forwarding.operator_confirms();
3133 self.forwarding
3134 .with_operator_route(
3135 ctx.connection_id,
3136 request.route_channel,
3137 request.route_epoch,
3138 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3139 )
3140 .map_err(RouterError::Forwarding)
3141 }
3142
3143 fn handle_scope_sync(
3147 &self,
3148 connection_id: ConnectionId,
3149 frame: Frame,
3150 generation: u64,
3151 scopes: Vec<ScopeRecord>,
3152 ) -> Result<Vec<Frame>, RouterError> {
3153 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3154 }
3155
3156 fn handle_scope_change(
3157 &self,
3158 connection_id: ConnectionId,
3159 frame: Frame,
3160 generation: u64,
3161 scopes: Vec<ScopeRecord>,
3162 end: Option<Vec<ScopeEnd>>,
3163 ) -> Result<Vec<Frame>, RouterError> {
3164 let is_apply = end.is_some();
3165 let Some(registration) = self
3166 .registry
3167 .get_module_by_connection(connection_id)
3168 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3169 else {
3170 return Ok(vec![control_error_frame(
3171 &frame,
3172 "not_registered",
3173 "scope.sync and scope.apply require an active module registration owned by this connection",
3174 )?]);
3175 };
3176 let owner = registration.manifest.module_id;
3177 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3178 let is_current_launch = |connection: ConnectionId| {
3179 self.hello_launch_nonces
3180 .lock()
3181 .unwrap_or_else(|poisoned| poisoned.into_inner())
3182 .presented(connection, current_nonce.as_deref())
3183 };
3184 let mut table = self
3189 .scopes
3190 .write()
3191 .unwrap_or_else(|poisoned| poisoned.into_inner());
3192 let outcome = match end {
3193 Some(end) => table.apply(
3194 &owner,
3195 connection_id,
3196 is_current_launch,
3197 generation,
3198 scopes,
3199 end,
3200 ),
3201 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3202 };
3203 let drained = match &outcome {
3204 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3205 Err(_) => Vec::new(),
3206 };
3207 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3208 drop(table);
3209 if has_deadlines {
3210 self.scope_deadline_added.notify_one();
3211 }
3212 match outcome {
3213 Ok(applied) => {
3214 let counts = ScopeOutcomeCounts::of(&applied.results);
3215 info!(
3216 owner = %owner,
3217 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3218 generation,
3219 records = applied.results.len(),
3220 created = counts.created,
3221 replaced = counts.replaced,
3222 updated = counts.updated,
3223 unchanged = counts.unchanged,
3224 refused = counts.refused,
3225 ended = applied.ended.len(),
3226 tag_changes = applied.tag_changes.len(),
3227 routes_closed = drained.len(),
3228 "scope change accepted"
3229 );
3230 for refused in applied
3236 .results
3237 .iter()
3238 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3239 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3240 {
3241 warn!(
3242 owner = %owner,
3243 generation,
3244 scope_ref = %refused.scope_ref,
3245 scope_epoch = refused.scope_epoch,
3246 code = refused.code.as_deref().unwrap_or(""),
3247 "scope record refused"
3248 );
3249 }
3250 self.close_scope_drained_routes(drained);
3251 let response = if is_apply {
3252 ModuleControlResponseToModule::ScopeApply {
3253 generation,
3254 results: applied.results,
3255 end_results: applied.end_results,
3256 ended: applied.ended,
3257 }
3258 } else {
3259 ModuleControlResponseToModule::ScopeSync {
3260 generation,
3261 results: applied.results,
3262 ended: applied.ended,
3263 }
3264 };
3265 Ok(vec![control_response_body_frame(
3266 &frame,
3267 &response,
3268 "scope change response",
3269 )?])
3270 }
3271 Err(refusal) => {
3272 info!(
3273 owner = %owner,
3274 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3275 generation,
3276 code = refusal.code,
3277 "scope change refused"
3278 );
3279 Ok(vec![control_error_frame(
3280 &frame,
3281 refusal.code,
3282 refusal.message,
3283 )?])
3284 }
3285 }
3286 }
3287
3288 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3294 if drained.is_empty() {
3295 return;
3296 }
3297 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3298 BTreeMap::new();
3299 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3300 for route in drained {
3301 warn!(
3302 module_id = %route.module_id,
3303 reason = ?route.reason,
3304 client_connection_id = route.client.connection_id.get(),
3305 route_channel = route.client.channel,
3306 "closing route because its scope changed"
3307 );
3308 pushes
3309 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3310 .or_insert_with(|| (route.reason, Vec::new()))
3311 .1
3312 .push(EndpointRoute {
3313 goodbye_target: route.client.clone(),
3314 principal: Principal::Unverified,
3315 bound_at: Instant::now(),
3316 draining: false,
3317 drain_reason: None,
3318 });
3319 goodbyes.push(route.module);
3320 goodbyes.push(route.client);
3321 }
3322 for ((module_id, _), (reason, routes)) in pushes {
3323 send_route_control_pushes(
3324 &self.forwarding,
3325 routes,
3326 ClientControlPush::RouteClosed {
3327 module_id,
3328 channels: Vec::new(),
3329 reason,
3330 drained: false,
3331 abandoned: 0,
3332 excluded_subscriptions: 0,
3333 terminal: Some(false),
3334 },
3335 );
3336 }
3337 self.emit_route_goodbyes(goodbyes);
3338 }
3339
3340 fn handle_scope_describe(
3343 &self,
3344 connection_id: ConnectionId,
3345 frame: Frame,
3346 owner: Principal,
3347 scope_ref: String,
3348 ) -> Result<Vec<Frame>, RouterError> {
3349 let registered = self
3350 .registry
3351 .get_module_by_connection(connection_id)
3352 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3353 let Some(registered) = registered else {
3354 return Ok(vec![control_error_frame(
3355 &frame,
3356 "not_registered",
3357 "scope.describe requires an active module registration owned by this connection",
3358 )?]);
3359 };
3360 let mut description = self
3361 .scopes
3362 .read()
3363 .unwrap_or_else(|poisoned| poisoned.into_inner())
3364 .describe(&owner, &scope_ref);
3365 if !registered
3368 .manifest
3369 .capabilities
3370 .as_ref()
3371 .is_some_and(|caps| {
3372 caps.provides
3373 .iter()
3374 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3375 })
3376 {
3377 if let Some(stamp) = description.stamp.as_mut() {
3378 stamp.attributes.on_behalf_of_agent = None;
3379 }
3380 }
3381 let owner_configured = match &owner {
3382 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3387 _ => false,
3388 };
3389 let response = ModuleControlResponseToModule::ScopeDescribe {
3390 status: description.status,
3391 scope_epoch: description.scope_epoch,
3392 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3393 owner_synced: description.owner_synced,
3394 owner_configured,
3395 scope: description.stamp,
3396 };
3397 Ok(vec![control_response_body_frame(
3398 &frame,
3399 &response,
3400 "ModuleControlResponseToModule::ScopeDescribe",
3401 )?])
3402 }
3403
3404 fn handle_catalog_update(
3405 &self,
3406 connection_id: ConnectionId,
3407 frame: Frame,
3408 provides: Vec<ProviderRole>,
3409 capabilities: Option<CapabilityDeclarations>,
3410 ready: Option<bool>,
3411 ) -> Result<Vec<Frame>, RouterError> {
3412 self.refresh_capability_requirements();
3413 let Some(registration) = self
3414 .registry
3415 .get_module_by_connection(connection_id)
3416 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3417 else {
3418 return Ok(vec![control_error_frame(
3419 &frame,
3420 "not_registered",
3421 "catalog.update requires an active module registration owned by this connection",
3422 )?]);
3423 };
3424
3425 if let Some(message) =
3426 catalog_update_frozen_field_message(®istration.manifest, &provides)
3427 {
3428 return Ok(vec![control_error_frame(
3429 &frame,
3430 "catalog_update_frozen_field",
3431 message,
3432 )?]);
3433 }
3434
3435 let mut candidate = registration.manifest.clone();
3436 candidate.provides = provides.clone();
3437 candidate.capabilities = capabilities
3438 .clone()
3439 .or_else(|| registration.manifest.capabilities.clone());
3440 if let Err(err) = candidate.validate_capability_grammar() {
3441 return Ok(vec![control_error_frame(
3442 &frame,
3443 "invalid_capability_grammar",
3444 err.to_string(),
3445 )?]);
3446 }
3447
3448 let mut conflicts = self
3451 .capability_evaluator
3452 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3453 if let Some(conflict) = conflicts.first() {
3454 let message = format!(
3455 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3456 conflict.capability, conflict.claimants[0], candidate.module_id
3457 );
3458 for conflict in &mut conflicts {
3459 conflict.source = DuplicateClaimSource::CatalogUpdate;
3460 }
3461 log_duplicate_claim_events(conflicts);
3462 return Ok(vec![control_error_frame(
3463 &frame,
3464 "reserved_capability",
3465 message,
3466 )?]);
3467 }
3468
3469 let updated = self
3470 .registry
3471 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3472 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3473 if updated.is_none() {
3474 return Ok(vec![control_error_frame(
3475 &frame,
3476 "not_registered",
3477 "catalog.update requires an active module registration owned by this connection",
3478 )?]);
3479 }
3480 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3481 log_duplicate_claim_events(
3482 self.capability_evaluator
3483 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3484 );
3485 }
3486 if capability_census_trigger(
3487 registration.manifest.capabilities.as_ref(),
3488 updated
3489 .as_ref()
3490 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3491 ) {
3492 self.enforce_capability_denies();
3493 }
3494 self.refresh_capability_requirements();
3495
3496 let response = ModuleControlResponseToModule::CatalogUpdate {};
3497 control_response_body_frame(
3498 &frame,
3499 &response,
3500 "ModuleControlResponseToModule::CatalogUpdate",
3501 )
3502 .map(|frame| vec![frame])
3503 }
3504
3505 pub(crate) fn with_activity_ledger_config(
3506 mut self,
3507 run_dir: &Path,
3508 incarnation: String,
3509 config: crate::activity::ActivityConfig,
3510 ) -> Self {
3511 self.activity = Some(crate::activity::ActivityLedger::open_config(
3512 run_dir.join("activity.jsonl"),
3513 incarnation,
3514 config,
3515 ));
3516 self.supervisor.set_activity(self.activity.clone());
3517 self
3518 }
3519
3520 pub(crate) async fn configure_activity(
3521 &self,
3522 config: crate::activity::ActivityConfig,
3523 ) -> Result<(), &'static str> {
3524 match &self.activity {
3525 Some(ledger) => ledger.configure(config).await,
3526 None => Ok(()),
3527 }
3528 }
3529
3530 fn cancel_activity_subscription(
3531 &self,
3532 ctx: &RouteCtx,
3533 frame: &Frame,
3534 ) -> Result<bool, RouterError> {
3535 let task = self
3536 .activity_streams
3537 .lock()
3538 .unwrap_or_else(|p| p.into_inner())
3539 .remove(&(ctx.connection_id, frame.header.corr));
3540 let Some(task) = task else {
3541 return Ok(false);
3542 };
3543 task.abort();
3544 let end = Frame::build_with_version(
3545 response_version(frame),
3546 FrameType::StreamEnd,
3547 control_flags(),
3548 0,
3549 0,
3550 frame.header.corr,
3551 Vec::new(),
3552 )
3553 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3554 let sink = ctx.egress.clone();
3555 tokio::spawn(async move {
3556 let _ = sink.send(end).await;
3557 });
3558 Ok(true)
3559 }
3560
3561 async fn handle_activity_read(
3562 &self,
3563 ctx: &RouteCtx,
3564 frame: Frame,
3565 ) -> Result<Vec<Frame>, RouterError> {
3566 use subc_control::{
3567 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3568 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3569 };
3570 #[derive(Deserialize)]
3571 struct Identity {
3572 #[serde(default)]
3573 consumer_identity: Option<ConsumerIdentity>,
3574 }
3575 let principal = match self
3579 .forwarding
3580 .module_id_for_connection(ctx.connection_id)
3581 .map_err(RouterError::Forwarding)?
3582 {
3583 Some(module_id) => Ok(Principal::Reserved { module_id }),
3584 None => {
3585 let identity: Identity = match serde_json::from_slice(&frame.body) {
3586 Ok(identity) => identity,
3587 Err(_) => {
3588 return Ok(vec![control_error_frame(
3589 &frame,
3590 ACTIVITY_MALFORMED,
3591 "malformed activity read",
3592 )?])
3593 }
3594 };
3595 self.route_open_principal(&frame, identity.consumer_identity)?
3596 }
3597 };
3598 let admitted = matches!(principal, Ok(Principal::Direct))
3599 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3600 if !admitted {
3601 return Ok(vec![control_error_frame(
3602 &frame,
3603 ACTIVITY_READ_FORBIDDEN,
3604 "activity reads require an operator or trusted mirror principal",
3605 )?]);
3606 }
3607 let Some(ledger) = &self.activity else {
3608 return Ok(vec![control_error_frame(
3609 &frame,
3610 ACTIVITY_UNAVAILABLE,
3611 "activity ledger unavailable",
3612 )?]);
3613 };
3614 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3615 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3616 {
3617 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3618 Ok(request) => ledger.query(request).await,
3619 Err(_) => Err(ACTIVITY_MALFORMED),
3620 };
3621 return match result {
3622 Ok(reply) => Ok(vec![control_response_body_frame(
3623 &frame,
3624 &reply,
3625 ops::ACTIVITY_QUERY,
3626 )?]),
3627 Err(code) => Ok(vec![control_error_frame(
3628 &frame,
3629 code,
3630 "activity query refused",
3631 )?]),
3632 };
3633 }
3634 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3635 Ok(request) => request,
3636 Err(_) => {
3637 return Ok(vec![control_error_frame(
3638 &frame,
3639 ACTIVITY_MALFORMED,
3640 "malformed activity subscription",
3641 )?])
3642 }
3643 };
3644 let mut subscription = match ledger.subscribe(request.since).await {
3645 Ok(subscription) => subscription,
3646 Err(code) => {
3647 return Ok(vec![control_error_frame(
3648 &frame,
3649 code,
3650 "activity subscription refused",
3651 )?])
3652 }
3653 };
3654 let key = (ctx.connection_id, frame.header.corr);
3655 let sink = ctx.egress.clone();
3656 let mut streams = self
3657 .activity_streams
3658 .lock()
3659 .unwrap_or_else(|p| p.into_inner());
3660 streams.retain(|_, task| !task.is_finished());
3661 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3662 return Ok(vec![control_error_frame(
3663 &frame,
3664 "duplicate_subscription",
3665 "activity subscription correlation id already in use",
3666 )?]);
3667 }
3668 let task = tokio::spawn(async move {
3669 loop {
3670 let entry = if let Some(entry) = subscription.replay.pop_front() {
3671 Some(entry)
3672 } else {
3673 subscription.receiver.recv().await
3674 };
3675 let Some(entry) = entry else {
3676 break;
3677 };
3678 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3679 return;
3680 };
3681 let Ok(next) = Frame::build_with_version(
3682 response_version(&frame),
3683 FrameType::StreamData,
3684 control_flags(),
3685 0,
3686 0,
3687 frame.header.corr,
3688 body,
3689 ) else {
3690 return;
3691 };
3692 if sink.send(next).await.is_err() {
3693 return;
3694 }
3695 }
3696 if let Some(code) = subscription.terminal_code() {
3699 if let Ok(end) =
3700 control_error_frame(&frame, code, "activity subscriber fell behind")
3701 {
3702 let _ = sink.send(end).await;
3703 }
3704 }
3705 });
3706 streams.insert(key, task.abort_handle());
3707 Ok(Vec::new())
3708 }
3709
3710 async fn handle_activity_record(
3711 &self,
3712 ctx: &RouteCtx,
3713 frame: Frame,
3714 ) -> Result<Vec<Frame>, RouterError> {
3715 use subc_control::{
3716 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3717 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3718 };
3719 let result = async {
3720 let module = self
3721 .forwarding
3722 .module_id_for_connection(ctx.connection_id)
3723 .ok()
3724 .flatten()
3725 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3726 let request: ActivityRecordRequest =
3727 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3728 crate::activity::validate(&module, &request)?;
3729 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3730 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3731 let at_ms = std::time::SystemTime::now()
3732 .duration_since(std::time::UNIX_EPOCH)
3733 .unwrap_or_default()
3734 .as_millis() as u64;
3735 ledger.record(module, request, actor, at_ms).await
3736 }
3737 .await;
3738 match result {
3739 Ok(entry) => Ok(vec![control_response_body_frame(
3740 &frame,
3741 &ActivityRecordResponse { seq: entry.seq },
3742 "activity.record",
3743 )?]),
3744 Err(code) => Ok(vec![control_error_frame(
3745 &frame,
3746 code,
3747 "activity recording refused",
3748 )?]),
3749 }
3750 }
3751
3752 fn activity_actor(
3753 &self,
3754 connection: ConnectionId,
3755 module: &str,
3756 on: Option<&subc_control::ActivityOn>,
3757 ) -> Result<subc_control::ActivityActor, &'static str> {
3758 use crate::forwarding::{DataRoute, DataRouteState};
3759 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3760 let principal = |id: &str| Principal::Reserved {
3761 module_id: id.to_owned(),
3762 };
3763 let mut actor = ActivityActor {
3764 principal: principal(module),
3765 scope_owner: None,
3766 agent_id: None,
3767 run_id: None,
3768 flow_id: None,
3769 project_root: None,
3770 };
3771 let scope = match on {
3772 None => return Ok(actor),
3773 Some(ActivityOn::Scope(scope)) => {
3774 let selector = subc_protocol::scope::ScopeSelector::new(
3775 principal(module),
3776 scope.scope_ref.clone(),
3777 Some(scope.epoch),
3778 );
3779 self.scopes
3780 .read()
3781 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3782 .attribute(&principal(module), module, &selector, true)
3783 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3784 }
3785 Some(ActivityOn::Route(route)) => {
3786 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3787 .forwarding
3788 .lookup_data_route(connection, route.channel, route.epoch)
3789 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3790 else {
3791 return Err(ACTIVITY_ACTOR_UNKNOWN);
3792 };
3793 if binding.module_id != module {
3794 return Err(ACTIVITY_ACTOR_UNKNOWN);
3795 }
3796 actor.principal = binding.principal.clone();
3797 actor.project_root = binding
3798 .project_root
3799 .as_ref()
3800 .map(|root| root.as_path().to_path_buf());
3801 let Some(bound) = &binding.scope else {
3802 return Ok(actor);
3803 };
3804 let owner = principal(&bound.owner);
3805 let selector = subc_protocol::scope::ScopeSelector::new(
3806 owner.clone(),
3807 bound.scope_ref.clone(),
3808 Some(bound.tag.scope_epoch),
3809 );
3810 let admitted = self
3811 .scopes
3812 .read()
3813 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3814 .attribute(&owner, module, &selector, true)
3815 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3816 if admitted.tag != bound.tag {
3817 return Err(ACTIVITY_ACTOR_UNKNOWN);
3818 }
3819 admitted
3820 }
3821 };
3822 actor.scope_owner = Some(scope.stamp.owner);
3823 actor.agent_id = scope.stamp.attributes.agent_id;
3824 actor.run_id = scope.stamp.attributes.run_id;
3825 actor.flow_id = scope.stamp.attributes.flow_id;
3826 Ok(actor)
3827 }
3828
3829 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3830 self.refresh_capability_requirements();
3831 let mut counters = self.counters.snapshot();
3837 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3838 obj.insert(
3839 "activity_unparseable_lines".into(),
3840 ledger.read_errors().into(),
3841 );
3842 obj.insert(
3843 "activity_lifecycle_append_failures".into(),
3844 ledger.lifecycle_failures().into(),
3845 );
3846 obj.insert(
3847 "activity_started_digest_skips".into(),
3848 ledger.lifecycle_digest_skips().into(),
3849 );
3850 }
3851 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3852 self.forwarding.client_route_concentration(),
3853 counters.as_object_mut(),
3854 ) {
3855 obj.insert(
3856 "client_connections_with_routes".into(),
3857 connections_with_routes.into(),
3858 );
3859 obj.insert("max_routes_on_one_connection".into(), max.into());
3860 }
3861 if let (Some(open_breakers), Some(obj)) = (
3867 self.route_bind_breakers.open_snapshot(),
3868 counters.as_object_mut(),
3869 ) {
3870 obj.insert("route_bind_breakers_open".into(), open_breakers);
3871 }
3872 let response = ClientControlResponse::ServerDescribe {
3873 protocol_ver: PROTOCOL_VERSION,
3874 subc_ops: subc_ops(),
3875 capabilities: self.describe_capabilities(),
3876 connected_clients: self.connected_clients.count(),
3877 counters: Some(counters),
3878 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3879 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3880 capability_requirements: self.capability_requirement_statuses(),
3881 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3882 config_path: self
3883 .rescan
3884 .as_ref()
3885 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
3886 };
3887 Ok(vec![control_response_body_frame(
3888 &frame,
3889 &response,
3890 "ClientControlResponse::ServerDescribe",
3891 )?])
3892 }
3893
3894 fn handle_catalog_list(
3895 &self,
3896 frame: Frame,
3897 module_id: Option<String>,
3898 ) -> Result<Vec<Frame>, RouterError> {
3899 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3900 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3901 })?;
3902 let entries = modules
3903 .into_iter()
3904 .filter(|registration| {
3905 module_id
3906 .as_deref()
3907 .map(|wanted| registration.manifest.module_id == wanted)
3908 .unwrap_or(true)
3909 })
3910 .map(|registration| {
3911 let not_ready = self.not_ready_reason(®istration);
3912 let roles = registration.manifest.provides;
3913 CatalogEntry::new(
3914 registration.manifest.module_id,
3915 roles,
3916 registration.control_ops,
3917 )
3918 .with_ready(not_ready.is_none())
3919 .with_not_ready(not_ready)
3920 .with_module_version(Some(registration.manifest.module_version))
3921 .with_capabilities(registration.manifest.capabilities)
3922 .with_self_signals(registration.manifest.self_signals)
3923 .with_events(registration.manifest.events)
3924 .with_operation_delivery(registration.manifest.operation_delivery)
3925 })
3926 .collect();
3927 let response = ClientControlResponse::CatalogList {
3928 generation,
3929 modules: entries,
3930 subc_ops: subc_ops(),
3931 };
3932 Ok(vec![control_response_body_frame(
3933 &frame,
3934 &response,
3935 "ClientControlResponse::CatalogList",
3936 )?])
3937 }
3938
3939 fn route_open_principal(
3940 &self,
3941 frame: &Frame,
3942 consumer_identity: Option<ConsumerIdentity>,
3943 ) -> Result<Result<Principal, Frame>, RouterError> {
3944 let Some(consumer_identity) = consumer_identity else {
3945 return Ok(Ok(Principal::Direct));
3946 };
3947
3948 if self.supervisor.spawned_consumer_authorized(
3949 &consumer_identity.module_id,
3950 &consumer_identity.launch_nonce,
3951 ) {
3952 return Ok(Ok(Principal::Reserved {
3953 module_id: consumer_identity.module_id,
3954 }));
3955 }
3956
3957 Ok(Err(control_error_frame(
3958 frame,
3959 "bad_consumer_identity",
3960 format!(
3961 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3962 consumer_identity.module_id
3963 ),
3964 )?))
3965 }
3966
3967 fn route_open_refusal_frame(
3974 &self,
3975 ctx: &RouteCtx,
3976 frame: &Frame,
3977 module_id: &str,
3978 reason: &'static str,
3979 code: &'static str,
3980 message: impl Into<String>,
3981 ) -> Result<Frame, RouterError> {
3982 self.observe_route_open_refusal(ctx, module_id, reason, code);
3983 control_error_frame(frame, code, message.into())
3984 }
3985
3986 fn route_open_breaker_refusal_frame(
4004 &self,
4005 ctx: &RouteCtx,
4006 frame: &Frame,
4007 module_id: &str,
4008 consecutive_timeouts: u32,
4009 retry_in: Duration,
4010 probe_in_flight: bool,
4011 ) -> Result<Frame, RouterError> {
4012 self.counters
4013 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4014 debug!(
4015 target: "control",
4016 code = "module_timeout",
4017 module_id = ?module_id,
4018 connection_id = ctx.connection_id.get(),
4019 consecutive_timeouts,
4020 retry_in_ms = retry_in.as_millis() as u64,
4021 probe_in_flight,
4022 "route.open refused by open bind-relay breaker"
4023 );
4024 let detail = if probe_in_flight {
4031 "one test route is already being tried; retry once it settles".to_string()
4032 } else {
4033 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4034 };
4035 control_error_frame(
4036 frame,
4037 "module_timeout",
4038 format!(
4039 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4040 timed out in a row); {detail}; its established routes are unaffected"
4041 ),
4042 )
4043 }
4044
4045 fn observe_route_open_refusal(
4058 &self,
4059 ctx: &RouteCtx,
4060 module_id: &str,
4061 reason: &'static str,
4062 code: &'static str,
4063 ) {
4064 self.counters.increment_route_open_refused(code);
4065 info!(
4066 target: "control",
4067 code,
4068 reason,
4069 module_id = ?module_id,
4070 connection_id = ctx.connection_id.get(),
4071 "route.open refused"
4072 );
4073 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4074 self.route_outages.record_not_serving(module_id, reason);
4075 }
4076 }
4077
4078 fn observe_route_open_accept(
4123 &self,
4124 ctx: &RouteCtx,
4125 module_id: &str,
4126 principal: &str,
4127 scope: &str,
4128 ) {
4129 self.route_outages.record_accepted(module_id);
4130 self.counters.increment_route_open_accepted(principal);
4131 info!(
4132 target: "control",
4133 principal,
4134 module_id,
4135 connection_id = ctx.connection_id.get(),
4136 scope = %scope,
4137 "route.open accepted"
4138 );
4139 }
4140
4141 fn supervised_absent_route_open_refusal_frame(
4142 &self,
4143 ctx: &RouteCtx,
4144 frame: &Frame,
4145 module_id: &str,
4146 code: &'static str,
4147 status: &crate::supervise::ModuleStatus,
4148 ) -> Result<Frame, RouterError> {
4149 self.counters.increment_route_open_refused(code);
4150 info!(
4151 target: "control",
4152 code,
4153 reason = "supervised_not_registered",
4154 module_id = ?module_id,
4155 connection_id = ctx.connection_id.get(),
4156 state = %status.state,
4157 enabled = status.enabled,
4158 live = status.live,
4159 "route.open refused"
4160 );
4161 self.route_outages
4165 .record_not_serving(module_id, "supervised_not_registered");
4166 let message = if let Some(pid) = status.waiting_for_previous_pid {
4167 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4168 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4169 })
4170 } else {
4171 format!(
4172 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4173 status.state, status.enabled, status.live
4174 )
4175 };
4176 control_error_frame(frame, code, message)
4177 }
4178
4179 async fn handle_route_open(
4180 &self,
4181 ctx: &RouteCtx,
4182 frame: Frame,
4183 request: RouteOpenRequest,
4184 timings: &mut RouteOpenTimings,
4185 ) -> Result<Vec<Frame>, RouterError> {
4186 let RouteOpenRequest {
4187 target,
4188 mut identity,
4189 consumer_identity,
4190 consumer_capabilities,
4191 role_versions,
4192 admission_facts,
4193 scope,
4194 } = request;
4195 let target_module_id = target_module_id(&target).to_string();
4196 timings.module_id.clone_from(&target_module_id);
4197 if self
4198 .registry
4199 .get_module_by_connection(ctx.connection_id)
4200 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4201 .is_some()
4202 {
4203 return Ok(vec![control_error_frame(
4204 &frame,
4205 "invalid_request",
4206 "module connections cannot open client routes",
4207 )?]);
4208 }
4209 debug!(
4210 connection_id = ctx.connection_id.get(),
4211 corr = frame.header.corr,
4212 module_id = %target_module_id,
4213 "handling route.open"
4214 );
4215
4216 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4222 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4223 self.observe_route_open_refusal(
4224 ctx,
4225 &target_module_id,
4226 "invalid_role_versions",
4227 error_codes::INVALID_REQUEST,
4228 );
4229 return Ok(vec![control_error_body_frame(
4230 &frame,
4231 ErrorBody {
4232 code: error_codes::INVALID_REQUEST.to_string(),
4233 message: error.to_string(),
4234 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4235 },
4236 )?]);
4237 }
4238
4239 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4240 if status.waiting_for_previous_pid.is_some() {
4241 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4242 ctx,
4243 &frame,
4244 &target_module_id,
4245 "module_warming",
4246 &status,
4247 )?]);
4248 }
4249 }
4250
4251 let Some(registration) = self
4270 .registry
4271 .get_module(&target_module_id)
4272 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4273 else {
4274 if let Some((status, warming)) =
4275 self.supervisor_status(&target_module_id, frame.header.corr)?
4276 {
4277 if status.protocol == ModuleProtocol::None {
4284 return Ok(vec![self.route_open_refusal_frame(
4285 ctx,
4286 &frame,
4287 &target_module_id,
4288 "protocol_none",
4289 error_codes::MODULE_NO_PROTOCOL,
4290 format!(
4291 "module_id '{target_module_id}' is declared protocol: none; \
4292 it speaks no subc wire and serves no routes"
4293 ),
4294 )?]);
4295 }
4296 let code = if warming {
4297 "module_warming"
4298 } else {
4299 "target_unavailable"
4300 };
4301 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4302 ctx,
4303 &frame,
4304 &target_module_id,
4305 code,
4306 &status,
4307 )?]);
4308 }
4309 if let Some(removed_ago_ms) =
4310 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4311 {
4312 return Ok(vec![self.route_open_refusal_frame(
4313 ctx,
4314 &frame,
4315 &target_module_id,
4316 "removed",
4317 error_codes::MODULE_REMOVED,
4318 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4319 )?]);
4320 }
4321 return Ok(vec![self.route_open_refusal_frame(
4322 ctx,
4323 &frame,
4324 &target_module_id,
4325 "not_registered",
4326 error_codes::UNKNOWN_MODULE,
4327 format!("module_id '{target_module_id}' is not registered"),
4328 )?]);
4329 };
4330
4331 if !registration.ready {
4335 self.counters
4336 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4337 info!(
4338 target: "control",
4339 code = error_codes::MODULE_WARMING,
4340 module_id = ?target_module_id,
4341 connection_id = ctx.connection_id.get(),
4342 reason = "declared_not_ready",
4343 "route.open refused"
4344 );
4345 self.route_outages
4348 .record_not_serving(&target_module_id, "declared_not_ready");
4349 return Ok(vec![control_error_body_frame(
4350 &frame,
4351 ErrorBody {
4352 code: error_codes::MODULE_WARMING.to_string(),
4353 message: format!(
4354 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4355 ),
4356 detail: Some(serde_json::json!({
4357 "reason": "declared_not_ready"
4358 })),
4359 },
4360 )?]);
4361 }
4362
4363 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4382 self.counters
4383 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4384 info!(
4385 target: "control",
4386 code = error_codes::MODULE_WARMING,
4387 module_id = ?target_module_id,
4388 connection_id = ctx.connection_id.get(),
4389 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4390 capability = %capability,
4391 "route.open refused"
4392 );
4393 return Ok(vec![control_error_body_frame(
4394 &frame,
4395 ErrorBody {
4396 code: error_codes::MODULE_WARMING.to_string(),
4397 message: format!(
4398 "module_id '{target_module_id}' requires capability '{capability}', \
4399 which no registered module provides; retry"
4400 ),
4401 detail: Some(serde_json::json!({
4402 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4403 "capability": capability,
4404 })),
4405 },
4406 )?]);
4407 }
4408
4409 if !target_has_required_role(&target, ®istration.manifest.provides) {
4410 return Ok(vec![self.route_open_refusal_frame(
4411 ctx,
4412 &frame,
4413 &target_module_id,
4414 "role_not_provided",
4415 "target_unavailable",
4416 format!("module_id '{target_module_id}' does not provide the requested target"),
4417 )?]);
4418 }
4419
4420 if registration.state != ChannelState::Active {
4421 return Ok(vec![self.route_open_refusal_frame(
4422 ctx,
4423 &frame,
4424 &target_module_id,
4425 "registration_not_active",
4426 "target_unavailable",
4427 format!("module_id '{target_module_id}' is not active"),
4428 )?]);
4429 }
4430
4431 if self
4432 .forwarding
4433 .module_is_draining(&target_module_id)
4434 .map_err(RouterError::Forwarding)?
4435 {
4436 return Ok(vec![self.route_open_refusal_frame(
4437 ctx,
4438 &frame,
4439 &target_module_id,
4440 "reloading",
4441 "module_reloading",
4442 format!("module_id '{target_module_id}' is reloading"),
4443 )?]);
4444 }
4445
4446 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4447 process_liveness.process_live(&target_module_id) == Some(false)
4448 }) {
4449 if process_liveness.process_replacing(&target_module_id) {
4456 return Ok(vec![self.route_open_refusal_frame(
4457 ctx,
4458 &frame,
4459 &target_module_id,
4460 "reloading",
4461 "module_reloading",
4462 format!("module_id '{target_module_id}' is reloading"),
4463 )?]);
4464 }
4465 return Ok(vec![self.route_open_refusal_frame(
4466 ctx,
4467 &frame,
4468 &target_module_id,
4469 "supervisor_not_live",
4470 "target_unavailable",
4471 format!("module_id '{target_module_id}' is not live"),
4472 )?]);
4473 }
4474
4475 if !self
4476 .forwarding
4477 .has_live_module_connection(&target_module_id)
4478 .map_err(RouterError::Forwarding)?
4479 {
4480 return Ok(vec![self.route_open_refusal_frame(
4481 ctx,
4482 &frame,
4483 &target_module_id,
4484 "no_forwarding_connection",
4485 "target_unavailable",
4486 format!("module_id '{target_module_id}' has no live forwarding connection"),
4487 )?]);
4488 }
4489
4490 if let Some(error) =
4491 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4492 {
4493 self.observe_route_open_refusal(
4494 ctx,
4495 &target_module_id,
4496 "op_not_allowed",
4497 "op_not_allowed",
4498 );
4499 return Ok(vec![error]);
4500 }
4501
4502 let principal = match self.route_open_principal(&frame, consumer_identity)? {
4503 Ok(principal) => principal,
4504 Err(error) => {
4505 self.observe_route_open_refusal(
4506 ctx,
4507 &target_module_id,
4508 "bad_consumer_identity",
4509 "bad_consumer_identity",
4510 );
4511 return Ok(vec![error]);
4512 }
4513 };
4514 timings.principal = principal_label(&principal);
4515
4516 if let Principal::Reserved {
4520 module_id: opening_module_id,
4521 } = &principal
4522 {
4523 if let Some(opening_registration) = self
4524 .registry
4525 .get_module(opening_module_id)
4526 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4527 {
4528 if let Some(capability) =
4529 denied_capability(&opening_registration.manifest, ®istration.manifest)
4530 {
4531 warn!(
4532 opening_module_id,
4533 target_module_id,
4534 capability,
4535 "refusing route.open because an attested capability deny edge matches"
4536 );
4537 return Ok(vec![self.route_open_refusal_frame(
4538 ctx,
4539 &frame,
4540 &target_module_id,
4541 "capability_deny_edge",
4542 "capability_forbidden",
4543 format!(
4544 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4545 ),
4546 )?]);
4547 }
4548 }
4549 }
4550
4551 if admission_facts.is_some() {
4552 let carrier_matches = matches!(
4553 &principal,
4554 Principal::Reserved { module_id }
4555 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4556 );
4557 if !carrier_matches {
4558 return Ok(vec![self.route_open_refusal_frame(
4559 ctx,
4560 &frame,
4561 &target_module_id,
4562 "admission_facts_carrier_not_permitted",
4563 "admission_facts_not_permitted",
4564 "admission facts may only be carried by the configured reserved module",
4565 )?]);
4566 }
4567
4568 let target_allowed = self
4569 .admission_facts_targets
4570 .as_ref()
4571 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4572 if !target_allowed {
4573 return Ok(vec![self.route_open_refusal_frame(
4574 ctx,
4575 &frame,
4576 &target_module_id,
4577 "admission_facts_target_not_listed",
4578 "admission_facts_target_not_allowed",
4579 format!(
4580 "admission facts are not permitted for target module_id '{target_module_id}'"
4581 ),
4582 )?]);
4583 }
4584
4585 }
4589
4590 let (bound_scope, mut scope_stamp) = match &scope {
4596 None => (None, None),
4597 Some(selector) => {
4598 let owner_configured = match &selector.owner {
4599 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4605 _ => false,
4606 };
4607 let mut scopes = self
4608 .scopes
4609 .write()
4610 .unwrap_or_else(|poisoned| poisoned.into_inner());
4611 let admitted = scopes
4612 .admit(&principal, &target_module_id, selector, owner_configured)
4613 .and_then(|admission| {
4614 crate::scopes::check_target_flow_support(
4615 &admission.stamp,
4616 &target_module_id,
4617 registration.manifest.capabilities.as_ref(),
4618 )?;
4619 crate::scopes::check_target_agent_run_support(
4620 &admission.stamp,
4621 &target_module_id,
4622 registration.manifest.capabilities.as_ref(),
4623 )?;
4624 let bound = scopes.reserve_bind(
4625 &admission,
4626 &principal,
4627 ctx.connection_id,
4628 selector.hold_presence,
4629 );
4630 Ok((admission, bound))
4631 });
4632 match admitted {
4633 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4634 Err(refusal) => {
4635 return Ok(vec![self.route_open_refusal_frame(
4636 ctx,
4637 &frame,
4638 &target_module_id,
4639 refusal.code,
4640 refusal.code,
4641 refusal.message,
4642 )?]);
4643 }
4644 }
4645 }
4646 };
4647
4648 timings.enter(RouteOpenPhase::ProjectRoot);
4670 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4671 Ok(project_root) => project_root,
4672 Err(err) => {
4673 return Ok(vec![control_error_frame(
4674 &frame,
4675 "invalid_project_root",
4676 err.to_string(),
4677 )?])
4678 }
4679 };
4680 timings.enter(RouteOpenPhase::Admission);
4681 identity.project_root = project_root.as_path().to_path_buf();
4682
4683 let _concurrency_guard = match self
4694 .route_bind_concurrency
4695 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4696 {
4697 Ok(guard) => guard,
4698 Err(in_flight) => {
4699 return Ok(vec![self.route_open_target_capacity_refusal(
4700 ctx,
4701 &frame,
4702 &target_module_id,
4703 in_flight,
4704 )?]);
4705 }
4706 };
4707
4708 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4711 RouteBindAdmission::Admitted { guard, probe } => {
4712 if probe {
4713 info!(
4714 module_id = %target_module_id,
4715 connection_id = ctx.connection_id.get(),
4716 "route.bind breaker half-open: admitting one probe"
4717 );
4718 }
4719 guard
4720 }
4721 RouteBindAdmission::Refused {
4722 consecutive_timeouts,
4723 retry_in,
4724 probe_in_flight,
4725 } => {
4726 return Ok(vec![self.route_open_breaker_refusal_frame(
4727 ctx,
4728 &frame,
4729 &target_module_id,
4730 consecutive_timeouts,
4731 retry_in,
4732 probe_in_flight,
4733 )?]);
4734 }
4735 };
4736
4737 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4742 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4743 timings.enter(RouteOpenPhase::EgressReservation);
4744 let pending = match self
4745 .forwarding
4746 .begin_route_bind_relay_for(
4747 ctx.connection_id,
4748 ctx.egress.clone(),
4749 response_version(&frame),
4750 frame.header.corr,
4751 &target_module_id,
4752 principal.clone(),
4753 bound_scope,
4754 Some(project_root),
4755 relay_deadline,
4756 )
4757 .await
4758 {
4759 Ok(pending) => pending,
4760 Err(err) => {
4761 return Ok(vec![self.route_open_refusal_frame(
4762 ctx,
4763 &frame,
4764 &target_module_id,
4765 "relay_reservation_failed",
4766 forwarding_error_code(&err),
4767 err.to_string(),
4768 )?])
4769 }
4770 };
4771 timings.enter(RouteOpenPhase::Admission);
4772 let crate::forwarding::PendingRouteBindRelay {
4773 endpoint,
4774 module_sink,
4775 negotiated_ver,
4776 client_channel,
4777 client_epoch,
4778 module_channel,
4779 module_epoch,
4780 corr: relay_corr,
4781 receiver,
4782 } = pending;
4783 let mut reservation =
4784 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4785
4786 let relay_registration = self
4791 .registry
4792 .get_module_by_connection(endpoint.connection_id)
4793 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4794 let relay_capabilities = relay_registration
4795 .as_ref()
4796 .and_then(|registration| registration.manifest.capabilities.as_ref());
4797 if let Some(selector) = &scope {
4798 if let Err(refusal) = self
4801 .scopes
4802 .read()
4803 .unwrap_or_else(|poisoned| poisoned.into_inner())
4804 .check_bind_proof(&principal, selector)
4805 {
4806 reservation.release_and_disarm();
4807 return Ok(vec![self.route_open_refusal_frame(
4808 ctx,
4809 &frame,
4810 &target_module_id,
4811 refusal.code,
4812 refusal.code,
4813 refusal.message,
4814 )?]);
4815 }
4816 }
4817 if let Some(stamp) = scope_stamp
4818 .as_ref()
4819 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4820 {
4821 if let Err(refusal) = crate::scopes::check_target_flow_support(
4822 stamp,
4823 &target_module_id,
4824 relay_capabilities,
4825 )
4826 .and_then(|()| {
4827 crate::scopes::check_target_agent_run_support(
4828 stamp,
4829 &target_module_id,
4830 relay_capabilities,
4831 )
4832 }) {
4833 reservation.release_and_disarm();
4834 return Ok(vec![self.route_open_refusal_frame(
4835 ctx,
4836 &frame,
4837 &target_module_id,
4838 refusal.code,
4839 refusal.code,
4840 refusal.message,
4841 )?]);
4842 }
4843 }
4844
4845 if !relay_capabilities.is_some_and(|caps| {
4848 caps.provides
4849 .iter()
4850 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
4851 }) {
4852 if let Some(stamp) = scope_stamp.as_mut() {
4853 stamp.attributes.on_behalf_of_agent = None;
4854 }
4855 }
4856 let call_mode = if relay_capabilities
4857 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4858 {
4859 Some(derive_call_mode(scope_stamp.as_ref()))
4860 } else {
4861 if let Some(stamp) = scope_stamp.as_mut() {
4862 stamp.expires_at_ms = None;
4863 }
4864 None
4865 };
4866
4867 debug!(
4868 connection_id = ctx.connection_id.get(),
4869 client_channel,
4870 client_epoch,
4871 module_channel,
4872 module_epoch,
4873 "reserved route handle pair"
4874 );
4875 let route_principal_label = principal_label(&principal);
4878 let scope_label = scope_stamp
4879 .as_ref()
4880 .map_or_else(|| "none".to_string(), route_open_scope_label);
4881 let relay = ModuleControlRequest::RouteBind {
4882 route_channel: module_channel,
4883 epoch: module_epoch,
4884 target,
4885 identity,
4886 principal: Some(principal),
4887 consumer_capabilities,
4888 role_versions,
4889 admission_facts,
4890 scope: scope_stamp,
4891 call_mode,
4892 };
4893 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4894 RouterError::backend(
4895 0,
4896 frame.header.corr,
4897 format!("failed to encode route.bind request: {err}"),
4898 )
4899 })?;
4900 let relay_frame = Frame::build_with_version(
4901 negotiated_ver,
4902 FrameType::Request,
4903 control_flags(),
4904 0,
4905 0,
4906 relay_corr,
4907 relay_body,
4908 )
4909 .map_err(RouterError::FrameBuild)?;
4910
4911 timings.enter(RouteOpenPhase::BindSend);
4912 let sent = module_sink.send(relay_frame).await;
4913 timings.enter(RouteOpenPhase::Admission);
4914 if let Err(err) = sent {
4915 reservation.release_and_disarm();
4916 return Ok(vec![self.route_open_refusal_frame(
4917 ctx,
4918 &frame,
4919 &target_module_id,
4920 "relay_send_failed",
4921 "target_unavailable",
4922 err.to_string(),
4923 )?]);
4924 }
4925
4926 #[cfg(test)]
4927 if let Some(hold) = &self.route_bind_relay_sent_hold {
4928 hold.wait().await;
4932 hold.wait().await;
4933 }
4934
4935 if self
4940 .forwarding
4941 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4942 .map_err(RouterError::Forwarding)?
4943 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4944 {
4945 self.send_abandoned_route_bind_goodbye(
4946 &module_sink,
4947 negotiated_ver,
4948 module_channel,
4949 module_epoch,
4950 );
4951 }
4952
4953 timings.enter(RouteOpenPhase::BindAckWait);
4954 let outcome = timeout_at(relay_deadline, receiver).await;
4955 timings.enter(RouteOpenPhase::Admission);
4956 match outcome {
4957 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4958 reservation.disarm();
4959 if breaker.record_accepted() {
4960 info!(
4961 module_id = %target_module_id,
4962 "route.bind breaker closed: the probe was accepted"
4963 );
4964 }
4965 self.observe_route_open_accept(
4966 ctx,
4967 &target_module_id,
4968 &route_principal_label,
4969 &scope_label,
4970 );
4971 Ok(Vec::new())
4972 }
4973 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4974 reservation.release_and_disarm();
4975 breaker.record_inconclusive();
4979 let scope_code = match body.code.as_str() {
4983 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4984 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4985 _ => None,
4986 };
4987 if let Some(code) = scope_code {
4988 self.observe_route_open_refusal(
4989 ctx,
4990 &target_module_id,
4991 "scope_changed_before_commit",
4992 code,
4993 );
4994 return Ok(vec![control_error_body_frame(&frame, body)?]);
4995 }
4996 self.counters
4997 .increment_route_open_refused("module_rejected");
4998 info!(
4999 target: "control",
5000 code = "module_rejected",
5001 module_code = ?body.code,
5002 module_id = ?target_module_id,
5003 connection_id = ctx.connection_id.get(),
5004 "route.open refused"
5005 );
5006 Ok(vec![control_error_body_frame(&frame, body)?])
5007 }
5008 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5009 reservation.release_and_disarm();
5010 breaker.record_inconclusive();
5011 tracing::warn!(
5018 module_id = %target_module_id,
5019 "route.bind relay abandoned: {message}"
5020 );
5021 Ok(vec![self.route_open_refusal_frame(
5022 ctx,
5023 &frame,
5024 &target_module_id,
5025 "relay_abandoned",
5026 "target_unavailable",
5027 message,
5028 )?])
5029 }
5030 Ok(Err(_)) => {
5031 reservation.release_and_disarm();
5032 breaker.record_inconclusive();
5033 Ok(vec![self.route_open_refusal_frame(
5034 ctx,
5035 &frame,
5036 &target_module_id,
5037 "relay_waiter_canceled",
5038 "target_unavailable",
5039 "route.bind relay waiter was canceled before the module responded",
5040 )?])
5041 }
5042 Err(_) => {
5043 reservation.release_and_disarm();
5044 if let Some(opened) = breaker.record_timeout(
5048 self.route_bind_breaker_threshold,
5049 self.route_bind_breaker_cooldown,
5050 ) {
5051 warn!(
5052 module_id = %target_module_id,
5053 consecutive_timeouts = opened.consecutive_timeouts,
5054 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5055 reopened_after_probe = opened.reopened_after_probe,
5056 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5057 );
5058 }
5059 tracing::warn!(
5067 module_id = %target_module_id,
5068 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5069 "route.bind relay timed out: module did not ack within budget"
5070 );
5071 Ok(vec![self.route_open_refusal_frame(
5072 ctx,
5073 &frame,
5074 &target_module_id,
5075 "relay_timed_out",
5076 "module_timeout",
5077 format!(
5078 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5079 route_bind_relay_timeout
5080 ),
5081 )?])
5082 }
5083 }
5084 }
5085
5086 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5087 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5088 snapshot: self.supervisor.spawn_snapshot(),
5089 };
5090 Ok(vec![control_response_body_frame(
5091 &frame,
5092 &response,
5093 "ClientControlResponse::SupervisorSpawnSnapshot",
5094 )?])
5095 }
5096
5097 fn handle_supervisor_spawn_subscribe(
5098 &self,
5099 ctx: &RouteCtx,
5100 frame: Frame,
5101 since: Option<SpawnCursor>,
5102 ) -> Result<Vec<Frame>, RouterError> {
5103 match self.supervisor.subscribe_spawns(
5104 ctx.connection_id,
5105 frame.header.corr,
5106 response_version(&frame),
5107 since,
5108 ctx.egress.clone(),
5109 ) {
5110 Ok(()) => Ok(Vec::new()),
5111 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5112 Ok(vec![control_error_body_frame(
5113 &frame,
5114 ErrorBody {
5115 code: "spawn_cursor_incarnation_mismatch".to_string(),
5116 message: "spawn cursor belongs to a different daemon incarnation"
5117 .to_string(),
5118 detail: Some(serde_json::json!({
5119 "current_daemon_incarnation": current
5120 })),
5121 },
5122 )?])
5123 }
5124 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5125 &frame,
5126 ErrorBody {
5127 code: "spawn_cursor_too_old".to_string(),
5128 message: "spawn cursor predates the retained event ring".to_string(),
5129 detail: Some(serde_json::json!({
5130 "oldest_retained_cursor": oldest
5131 })),
5132 },
5133 )?]),
5134 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5135 0,
5136 frame.header.corr,
5137 format!("failed to open supervisor spawn subscription: {error}"),
5138 )),
5139 }
5140 }
5141
5142 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5143 let generation = self
5144 .registry
5145 .generation()
5146 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5147 let mut modules = Vec::new();
5148 for module in self.supervisor.list() {
5149 let status = module.status_for_control("list").map_err(|err| {
5150 RouterError::backend(
5151 0,
5152 frame.header.corr,
5153 format!("failed to read supervisor status: {err}"),
5154 )
5155 })?;
5156 let (configured, _) = module.configuration().map_err(|err| {
5157 RouterError::backend(
5158 0,
5159 frame.header.corr,
5160 format!("failed to read module configuration: {err}"),
5161 )
5162 })?;
5163 let image = module.running_image_agreement().await;
5165 let resources = Some(module.child_resource_usage());
5168 let mut verdict =
5169 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5170 verdict.spec = Some(launch_spec_agreement(
5171 &configured,
5172 status.process_alive,
5173 status.launch_spec_digest.as_ref(),
5174 ));
5175 let pending_reload = Some(verdict);
5176 modules.push(
5179 SupervisorEntry::new(
5180 status.module_id,
5181 status.state.to_string(),
5182 status.enabled,
5183 status.live,
5184 status.health.status,
5185 )
5186 .with_launch_nonce_env(Some(!cfg!(unix)))
5187 .with_launch_nonce_source(status.launch_nonce_source)
5188 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5189 .with_protocol(status.protocol)
5190 .with_pending_reload(pending_reload)
5191 .with_last_probe_ms(status.health.last_probe_ms)
5192 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5193 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5194 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5195 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5196 .with_restart_count(Some(status.restart_count))
5197 .with_max_restarts(Some(status.max_restarts))
5198 .with_lifetime_restarts(Some(status.lifetime_restarts))
5199 .with_spawn_generation(Some(status.spawn_generation))
5200 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5201 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5202 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5203 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5204 .with_resources(resources),
5205 );
5206 }
5207 let response = ClientControlResponse::SupervisorList {
5208 generation,
5209 modules,
5210 };
5211 Ok(vec![control_response_body_frame(
5212 &frame,
5213 &response,
5214 "ClientControlResponse::SupervisorList",
5215 )?])
5216 }
5217
5218 fn handle_supervisor_stderr_tail(
5219 &self,
5220 frame: Frame,
5221 module_id: String,
5222 max_lines: Option<u32>,
5223 max_bytes: Option<u32>,
5224 ) -> Result<Vec<Frame>, RouterError> {
5225 let Some(module) = self.supervisor.get(&module_id) else {
5226 return Ok(vec![control_error_frame(
5227 &frame,
5228 "unknown_module",
5229 format!("module_id '{module_id}' is not supervised"),
5230 )?]);
5231 };
5232
5233 let snapshot = module.stderr_tail(
5234 max_lines.map(|value| value as usize),
5235 max_bytes.map(|value| value as usize),
5236 );
5237
5238 let response = ClientControlResponse::SupervisorStderrTail {
5239 module_id,
5240 tail: StderrTail {
5241 capture: match snapshot.capture {
5242 CaptureState::Captured => StderrCaptureState::Captured,
5243 CaptureState::Incomplete { reason } => {
5244 StderrCaptureState::Incomplete { reason }
5245 }
5246 CaptureState::NotCaptured { reason } => {
5247 StderrCaptureState::NotCaptured { reason }
5248 }
5249 },
5250 entries: snapshot
5251 .entries
5252 .into_iter()
5253 .map(|entry| match entry {
5254 TailEntry::Line {
5255 text,
5256 truncated,
5257 at_ms,
5258 } => StderrTailEntry::Line {
5259 text,
5260 truncated,
5261 at_ms,
5262 },
5263 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5264 })
5265 .collect(),
5266 dropped_lines: snapshot.dropped_lines,
5267 },
5268 };
5269 Ok(vec![control_response_body_frame(
5270 &frame,
5271 &response,
5272 "ClientControlResponse::SupervisorStderrTail",
5273 )?])
5274 }
5275
5276 async fn handle_supervisor_terminals(
5277 &self,
5278 frame: Frame,
5279 module_id: String,
5280 ) -> Result<Vec<Frame>, RouterError> {
5281 let Some(module) = self.supervisor.get(&module_id) else {
5282 return Ok(vec![control_error_frame(
5283 &frame,
5284 "unknown_module",
5285 format!("module_id '{module_id}' is not supervised"),
5286 )?]);
5287 };
5288
5289 let terminals = module
5292 .read_durable_terminal_history()
5293 .await
5294 .map_err(|error| {
5295 RouterError::backend(
5296 0,
5297 frame.header.corr,
5298 format!("failed to read terminal history: {error}"),
5299 )
5300 })?;
5301 let response = ClientControlResponse::SupervisorTerminals {
5302 module_id,
5303 terminals,
5304 };
5305 Ok(vec![control_response_body_frame(
5306 &frame,
5307 &response,
5308 "ClientControlResponse::SupervisorTerminals",
5309 )?])
5310 }
5311
5312 fn handle_supervisor_routes(
5313 &self,
5314 frame: Frame,
5315 module_id: Option<String>,
5316 ) -> Result<Vec<Frame>, RouterError> {
5317 let modules = self
5318 .forwarding
5319 .route_census(module_id.as_deref())
5320 .map_err(RouterError::Forwarding)?
5321 .into_iter()
5322 .map(|(module_id, routes)| SupervisorRouteModule {
5323 module_id,
5324 routes: routes
5325 .into_iter()
5326 .map(|route| SupervisorRoute {
5327 consumer: match route.principal {
5328 Principal::Reserved { module_id } => {
5329 SupervisorRouteConsumer::Reserved { module_id }
5330 }
5331 Principal::Direct | Principal::Unverified => {
5332 SupervisorRouteConsumer::Direct {
5333 connection_id: route.goodbye_target.connection_id.get(),
5334 }
5335 }
5336 },
5337 age_ms: Instant::now()
5338 .saturating_duration_since(route.bound_at)
5339 .as_millis()
5340 .try_into()
5341 .unwrap_or(u64::MAX),
5342 draining: route.draining,
5343 drain_reason: route.drain_reason,
5344 })
5345 .collect(),
5346 })
5347 .collect();
5348 let response = ClientControlResponse::SupervisorRoutes { modules };
5349 Ok(vec![control_response_body_frame(
5350 &frame,
5351 &response,
5352 "ClientControlResponse::SupervisorRoutes",
5353 )?])
5354 }
5355
5356 async fn handle_supervisor_provenance(
5357 &self,
5358 frame: Frame,
5359 module_id: Option<String>,
5360 ) -> Result<Vec<Frame>, RouterError> {
5361 let mut selected = if let Some(module_id) = module_id {
5362 let Some(module) = self.supervisor.get(&module_id) else {
5363 return Ok(vec![control_error_frame(
5364 &frame,
5365 "unknown_module",
5366 format!("module_id '{module_id}' is not supervised"),
5367 )?]);
5368 };
5369 vec![module]
5370 } else {
5371 self.supervisor.list()
5372 };
5373
5374 let mut modules = Vec::with_capacity(selected.len());
5375 for module in selected.drain(..) {
5376 let (status, observed_image) = module
5377 .status_and_running_image_agreement()
5378 .await
5379 .map_err(|err| {
5380 RouterError::backend(
5381 0,
5382 frame.header.corr,
5383 format!("failed to read supervisor status: {err}"),
5384 )
5385 })?;
5386 let module_declared = self
5387 .registry
5388 .get_module(&status.module_id)
5389 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5390 .and_then(|registration| registration.manifest.provenance)
5391 .map(|build| ModuleDeclaredProvenance::Reported { build })
5392 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5393 #[cfg(test)]
5394 let running_image = match &self.provenance_probe_override {
5395 Some(result) => result.clone(),
5396 None => observed_image,
5397 };
5398 #[cfg(not(test))]
5399 let running_image = observed_image;
5400 modules.push(SupervisorModuleProvenance {
5401 module_id: status.module_id,
5402 module_declared,
5403 daemon_observed: SupervisorObservedProcess {
5404 launch_nonce_source: status.launch_nonce_source,
5405 pid: status.pid,
5406 spawned_at_ms: status.spawned_at_ms,
5407 spawned_from: status.spawned_from,
5408 running_image,
5409 },
5410 });
5411 }
5412 let daemon = SupervisorDaemonProvenance {
5413 daemon_build: self.daemon_provenance.build.clone(),
5414 daemon_observed: DaemonObservedProcess {
5415 pid: self.daemon_provenance.pid,
5416 started_at_ms: self
5417 .daemon_provenance
5418 .start_clock
5419 .map(|clock| clock.started_at_ms())
5420 .or(self.daemon_provenance.started_at_ms),
5421 running_image: self
5422 .daemon_provenance
5423 .probe
5424 .observe(
5425 self.daemon_provenance.pid,
5426 self.daemon_provenance.executable_path.as_deref(),
5427 self.daemon_provenance.executable_identity.clone(),
5428 self.daemon_provenance.process_start_time,
5429 )
5430 .await,
5431 },
5432 };
5433 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5434 Ok(vec![control_response_body_frame(
5435 &frame,
5436 &response,
5437 "ClientControlResponse::SupervisorProvenance",
5438 )?])
5439 }
5440
5441 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5442 self.refresh_capability_requirements();
5443 let generation = self
5444 .registry
5445 .generation()
5446 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5447 let modules = self
5448 .supervisor
5449 .list()
5450 .into_iter()
5451 .map(|module| {
5452 let status = module.status_for_control("health").map_err(|err| {
5453 RouterError::backend(
5454 0,
5455 frame.header.corr,
5456 format!("failed to read supervisor health: {err}"),
5457 )
5458 })?;
5459 let module_id = status.module_id;
5460 let capability_detail = self
5461 .capability_evaluator
5462 .required_problem_detail(&module_id);
5463 Ok(SupervisorHealthEntry {
5464 module_id,
5465 status: status.health.status,
5466 detail: append_capability_problem_detail(
5467 status.health.detail,
5468 capability_detail,
5469 ),
5470 metrics: status.health.metrics,
5471 consecutive_failures: status.health.consecutive_failures,
5472 late_answer_count: status.health.late_answer_count,
5473 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5474 last_action: status.health.last_action,
5475 last_action_ms: status.health.last_action_ms,
5476 last_probe_ms: status.health.last_probe_ms,
5477 })
5478 })
5479 .collect::<Result<Vec<_>, RouterError>>()?;
5480 let response = ClientControlResponse::SupervisorHealth {
5481 generation,
5482 modules,
5483 };
5484 Ok(vec![control_response_body_frame(
5485 &frame,
5486 &response,
5487 "ClientControlResponse::SupervisorHealth",
5488 )?])
5489 }
5490
5491 async fn handle_supervisor_restart(
5492 &self,
5493 frame: Frame,
5494 module_id: String,
5495 drain_timeout_ms: Option<u64>,
5496 requester: Principal,
5497 ) -> Result<Vec<Frame>, RouterError> {
5498 let operation_lock = self.supervisor.operation_lock();
5499 let _operation_guard = operation_lock.lock().await;
5500 let Some(module) = self.supervisor.get(&module_id) else {
5501 return Ok(vec![control_error_frame(
5502 &frame,
5503 "unknown_module",
5504 format!("module_id '{module_id}' is not supervised"),
5505 )?]);
5506 };
5507
5508 self.route_outages.mark_operator_action(&module_id);
5509 if let Err(err) = module
5510 .restart_requested(drain_timeout_ms, Some(requester))
5511 .await
5512 {
5513 self.route_outages
5514 .operator_action_ended_unrefused(&module_id);
5515 let (code, message) = match err {
5516 crate::supervise::SuperviseError::Disabled { .. } => {
5517 ("module_disabled", err.to_string())
5518 }
5519 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5520 ("swap_in_progress", err.to_string())
5521 }
5522 _ => (
5523 "target_unavailable",
5524 format!("failed to restart module_id '{module_id}': {err}"),
5525 ),
5526 };
5527 return Ok(vec![control_error_frame(&frame, code, message)?]);
5528 }
5529
5530 let response = ClientControlResponse::SupervisorAck {
5531 module_id,
5532 applied: true,
5533 };
5534 Ok(vec![control_response_body_frame(
5535 &frame,
5536 &response,
5537 "ClientControlResponse::SupervisorAck",
5538 )?])
5539 }
5540
5541 async fn handle_supervisor_swap(
5545 &self,
5546 frame: Frame,
5547 module_id: String,
5548 ready_timeout_ms: Option<u64>,
5549 requester: Principal,
5550 ) -> Result<Vec<Frame>, RouterError> {
5551 let module = {
5558 let operation_lock = self.supervisor.operation_lock();
5559 let _operation_guard = operation_lock.lock().await;
5560 self.supervisor.get(&module_id)
5561 };
5562 let Some(module) = module else {
5563 return Ok(vec![control_error_frame(
5564 &frame,
5565 "unknown_module",
5566 format!("module_id '{module_id}' is not supervised"),
5567 )?]);
5568 };
5569
5570 self.route_outages.mark_operator_action(&module_id);
5571 if let Err(err) = module
5572 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5573 .await
5574 {
5575 self.route_outages
5576 .operator_action_ended_unrefused(&module_id);
5577 use crate::supervise::SuperviseError;
5578 let message = err.to_string();
5579 let error = match err {
5580 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5581 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5582 code: "swap_refused".to_string(),
5583 message,
5584 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5585 },
5586 SuperviseError::SwapFailed {
5587 arm,
5588 candidate_exit,
5589 ..
5590 } => ErrorBody {
5591 code: "swap_failed".to_string(),
5592 message,
5593 detail: Some(serde_json::json!({
5594 "arm": arm.as_str(),
5595 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5596 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5597 })),
5598 },
5599 _ => ErrorBody::new(
5600 "target_unavailable",
5601 format!("failed to swap module_id '{module_id}': {message}"),
5602 ),
5603 };
5604 return Ok(vec![control_error_body_frame(&frame, error)?]);
5605 }
5606 self.route_outages
5610 .operator_action_ended_unrefused(&module_id);
5611
5612 let response = ClientControlResponse::SupervisorAck {
5613 module_id,
5614 applied: true,
5615 };
5616 Ok(vec![control_response_body_frame(
5617 &frame,
5618 &response,
5619 "ClientControlResponse::SupervisorAck",
5620 )?])
5621 }
5622
5623 async fn handle_supervisor_reload(
5624 &self,
5625 frame: Frame,
5626 module_id: String,
5627 requester: Principal,
5628 ) -> Result<Vec<Frame>, RouterError> {
5629 let operation_lock = self.supervisor.operation_lock();
5630 let _operation_guard = operation_lock.lock().await;
5631 let Some(module) = self.supervisor.get(&module_id) else {
5632 return Ok(vec![control_error_frame(
5633 &frame,
5634 "unknown_module",
5635 format!("module_id '{module_id}' is not supervised"),
5636 )?]);
5637 };
5638
5639 self.route_outages.mark_operator_action(&module_id);
5640 if let Err(err) = module.reload_requested(Some(requester)).await {
5641 self.route_outages
5642 .operator_action_ended_unrefused(&module_id);
5643 let (code, message) = match err {
5644 crate::supervise::SuperviseError::Disabled { .. } => {
5645 ("module_disabled", err.to_string())
5646 }
5647 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5648 ("swap_in_progress", err.to_string())
5649 }
5650 _ => (
5651 "reload_failed",
5652 format!("failed to reload module_id '{module_id}': {err}"),
5653 ),
5654 };
5655 return Ok(vec![control_error_frame(&frame, code, message)?]);
5656 }
5657
5658 let response = ClientControlResponse::SupervisorAck {
5659 module_id,
5660 applied: true,
5661 };
5662 Ok(vec![control_response_body_frame(
5663 &frame,
5664 &response,
5665 "ClientControlResponse::SupervisorAck",
5666 )?])
5667 }
5668
5669 async fn handle_supervisor_rescan(
5673 &self,
5674 frame: Frame,
5675 mode: RescanMode,
5676 requester: Principal,
5677 ) -> Result<Vec<Frame>, RouterError> {
5678 let preview = mode.previews();
5679 let Some(context) = self.rescan.clone() else {
5680 return Ok(vec![control_error_frame(
5681 &frame,
5682 "rescan_unavailable",
5683 "the daemon was not started with a reloadable config path".to_string(),
5684 )?]);
5685 };
5686
5687 let operation_lock = self.supervisor.operation_lock();
5688 let _operation_guard = operation_lock.lock().await;
5689 let loaded = match self.load_rescan_config(&mode, &context) {
5693 Ok(config) => config,
5694 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5695 };
5696 let Some((config, activity_config)) = loaded else {
5704 return Ok(vec![control_error_frame(
5705 &frame,
5706 "invalid_daemon_config",
5707 format!(
5708 "daemon config not found at {}; refusing to rescan (an absent config would \
5709 retire every supervised module)",
5710 context.config_path.display()
5711 ),
5712 )?]);
5713 };
5714 let (
5715 configured_port,
5716 storage_config,
5717 admission_facts_carrier_module_id,
5718 admission_facts_targets,
5719 scope_authority_owners,
5720 privileged_operator_prompt,
5721 modules,
5722 reserved_capabilities,
5723 ) = (
5724 config.port,
5725 config.storage,
5726 config.admission_facts_carrier_module_id,
5727 config.admission_facts_targets,
5728 config.scope_authority_owners,
5729 config.privileged_operator_prompt,
5730 config.modules,
5731 config.reserved_capabilities,
5732 );
5733
5734 let mut restart_required = Vec::new();
5743 for section in RestartRequiredSection::ALL {
5744 let changed = match section {
5745 RestartRequiredSection::Port => configured_port != context.configured_port,
5746 RestartRequiredSection::Storage => storage_config != context.storage_config,
5747 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5748 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5749 }
5750 RestartRequiredSection::AdmissionFactsTargets => {
5751 admission_facts_targets != context.admission_facts_targets
5752 }
5753 RestartRequiredSection::ScopeAuthorityOwners => {
5754 scope_authority_owners != context.scope_authority_owners
5755 }
5756 RestartRequiredSection::PrivilegedOperatorPrompt => {
5757 privileged_operator_prompt != context.privileged_operator_prompt
5758 }
5759 };
5760 if changed {
5761 restart_required.push(section.label().to_string());
5762 }
5763 }
5764 if !restart_required.is_empty() {
5765 warn!(
5766 config_path = %context.config_path.display(),
5767 sections = %restart_required.join(", "),
5768 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5769 );
5770 }
5771
5772 for configured in &modules {
5773 if let Err(err) = validate_spec(&configured.module_spec()) {
5774 return Ok(vec![control_error_frame(
5775 &frame,
5776 "invalid_daemon_config",
5777 format!("supervisor rescan rejected daemon config: {err}"),
5778 )?]);
5779 }
5780 }
5781
5782 let configured_capabilities = modules
5783 .iter()
5784 .map(|module| (module.module_id.clone(), module.enabled))
5785 .collect::<Vec<_>>();
5786 let preview_capability_warnings = if preview {
5787 let (_, registrations) = self.runtime_capability_snapshot()?;
5788 let current_modules = self
5789 .supervisor
5790 .list()
5791 .into_iter()
5792 .map(|module| module.module_id().to_string())
5793 .collect::<BTreeSet<_>>();
5794 let resulting_modules = configured_capabilities.clone();
5795 let removed = current_modules
5796 .into_iter()
5797 .filter(|module_id| {
5798 !resulting_modules
5799 .iter()
5800 .any(|(configured_id, _)| configured_id == module_id)
5801 })
5802 .collect::<Vec<_>>();
5803 let disabled = match &mode {
5806 RescanMode::Candidate(_) => resulting_modules
5807 .iter()
5808 .filter(|(_, enabled)| !enabled)
5809 .map(|(module_id, _)| module_id.clone())
5810 .collect::<Vec<_>>(),
5811 _ => Vec::new(),
5812 };
5813 self.capability_evaluator.preview_warnings(
5814 resulting_modules,
5815 &removed,
5816 &disabled,
5817 ®istrations,
5818 )
5819 } else {
5820 Vec::new()
5821 };
5822 let reconciled = match &mode {
5823 RescanMode::Candidate(candidate) => {
5824 self.preview_candidate(&context, modules, candidate)
5825 }
5826 _ => {
5827 self.reconcile_supervised_modules(
5828 &context.supervisor,
5829 modules,
5830 preview,
5831 requester.clone(),
5832 )
5833 .await
5834 }
5835 };
5836 let result = match reconciled {
5837 Ok(result) => result,
5838 Err(message) => {
5839 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5840 }
5841 };
5842 if !preview {
5843 if let Some(ledger) = &self.activity {
5844 ledger.module_rescanned(&result.added, &result.removed, requester);
5845 }
5846 if let Err(code) = self.configure_activity(activity_config).await {
5847 warn!(
5848 code,
5849 "activity retention could not be applied during rescan"
5850 );
5851 }
5852 self.capability_evaluator
5853 .configure(configured_capabilities, reserved_capabilities);
5854 self.capability_evaluator.wake_deadline_loop();
5855 self.refresh_capability_requirements();
5856 }
5857 let mut result = result;
5858 result.restart_required = restart_required;
5859 result.capability_warnings = preview_capability_warnings;
5860 let response = ClientControlResponse::SupervisorRescan { result };
5861 Ok(vec![control_response_body_frame(
5862 &frame,
5863 &response,
5864 "ClientControlResponse::SupervisorRescan",
5865 )?])
5866 }
5867
5868 async fn handle_supervisor_release_reserved(
5869 &self,
5870 frame: Frame,
5871 module_id: String,
5872 ) -> Result<Vec<Frame>, RouterError> {
5873 let Some(context) = self.rescan.clone() else {
5874 return Ok(vec![control_error_frame(
5875 &frame,
5876 "release_unavailable",
5877 "reserved-id release requires a daemon started with a reloadable config path",
5878 )?]);
5879 };
5880 let operation_lock = self.supervisor.operation_lock();
5881 let _operation_guard = operation_lock.lock().await;
5882 let loaded = match crate::daemon_config::load(&context.config_path) {
5883 Ok(Some(config)) => config,
5884 Ok(None) => {
5885 return Ok(vec![control_error_frame(
5886 &frame,
5887 "invalid_daemon_config",
5888 format!(
5889 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5890 context.config_path.display()
5891 ),
5892 )?])
5893 }
5894 Err(err) => {
5895 return Ok(vec![control_error_frame(
5896 &frame,
5897 "invalid_daemon_config",
5898 format!("unable to verify reserved-id release against daemon config: {err}"),
5899 )?])
5900 }
5901 };
5902 if loaded
5903 .modules
5904 .iter()
5905 .any(|configured| configured.module_id == module_id)
5906 {
5907 return Ok(vec![control_error_frame(
5908 &frame,
5909 "reserved_module_configured",
5910 format!(
5911 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5912 ),
5913 )?]);
5914 }
5915 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5916 return Ok(vec![control_error_frame(
5917 &frame,
5918 "reserved_gate_not_retained",
5919 format!(
5920 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5921 ),
5922 )?]);
5923 }
5924
5925 let response = ClientControlResponse::SupervisorAck {
5926 module_id,
5927 applied: true,
5928 };
5929 Ok(vec![control_response_body_frame(
5930 &frame,
5931 &response,
5932 "ClientControlResponse::SupervisorAck",
5933 )?])
5934 }
5935
5936 fn plan_rescan(
5940 &self,
5941 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5942 ) -> Result<RescanPlan, String> {
5943 let mut current = BTreeMap::new();
5944 for module in self.supervisor.list() {
5945 let (spec, health) = module.configuration().map_err(|err| {
5946 format!(
5947 "failed to read configuration for module_id '{}': {err}",
5948 module.module_id()
5949 )
5950 })?;
5951 let enabled = module
5952 .status()
5953 .map_err(|err| {
5954 format!(
5955 "failed to read status for module_id '{}': {err}",
5956 module.module_id()
5957 )
5958 })?
5959 .enabled;
5960 current.insert(
5961 module.module_id().to_string(),
5962 (module, spec, health, enabled),
5963 );
5964 }
5965 let configured = configured_modules
5966 .into_iter()
5967 .map(|module| (module.module_id.clone(), module))
5968 .collect::<BTreeMap<_, _>>();
5969
5970 let added = configured
5971 .keys()
5972 .filter(|module_id| !current.contains_key(*module_id))
5973 .cloned()
5974 .collect::<Vec<_>>();
5975 let removed = current
5976 .keys()
5977 .filter(|module_id| !configured.contains_key(*module_id))
5978 .cloned()
5979 .collect::<Vec<_>>();
5980 let mut changed_pending_reload = Vec::new();
5981 let mut configuration_changes = BTreeSet::new();
5982 let mut enabled_changes = BTreeSet::new();
5983 let mut unchanged = 0_u32;
5984
5985 for (module_id, configured_module) in &configured {
5986 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5987 else {
5988 continue;
5989 };
5990 let launch_changed = *current_spec != configured_module.module_spec();
5994 let configuration_changed =
5995 launch_changed || *current_health != configured_module.health;
5996 let enabled_changed = *current_enabled != configured_module.enabled;
5997 if configuration_changed {
5998 configuration_changes.insert(module_id.clone());
5999 }
6000 if launch_changed {
6001 changed_pending_reload.push(module_id.clone());
6002 }
6003 if enabled_changed {
6004 enabled_changes.insert(module_id.clone());
6005 }
6006 if !configuration_changed && !enabled_changed {
6007 unchanged = unchanged.saturating_add(1);
6008 }
6009 }
6010 Ok(RescanPlan {
6011 current,
6012 configured,
6013 added,
6014 removed,
6015 changed_pending_reload,
6016 configuration_changes,
6017 enabled_changes,
6018 unchanged,
6019 })
6020 }
6021
6022 async fn reconcile_supervised_modules(
6031 &self,
6032 supervisor: &Supervisor,
6033 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6034 preview: bool,
6035 requester: Principal,
6036 ) -> Result<SupervisorRescanResult, String> {
6037 let plan = self.plan_rescan(configured_modules)?;
6038
6039 if preview {
6045 return Ok(plan.preview_result());
6046 }
6047 let RescanPlan {
6048 current,
6049 configured,
6050 added,
6051 removed,
6052 changed_pending_reload,
6053 configuration_changes,
6054 enabled_changes,
6055 unchanged,
6056 } = plan;
6057 if !(added.is_empty()
6062 && removed.is_empty()
6063 && configuration_changes.is_empty()
6064 && enabled_changes.is_empty())
6065 {
6066 if let Some(context) = &self.rescan {
6067 context
6068 .generation
6069 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6070 }
6071 }
6072
6073 for module_id in &removed {
6074 let module = ¤t
6075 .get(module_id)
6076 .expect("removed module came from current supervisor state")
6077 .0;
6078 module
6079 .retire_requested(Some(requester.clone()))
6080 .await
6081 .map_err(|err| {
6082 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6083 })?;
6084 self.supervisor.record_rescan_removal(module_id);
6118 self.supervisor.retire(module_id);
6119 self.route_outages.forget(module_id);
6120 }
6121
6122 for module_id in configured.keys() {
6123 let Some((module, _, _, _)) = current.get(module_id) else {
6124 continue;
6125 };
6126 let configured_module = configured
6127 .get(module_id)
6128 .expect("configured module id came from configured map");
6129 if configuration_changes.contains(module_id) {
6130 module
6131 .update_configuration(
6132 configured_module.module_spec(),
6133 configured_module.health.clone(),
6134 configured_module.drain_timeout_ms,
6135 )
6136 .await
6137 .map_err(|err| {
6138 format!(
6139 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6140 )
6141 })?;
6142 }
6143 if enabled_changes.contains(module_id) {
6144 self.route_outages.mark_operator_action(module_id);
6147 module
6148 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6149 .await
6150 .map_err(|err| {
6151 self.route_outages.operator_action_ended_unrefused(module_id);
6152 format!(
6153 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6154 configured_module.enabled
6155 )
6156 })?;
6157 }
6158 }
6159
6160 for module_id in &added {
6161 let configured_module = configured
6162 .get(module_id)
6163 .expect("added module id came from configured map");
6164 supervisor
6165 .supervise_configured_requested(
6166 configured_module.module_spec(),
6167 configured_module.enabled,
6168 configured_module.health.clone(),
6169 configured_module.drain_timeout_ms,
6170 configured_module.restart,
6171 Some(requester.clone()),
6172 )
6173 .map_err(|err| {
6174 format!("failed to add module_id '{module_id}' during rescan: {err}")
6175 })?;
6176 }
6177
6178 Ok(SupervisorRescanResult {
6179 added,
6180 removed,
6181 changed_pending_reload,
6182 enabled_changes: enabled_changes.iter().cloned().collect(),
6183 unchanged,
6184 preview: false,
6185 restart_required: Vec::new(),
6188 capability_warnings: Vec::new(),
6189 module_actions: Vec::new(),
6190 earlier_pending: Vec::new(),
6191 expect: None,
6192 })
6193 }
6194
6195 async fn handle_supervisor_set_enabled(
6196 &self,
6197 frame: Frame,
6198 module_id: String,
6199 enabled: bool,
6200 requester: Principal,
6201 ) -> Result<Vec<Frame>, RouterError> {
6202 let operation_lock = self.supervisor.operation_lock();
6203 let _operation_guard = operation_lock.lock().await;
6204 let Some(module) = self.supervisor.get(&module_id) else {
6205 return Ok(vec![control_error_frame(
6206 &frame,
6207 "unknown_module",
6208 format!("module_id '{module_id}' is not supervised"),
6209 )?]);
6210 };
6211
6212 self.route_outages.mark_operator_action(&module_id);
6215 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6216 Ok(applied) => applied,
6217 Err(err) => {
6218 self.route_outages
6219 .operator_action_ended_unrefused(&module_id);
6220 return Ok(vec![control_error_frame(
6221 &frame,
6222 "target_unavailable",
6223 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6224 )?]);
6225 }
6226 };
6227 if !applied {
6228 self.route_outages
6231 .operator_action_ended_unrefused(&module_id);
6232 }
6233
6234 self.capability_evaluator.wake_deadline_loop();
6235 self.refresh_capability_requirements();
6236 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6237 Ok(vec![control_response_body_frame(
6238 &frame,
6239 &response,
6240 "ClientControlResponse::SupervisorAck",
6241 )?])
6242 }
6243
6244 async fn handle_supervisor_health_probe(
6245 &self,
6246 frame: Frame,
6247 module_id: String,
6248 ) -> Result<Vec<Frame>, RouterError> {
6249 self.refresh_capability_requirements();
6250 let Some(registration) = self
6251 .registry
6252 .get_module(&module_id)
6253 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6254 else {
6255 return Ok(vec![control_error_frame(
6256 &frame,
6257 "unknown_module",
6258 format!("module_id '{module_id}' is not registered"),
6259 )?]);
6260 };
6261
6262 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6275 {
6276 return Ok(vec![control_error_frame(
6277 &frame,
6278 "health_not_advertised",
6279 format!("module_id '{module_id}' did not advertise health.check"),
6280 )?]);
6281 }
6282
6283 let deadline = Instant::now() + self.health_probe_timeout;
6284 let pending = match self.forwarding.begin_module_control_rpc_for(
6285 &module_id,
6286 MODULE_CONTROL_OP_HEALTH_CHECK,
6287 deadline,
6288 ) {
6289 Ok(pending) => pending,
6290 Err(err) => {
6291 return Ok(vec![control_error_frame(
6292 &frame,
6293 forwarding_error_code(&err),
6294 err.to_string(),
6295 )?])
6296 }
6297 };
6298
6299 let PendingModuleControlRpc {
6300 endpoint,
6301 module_sink,
6302 negotiated_ver,
6303 corr: probe_corr,
6304 receiver,
6305 } = pending;
6306 let mut guard =
6307 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6308 let probe_body =
6309 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6310 RouterError::backend(
6311 0,
6312 frame.header.corr,
6313 format!("failed to encode health.check request: {err}"),
6314 )
6315 })?;
6316 let probe_frame = Frame::build_with_version(
6317 negotiated_ver,
6318 FrameType::Request,
6319 control_flags(),
6320 0,
6321 0,
6322 probe_corr,
6323 probe_body,
6324 )
6325 .map_err(RouterError::FrameBuild)?;
6326
6327 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6330 Ok(Ok(())) => {}
6331 Ok(Err(err)) => {
6332 return Ok(vec![control_error_frame(
6333 &frame,
6334 "target_unavailable",
6335 err.to_string(),
6336 )?]);
6337 }
6338 Err(_) => {
6339 return Ok(vec![control_error_frame(
6340 &frame,
6341 "module_timeout",
6342 format!(
6343 "module_id '{module_id}' did not answer health.check within {:?}",
6344 self.health_probe_timeout
6345 ),
6346 )?]);
6347 }
6348 }
6349
6350 match timeout_at(deadline, receiver).await {
6351 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6352 guard.disarm();
6353 let Some(report) = response.health_report() else {
6354 return Ok(vec![control_error_frame(
6355 &frame,
6356 "invalid_control_body",
6357 "health.check RPC returned a non-health response",
6358 )?]);
6359 };
6360 let HealthReport {
6365 status,
6366 detail,
6367 metrics,
6368 } = report;
6369 let capability_detail = self
6370 .capability_evaluator
6371 .required_problem_detail(&module_id);
6372 let response = ClientControlResponse::SupervisorHealthProbe {
6373 module_id,
6374 status,
6375 detail: append_capability_problem_detail(detail, capability_detail),
6376 metrics,
6377 };
6378 Ok(vec![control_response_body_frame(
6379 &frame,
6380 &response,
6381 "ClientControlResponse::SupervisorHealthProbe",
6382 )?])
6383 }
6384 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6385 guard.disarm();
6386 Ok(vec![control_error_body_frame(&frame, body)?])
6387 }
6388 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6389 guard.disarm();
6390 Ok(vec![control_error_frame(
6391 &frame,
6392 "target_unavailable",
6393 message,
6394 )?])
6395 }
6396 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6397 guard.disarm();
6398 Ok(vec![control_error_frame(
6399 &frame,
6400 "invalid_control_body",
6401 message,
6402 )?])
6403 }
6404 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6405 guard.disarm();
6406 Ok(vec![control_error_frame(
6407 &frame,
6408 "invalid_control_body",
6409 format!("expected module-control op '{expected}', got '{actual}'"),
6410 )?])
6411 }
6412 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6413 guard.disarm();
6414 Ok(vec![control_error_frame(
6415 &frame,
6416 "module_timeout",
6417 format!(
6418 "module_id '{module_id}' answered health.check after {:?}",
6419 self.health_probe_timeout
6420 ),
6421 )?])
6422 }
6423 Ok(Err(_)) => Ok(vec![control_error_frame(
6424 &frame,
6425 "target_unavailable",
6426 "health.check waiter was canceled before the module responded",
6427 )?]),
6428 Err(_) => Ok(vec![control_error_frame(
6429 &frame,
6430 "module_timeout",
6431 format!(
6432 "module_id '{module_id}' did not answer health.check within {:?}",
6433 self.health_probe_timeout
6434 ),
6435 )?]),
6436 }
6437 }
6438
6439 fn supervisor_status(
6440 &self,
6441 module_id: &str,
6442 corr: u64,
6443 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6444 self.supervisor
6445 .get(module_id)
6446 .map(|module| {
6447 let warming = module.is_warming_for_control("status").map_err(|err| {
6448 RouterError::backend(
6449 0,
6450 corr,
6451 format!(
6452 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6453 ),
6454 )
6455 })?;
6456 module.status_for_control("status").map_err(|err| {
6457 RouterError::backend(
6458 0,
6459 corr,
6460 format!(
6461 "failed to read supervisor status for module_id '{module_id}': {err}"
6462 ),
6463 )
6464 }).map(|status| (status, warming))
6465 })
6466 .transpose()
6467 }
6468
6469 fn guard_module_control_op(
6470 &self,
6471 frame: &Frame,
6472 module_id: &str,
6473 op: &str,
6474 ) -> Result<Option<Frame>, RouterError> {
6475 if self.module_grants_op(module_id, op, frame.header.corr)? {
6476 return Ok(None);
6477 }
6478
6479 Ok(Some(control_error_frame(
6480 frame,
6481 "op_not_allowed",
6482 format!("module_id '{module_id}' did not grant control op '{op}'"),
6483 )?))
6484 }
6485
6486 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6487 let Some(registration) = self
6488 .registry
6489 .get_module(module_id)
6490 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6491 else {
6492 return Ok(false);
6493 };
6494 Ok(module_registration_grants_op(®istration.control_ops, op))
6495 }
6496
6497 fn handle_status_update(
6498 &self,
6499 endpoint: ModuleEndpointId,
6500 frame: Frame,
6501 ) -> Result<Vec<Frame>, RouterError> {
6502 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6503 Ok(update) => update,
6504 Err(err) => {
6505 if is_known_module_push_op(&frame.body) {
6510 return Ok(vec![control_error_frame(
6511 &frame,
6512 "invalid_control_body",
6513 format!("malformed module control push body: {err}"),
6514 )?]);
6515 }
6516 return Ok(Vec::new());
6517 }
6518 };
6519
6520 match update {
6521 ModuleControlPush::RouteStatus {
6522 route_channel,
6523 route_epoch,
6524 status,
6525 } => {
6526 self.forwarding
6527 .cache_status(endpoint, route_channel, route_epoch, status)
6528 .map_err(RouterError::Forwarding)?;
6529 }
6530 }
6531 Ok(Vec::new())
6532 }
6533
6534 fn handle_route_poll(
6535 &self,
6536 ctx: &RouteCtx,
6537 frame: Frame,
6538 route_channel: u16,
6539 route_epoch: u32,
6540 kind: PollKind,
6541 ) -> Result<Vec<Frame>, RouterError> {
6542 let snapshot = self
6543 .forwarding
6544 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6545 .map_err(RouterError::Forwarding)?;
6546 let response = match (kind, snapshot) {
6547 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6548 ClientControlResponse::RoutePoll {
6549 route_channel,
6550 route_epoch,
6551 status,
6552 live: None,
6553 }
6554 }
6555 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6556 route_channel,
6557 route_epoch,
6558 status: None,
6559 live: None,
6560 },
6561 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6562 let live = self
6588 .process_liveness
6589 .as_ref()
6590 .and_then(|source| source.process_live(&module_id))
6591 .unwrap_or(true);
6592 ClientControlResponse::RoutePoll {
6593 route_channel,
6594 route_epoch,
6595 status: None,
6596 live: Some(live),
6597 }
6598 }
6599 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6600 route_channel,
6601 route_epoch,
6602 status: None,
6603 live: Some(false),
6604 },
6605 };
6606
6607 Ok(vec![control_response_body_frame(
6608 &frame,
6609 &response,
6610 "ClientControlResponse::RoutePoll",
6611 )?])
6612 }
6613
6614 pub(crate) fn observe_module_control_completion(
6615 &self,
6616 completion: ModuleControlRpcCompletion,
6617 ) -> bool {
6618 match completion {
6619 ModuleControlRpcCompletion::Unknown => false,
6620 ModuleControlRpcCompletion::Settled => true,
6621 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6622 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6623 info!(
6624 module_id = %module_id,
6625 latency_ms,
6626 "late health.check answer proves the module is alive"
6627 );
6628 match self
6629 .supervisor
6630 .record_late_health_answer(&module_id, latency_ms)
6631 {
6632 Ok(true) => {}
6633 Ok(false) => debug!(
6634 module_id = %module_id,
6635 latency_ms,
6636 "late health.check answer has no active supervisor snapshot"
6637 ),
6638 Err(err) => warn!(
6639 module_id = %module_id,
6640 latency_ms,
6641 error = %err,
6642 "failed to record late health.check answer"
6643 ),
6644 }
6645 true
6646 }
6647 }
6648 }
6649
6650 fn refuse_to_end_module_connection_for_a_client(
6671 &self,
6672 module_connection_id: ConnectionId,
6673 corr: u64,
6674 err: ForwardingError,
6675 ) -> Result<(), RouterError> {
6676 if let ForwardingError::ConnectionClosing { connection_id } = err {
6677 if connection_id != module_connection_id {
6678 warn!(
6679 module_connection_id = module_connection_id.get(),
6680 client_connection_id = connection_id.get(),
6681 corr,
6682 "dropping a route.bind response for a closing client; the module connection keeps serving"
6683 );
6684 return Ok(());
6685 }
6686 }
6687 Err(RouterError::Forwarding(err))
6688 }
6689
6690 fn handle_module_relay_response(
6691 &self,
6692 connection_id: ConnectionId,
6693 frame: Frame,
6694 ) -> Result<Vec<Frame>, RouterError> {
6695 let mut secondary_error = None;
6696 let outcome = match frame.header.ty {
6697 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6698 Ok(probe) if probe.op == "route.bind" => {
6699 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6700 Ok(ModuleControlResponse::RouteBindAck {}) => {
6701 RouteBindRelayOutcome::Accepted
6702 }
6703 Ok(other) => {
6704 let message =
6705 format!("route.bind response carried unexpected body: {other:?}");
6706 secondary_error = Some(control_error_frame(
6707 &frame,
6708 "invalid_control_body",
6709 message.clone(),
6710 )?);
6711 RouteBindRelayOutcome::ModuleGone(message)
6712 }
6713 Err(err) => {
6714 let message = format!("malformed route.bind response body: {err}");
6715 secondary_error = Some(control_error_frame(
6716 &frame,
6717 "invalid_control_body",
6718 message.clone(),
6719 )?);
6720 RouteBindRelayOutcome::ModuleGone(message)
6721 }
6722 }
6723 }
6724 Ok(probe) => {
6725 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6726 {
6727 Ok(response) => ModuleControlRpcOutcome::Response(response),
6728 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6729 "malformed {} response body: {err}",
6730 probe.op
6731 )),
6732 };
6733 let completion = self
6734 .forwarding
6735 .complete_module_control_rpc(
6736 connection_id,
6737 frame.header.corr,
6738 Some(&probe.op),
6739 outcome,
6740 )
6741 .map_err(RouterError::Forwarding)?;
6742 if !self.observe_module_control_completion(completion) {
6743 debug!(
6744 connection_id = connection_id.get(),
6745 corr = frame.header.corr,
6746 op = %probe.op,
6747 "dropping late or unknown module-control RPC response"
6748 );
6749 }
6750 return Ok(Vec::new());
6751 }
6752 Err(err) => {
6753 if let Some(expected_op) = self
6754 .forwarding
6755 .pending_module_control_op(connection_id, frame.header.corr)
6756 .map_err(RouterError::Forwarding)?
6757 {
6758 let completion = self
6759 .forwarding
6760 .complete_module_control_rpc(
6761 connection_id,
6762 frame.header.corr,
6763 None,
6764 ModuleControlRpcOutcome::MalformedResponse(format!(
6765 "malformed {expected_op} response body: {err}"
6766 )),
6767 )
6768 .map_err(RouterError::Forwarding)?;
6769 if !self.observe_module_control_completion(completion) {
6770 debug!(
6771 connection_id = connection_id.get(),
6772 corr = frame.header.corr,
6773 "dropping late malformed module-control RPC response"
6774 );
6775 }
6776 return Ok(Vec::new());
6777 }
6778 let message = format!("malformed route.bind response body: {err}");
6779 secondary_error = Some(control_error_frame(
6780 &frame,
6781 "invalid_control_body",
6782 message.clone(),
6783 )?);
6784 RouteBindRelayOutcome::ModuleGone(message)
6785 }
6786 },
6787 FrameType::Error => {
6788 if self
6789 .forwarding
6790 .pending_module_control_op(connection_id, frame.header.corr)
6791 .map_err(RouterError::Forwarding)?
6792 .is_some()
6793 {
6794 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6795 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6796 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6797 "malformed module-control ERROR body: {err}"
6798 )),
6799 };
6800 let completion = self
6801 .forwarding
6802 .complete_module_control_rpc(
6803 connection_id,
6804 frame.header.corr,
6805 None,
6806 outcome,
6807 )
6808 .map_err(RouterError::Forwarding)?;
6809 if !self.observe_module_control_completion(completion) {
6810 debug!(
6811 connection_id = connection_id.get(),
6812 corr = frame.header.corr,
6813 "dropping late or unknown module-control RPC error"
6814 );
6815 }
6816 return Ok(Vec::new());
6817 }
6818 match serde_json::from_slice::<ErrorBody>(&frame.body) {
6819 Ok(body) => RouteBindRelayOutcome::Rejected(body),
6820 Err(err) => {
6821 let message = format!("malformed route.bind ERROR body: {err}");
6822 secondary_error = Some(control_error_frame(
6823 &frame,
6824 "invalid_control_body",
6825 message.clone(),
6826 )?);
6827 RouteBindRelayOutcome::ModuleGone(message)
6828 }
6829 }
6830 }
6831 ty => {
6832 return Ok(vec![control_error_frame(
6833 &frame,
6834 "unsupported_control_frame",
6835 format!("unsupported module channel-0 frame {ty:?}"),
6836 )?])
6837 }
6838 };
6839
6840 let settled =
6841 self.forwarding
6842 .complete_pending_relay(connection_id, frame.header.corr, outcome);
6843 let completion = match settled {
6844 Ok(completion) => completion,
6845 Err(err) => {
6846 self.refuse_to_end_module_connection_for_a_client(
6847 connection_id,
6848 frame.header.corr,
6849 err,
6850 )?;
6851 return Ok(secondary_error.into_iter().collect());
6852 }
6853 };
6854 if let Some(target) = completion.abandoned.as_ref() {
6855 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6856 }
6857 if !completion.settled {
6858 debug!(
6859 connection_id = connection_id.get(),
6860 corr = frame.header.corr,
6861 frame_type = ?frame.header.ty,
6862 "dropping late or unknown route.bind relay response"
6863 );
6864 }
6865 Ok(secondary_error.into_iter().collect())
6866 }
6867
6868 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6869 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6870 self.cleanup_connection_with_end_reason(
6874 connection_id,
6875 RegistrationEndReason::ExplicitGoodbye,
6876 )
6877 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6878 Ok(Vec::new())
6879 }
6880}
6881
6882impl Default for ControlHandler {
6883 fn default() -> Self {
6884 Self::new(Arc::new(Registry::default()))
6885 }
6886}
6887
6888impl crate::supervise::SwapPromotionObserver for ControlHandler {
6889 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6890 self.apply_registration_capabilities(registration);
6891 }
6892}
6893
6894fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6895 CapabilityRequirementStatus {
6896 consumer: status.consumer,
6897 capability: status.capability,
6898 need: match status.need {
6899 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6900 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6901 },
6902 verdict: status.verdict.as_str().to_string(),
6903 episode_seq: status.episode_seq,
6904 config_satisfiable: status.config_satisfiable,
6905 runtime_available: status.runtime_available,
6906 detail: status.detail,
6907 }
6908}
6909
6910fn append_capability_problem_detail(
6911 detail: Option<String>,
6912 capability_detail: Option<String>,
6913) -> Option<String> {
6914 match (detail, capability_detail) {
6915 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6916 (Some(detail), None) => Some(detail),
6917 (None, Some(capability_detail)) => Some(capability_detail),
6918 (None, None) => None,
6919 }
6920}
6921
6922fn subc_ops() -> Vec<String> {
6923 SUBC_CONTROL_OPS
6924 .iter()
6925 .map(|op| (*op).to_string())
6926 .collect()
6927}
6928
6929fn module_subc_ops() -> Vec<String> {
6930 SUBC_CONTROL_OPS
6931 .iter()
6932 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6933 .map(|op| (*op).to_string())
6934 .collect()
6935}
6936
6937#[cfg(test)]
6938fn module_baseline_control_ops() -> Vec<String> {
6939 MODULE_BASELINE_CONTROL_OPS
6940 .iter()
6941 .map(|op| (*op).to_string())
6942 .collect()
6943}
6944
6945fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6946 let mut seen = HashSet::new();
6947 let mut effective = Vec::new();
6948 for op in MODULE_BASELINE_CONTROL_OPS {
6949 if seen.insert((*op).to_string()) {
6950 effective.push((*op).to_string());
6951 }
6952 }
6953 for op in declared.unwrap_or_default() {
6954 if seen.insert(op.clone()) {
6955 effective.push(op);
6956 }
6957 }
6958 effective
6959}
6960
6961fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6962 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6963}
6964
6965fn target_module_id(target: &RouteTarget) -> &str {
6966 match target {
6967 RouteTarget::ToolProvider { module_id }
6968 | RouteTarget::ManagementSurface { module_id }
6969 | RouteTarget::InternalService { module_id, .. } => module_id,
6970 }
6971}
6972
6973fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6974 roles.iter().any(|role| match (target, role) {
6975 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6976 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6977 (
6978 RouteTarget::InternalService { service_id, .. },
6979 ProviderRole::InternalService {
6980 service_id: provided,
6981 ..
6982 },
6983 ) => service_id == provided,
6984 _ => false,
6985 })
6986}
6987
6988fn is_routable_role(role: &ProviderRole) -> bool {
6989 matches!(
6990 role,
6991 ProviderRole::ToolProvider { .. }
6992 | ProviderRole::ManagementSurface { .. }
6993 | ProviderRole::InternalService { .. }
6994 )
6995}
6996
6997#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6998enum ControlRequestBodyError {
6999 UnknownOp,
7000 InvalidBody,
7001}
7002
7003#[derive(Debug, Deserialize)]
7004struct ControlOpProbe {
7005 op: String,
7006}
7007
7008const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7011
7012fn is_known_module_push_op(body: &[u8]) -> bool {
7013 serde_json::from_slice::<ControlOpProbe>(body)
7014 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7015 .unwrap_or(false)
7016}
7017
7018fn is_known_module_request_op(body: &[u8]) -> bool {
7019 serde_json::from_slice::<ControlOpProbe>(body)
7020 .map(|probe| is_module_to_subc_op(&probe.op))
7021 .unwrap_or(false)
7022}
7023
7024fn is_module_to_subc_op(op: &str) -> bool {
7025 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7026}
7027
7028fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7029 debug!(
7030 op = %op,
7031 connection_id = connection_id.get(),
7032 corr,
7033 "control dispatch"
7034 );
7035}
7036
7037fn log_slow_control_dispatch(
7038 dispatch_started_at: Option<StdInstant>,
7039 op: &'static str,
7040 connection_id: ConnectionId,
7041 corr: u64,
7042 route_open_timings: Option<&RouteOpenTimings>,
7043) {
7044 let Some(dispatch_started_at) = dispatch_started_at else {
7045 return;
7046 };
7047 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7049 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7050 if let Some(timings) = route_open_timings {
7051 warn!(
7052 op = %op,
7053 connection_id = connection_id.get(),
7054 corr,
7055 elapsed_ms = elapsed.as_millis() as u64,
7056 module_id = ?timings.module_id,
7057 principal = %timings.principal,
7058 admission_ms = timings.admission.as_millis() as u64,
7059 project_root_ms = timings.project_root.as_millis() as u64,
7060 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7061 bind_send_ms = timings.bind_send.as_millis() as u64,
7062 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7063 "slow control dispatch"
7064 );
7065 return;
7066 }
7067 warn!(
7068 op = %op,
7069 connection_id = connection_id.get(),
7070 corr,
7071 elapsed_ms = elapsed.as_millis() as u64,
7072 "slow control dispatch"
7073 );
7074 }
7075}
7076
7077#[cfg(test)]
7079type TestCallerLookup =
7080 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7081
7082fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7084 match request {
7085 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7086 Some(PrivilegedOperation::Restart { module_id })
7087 }
7088 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7089 Some(PrivilegedOperation::Swap { module_id })
7090 }
7091 ClientControlRequest::SupervisorReload { module_id } => {
7092 Some(PrivilegedOperation::Reload { module_id })
7093 }
7094 ClientControlRequest::SupervisorRescan { preview } => {
7095 if *preview {
7096 None
7097 } else {
7098 Some(PrivilegedOperation::Rescan)
7099 }
7100 }
7101 ClientControlRequest::SupervisorRescanChecked {
7106 preview,
7107 candidate,
7108 expect,
7109 } => (!*preview && candidate.is_none() && expect.is_some())
7110 .then_some(PrivilegedOperation::Rescan),
7111 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7112 Some(PrivilegedOperation::SetEnabled {
7113 module_id,
7114 enabled: *enabled,
7115 })
7116 }
7117 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7118 Some(PrivilegedOperation::ReleaseReserved { module_id })
7119 }
7120 ClientControlRequest::ServerDescribe {}
7121 | ClientControlRequest::CatalogList { .. }
7122 | ClientControlRequest::RouteOpen { .. }
7123 | ClientControlRequest::RoutePoll { .. }
7124 | ClientControlRequest::SupervisorList {}
7125 | ClientControlRequest::SupervisorHealth {}
7126 | ClientControlRequest::SupervisorHealthProbe { .. }
7127 | ClientControlRequest::SupervisorRoutes { .. }
7128 | ClientControlRequest::SupervisorProvenance { .. }
7129 | ClientControlRequest::SupervisorStderrTail { .. }
7130 | ClientControlRequest::SupervisorTerminals { .. }
7131 | ClientControlRequest::SupervisorSpawnSnapshot {}
7132 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7133 }
7134}
7135
7136fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7137 match request {
7138 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7139 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7140 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7141 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7142 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7143 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7144 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7145 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7146 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7147 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7148 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7149 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7150 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7151 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7152 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7153 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7154 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7155 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7156 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7157 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7158 }
7159}
7160
7161fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7167 use subc_protocol::scope::ScopeKind;
7168 const MAX_REF_CHARS: usize = 64;
7169 let kind = match stamp.kind {
7170 ScopeKind::Head => "head",
7171 ScopeKind::Worker => "worker",
7172 ScopeKind::Ephemeral => "ephemeral",
7173 };
7174 let mut scope_ref: String = stamp.scope_ref.chars().take(MAX_REF_CHARS).collect();
7175 if stamp.scope_ref.chars().nth(MAX_REF_CHARS).is_some() {
7176 scope_ref.push('…');
7177 }
7178 format!(
7179 "{kind}/{}/{scope_ref:?}@{}",
7180 principal_label(&stamp.owner),
7181 stamp.scope_epoch
7182 )
7183}
7184
7185fn principal_label(principal: &Principal) -> String {
7186 match principal {
7187 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7188 Principal::Direct => "direct".to_string(),
7189 other => format!("{other:?}"),
7190 }
7191}
7192
7193fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7194 match request {
7195 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7196 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7197 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7198 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7199 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7200 }
7201}
7202
7203fn parse_client_control_request(
7204 body: &[u8],
7205) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7206 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7207 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7208 Ok(probe)
7209 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7210 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7211 {
7212 ControlRequestBodyError::InvalidBody
7213 }
7214 Ok(_) => ControlRequestBodyError::UnknownOp,
7215 Err(_) => ControlRequestBodyError::InvalidBody,
7216 };
7217 (err, classification)
7218 })
7219}
7220
7221fn parse_module_control_request_from_module(
7222 body: &[u8],
7223) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7224 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7225 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7226 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7227 Ok(_) => ControlRequestBodyError::UnknownOp,
7228 Err(_) => ControlRequestBodyError::InvalidBody,
7229 };
7230 (err, classification)
7231 })
7232}
7233
7234#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7235enum ProviderRoleKind {
7236 ToolProvider,
7237 PipelineStage,
7238 ManagementSurface,
7239 InternalService,
7240}
7241
7242fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7243 match role {
7244 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7245 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7246 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7247 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7248 }
7249}
7250
7251fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7252 roles.iter().map(provider_role_kind).collect()
7253}
7254
7255const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7258
7259#[derive(Debug, Default, PartialEq, Eq)]
7261struct ScopeOutcomeCounts {
7262 created: usize,
7263 replaced: usize,
7264 updated: usize,
7265 unchanged: usize,
7266 refused: usize,
7267}
7268
7269impl ScopeOutcomeCounts {
7270 fn of(results: &[ScopeRecordResult]) -> Self {
7271 let mut counts = Self::default();
7272 for result in results {
7273 let slot = match result.outcome {
7274 ScopeRecordOutcome::Created => &mut counts.created,
7275 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7276 ScopeRecordOutcome::Updated => &mut counts.updated,
7277 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7278 ScopeRecordOutcome::Refused => &mut counts.refused,
7279 };
7280 *slot += 1;
7281 }
7282 counts
7283 }
7284}
7285
7286#[cfg(test)]
7287mod scope_outcome_count_tests {
7288 use super::*;
7289
7290 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7291 ScopeRecordResult {
7292 scope_ref: "r".to_string(),
7293 scope_epoch: 1,
7294 outcome,
7295 code: None,
7296 message: None,
7297 version: None,
7298 parent_state: None,
7299 }
7300 }
7301
7302 #[test]
7305 fn every_outcome_is_counted_in_its_own_field() {
7306 let results = [
7307 result(ScopeRecordOutcome::Created),
7308 result(ScopeRecordOutcome::Created),
7309 result(ScopeRecordOutcome::Replaced),
7310 result(ScopeRecordOutcome::Updated),
7311 result(ScopeRecordOutcome::Unchanged),
7312 result(ScopeRecordOutcome::Refused),
7313 result(ScopeRecordOutcome::Refused),
7314 result(ScopeRecordOutcome::Refused),
7315 ];
7316 assert_eq!(
7317 ScopeOutcomeCounts::of(&results),
7318 ScopeOutcomeCounts {
7319 created: 2,
7320 replaced: 1,
7321 updated: 1,
7322 unchanged: 1,
7323 refused: 3,
7324 }
7325 );
7326 }
7327}
7328
7329fn capability_census_trigger(
7333 old: Option<&CapabilityDeclarations>,
7334 new: Option<&CapabilityDeclarations>,
7335) -> bool {
7336 let old_provides = old
7337 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7338 .unwrap_or_default();
7339 let old_denies = old
7340 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7341 .unwrap_or_default();
7342 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7343 provides: Vec::new(),
7344 requires: Vec::new(),
7345 must_never_reach: Vec::new(),
7346 });
7347
7348 new.provides
7349 .iter()
7350 .any(|capability| !old_provides.contains(capability))
7351 || new
7352 .must_never_reach
7353 .iter()
7354 .any(|capability| !old_denies.contains(capability))
7355}
7356
7357fn denied_capability<'a>(
7360 opening_manifest: &'a ModuleManifest,
7361 target_manifest: &ModuleManifest,
7362) -> Option<&'a str> {
7363 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7364 let target_capabilities = target_manifest.capabilities.as_ref()?;
7365 opening_capabilities
7366 .must_never_reach
7367 .iter()
7368 .find(|denied| {
7369 target_capabilities
7370 .provides
7371 .iter()
7372 .any(|provided| provided == *denied)
7373 })
7374 .map(String::as_str)
7375}
7376
7377fn catalog_update_frozen_field_message(
7378 registered: &ModuleManifest,
7379 provides: &[ProviderRole],
7380) -> Option<String> {
7381 let old_has_provides = !registered.provides.is_empty();
7382 let new_has_provides = !provides.is_empty();
7383 if old_has_provides != new_has_provides {
7384 return Some(format!(
7385 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7386 registered.module_id
7387 ));
7388 }
7389
7390 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7391 return Some(format!(
7392 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7393 registered.module_id
7394 ));
7395 }
7396
7397 let registered_concurrency = manifest_concurrency(registered);
7398 let mut candidate = registered.clone();
7399 candidate.provides = provides.to_vec();
7400 let candidate_concurrency = manifest_concurrency(&candidate);
7401 if candidate_concurrency != registered_concurrency {
7402 return Some(format!(
7403 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7404 registered.module_id, registered_concurrency, candidate_concurrency
7405 ));
7406 }
7407
7408 None
7411}
7412
7413fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7414 manifest.provides.iter().any(is_routable_role)
7415}
7416
7417fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7423 manifest
7424 .provides
7425 .iter()
7426 .find_map(|provider| match provider {
7427 ProviderRole::ToolProvider { concurrency, .. }
7428 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7429 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7430 })
7431 .unwrap_or(Concurrency::ModuleManaged)
7432}
7433
7434fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7441 let has_management_surface = manifest
7442 .provides
7443 .iter()
7444 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7445 if !has_management_surface {
7446 return false;
7447 }
7448 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7449 return false;
7450 };
7451 let Some(provides) = raw
7452 .get("manifest")
7453 .and_then(|manifest| manifest.get("provides"))
7454 .and_then(serde_json::Value::as_array)
7455 else {
7456 return false;
7457 };
7458 provides.iter().any(|role| {
7463 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7464 && role.get("concurrency").is_none()
7465 })
7466}
7467
7468fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7469 if peer_version != PROTOCOL_VERSION {
7470 return Err(format!(
7471 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7472 ));
7473 }
7474 Ok(PROTOCOL_VERSION)
7475}
7476
7477fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7478 Frame::build_with_version(
7479 response_version(frame),
7480 FrameType::Pong,
7481 frame.header.flags,
7482 0,
7483 0,
7484 frame.header.corr,
7485 Vec::new(),
7486 )
7487 .map_err(RouterError::FrameBuild)
7488}
7489
7490pub(crate) fn control_error_frame(
7491 frame: &Frame,
7492 code: &'static str,
7493 message: impl Into<String>,
7494) -> Result<Frame, RouterError> {
7495 control_error_body_frame(
7496 frame,
7497 ErrorBody {
7498 code: code.to_string(),
7499 message: message.into(),
7500 detail: None,
7501 },
7502 )
7503}
7504
7505fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7506 let body = serde_json::to_vec(&error).map_err(|err| {
7507 RouterError::backend(
7508 0,
7509 frame.header.corr,
7510 format!("failed to encode control ERROR: {err}"),
7511 )
7512 })?;
7513
7514 Frame::build_with_version(
7515 response_version(frame),
7516 FrameType::Error,
7517 control_flags(),
7518 0,
7519 0,
7520 frame.header.corr,
7521 body,
7522 )
7523 .map_err(RouterError::FrameBuild)
7524}
7525
7526fn control_response_body_frame<T: Serialize>(
7527 frame: &Frame,
7528 reply: &T,
7529 label: &'static str,
7530) -> Result<Frame, RouterError> {
7531 let body = serde_json::to_vec(reply).map_err(|err| {
7532 RouterError::backend(
7533 0,
7534 frame.header.corr,
7535 format!("failed to encode {label}: {err}"),
7536 )
7537 })?;
7538
7539 Frame::build_with_version(
7540 response_version(frame),
7541 FrameType::Response,
7542 control_flags(),
7543 0,
7544 0,
7545 frame.header.corr,
7546 body,
7547 )
7548 .map_err(RouterError::FrameBuild)
7549}
7550
7551fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7581 match err {
7582 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7583 ForwardingError::NoModuleConnection => "target_unavailable",
7584 ForwardingError::ModuleReloading { .. } => "module_reloading",
7585 ForwardingError::ClientRouteChannelExhausted { .. }
7586 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7587 ForwardingError::StaleModuleEndpoint
7588 | ForwardingError::UnknownReservation { .. }
7589 | ForwardingError::ConnectionClosing { .. }
7590 | ForwardingError::ClientEgressClosed { .. }
7591 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7592 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7595 ForwardingError::RelayCorrelationExhausted
7596 | ForwardingError::RouteOpenBuild(_)
7597 | ForwardingError::Poisoned => "forwarding_error",
7598 }
7599}
7600
7601fn response_version(frame: &Frame) -> u8 {
7602 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7603 frame.header.ver
7604 } else {
7605 PROTOCOL_VERSION
7606 }
7607}
7608
7609fn control_flags() -> Flags {
7610 Flags::new(false, Priority::Passive, false)
7611}
7612
7613fn send_goodbye_target_best_effort(
7618 counters: &DaemonCounters,
7619 target: &GoodbyeTarget,
7620 context: &'static str,
7621) {
7622 let Ok(frame) = Frame::build_with_version(
7623 target.negotiated_ver,
7624 FrameType::Goodbye,
7625 control_flags(),
7626 target.channel,
7627 target.epoch,
7628 0,
7629 Vec::new(),
7630 ) else {
7631 return;
7632 };
7633 crate::forwarding::send_module_route_goodbye(
7634 counters,
7635 &target.sink,
7636 frame,
7637 target.module_id.as_deref(),
7638 context,
7639 );
7640}
7641
7642pub(crate) fn send_route_control_pushes(
7643 forwarding: &ForwardingTable,
7644 routes: Vec<EndpointRoute>,
7645 push: ClientControlPush,
7646) {
7647 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7648 for route in routes {
7649 let target = route.goodbye_target;
7650 if let Some((existing, channels)) = targets
7651 .iter_mut()
7652 .find(|(existing, _)| existing.connection_id == target.connection_id)
7653 {
7654 debug_assert_eq!(
7655 existing.negotiated_ver, target.negotiated_ver,
7656 "one connection cannot negotiate multiple frame versions"
7657 );
7658 if !channels.contains(&target.channel) {
7659 channels.push(target.channel);
7660 }
7661 continue;
7662 }
7663 let channel = target.channel;
7664 targets.push((target, vec![channel]));
7665 }
7666 for (target, mut channels) in targets {
7667 channels.sort_unstable();
7668 let mut push = push.clone();
7669 match &mut push {
7670 ClientControlPush::RouteClosing {
7671 channels: covered, ..
7672 }
7673 | ClientControlPush::RouteClosed {
7674 channels: covered, ..
7675 } => *covered = channels,
7676 }
7677 let body = match serde_json::to_vec(&push) {
7678 Ok(body) => body,
7679 Err(err) => {
7680 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7681 continue;
7682 }
7683 };
7684 let frame = match Frame::build_with_version(
7685 target.negotiated_ver,
7686 FrameType::Push,
7687 control_flags(),
7688 0,
7689 0,
7690 0,
7691 body.clone(),
7692 ) {
7693 Ok(frame) => frame,
7694 Err(err) => {
7695 warn!(
7696 route_channel = target.channel,
7697 error = %err,
7698 "failed to build route lifecycle control PUSH frame"
7699 );
7700 continue;
7701 }
7702 };
7703 if let Err(err) = target.sink.try_send(frame) {
7704 if target.close_on_delivery_failure() {
7705 warn!(
7706 target_connection_id = target.connection_id.get(),
7707 route_channel = target.channel,
7708 error = %err,
7709 "route lifecycle control PUSH was not delivered to client; closing target connection"
7710 );
7711 let _ = forwarding.escalate_client_delivery_failure(
7712 target.connection_id,
7713 target.channel,
7714 target.epoch,
7715 CloseReason::new(
7716 "route_lifecycle_push_delivery_failed",
7717 format!(
7718 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7719 target.channel
7720 ),
7721 ),
7722 crate::forwarding::UndeliveredFrame {
7723 module_id: target.module_id.as_deref(),
7724 sink: &target.sink,
7725 },
7726 );
7727 }
7728 }
7729 }
7730}
7731
7732#[cfg(test)]
7733mod tests {
7734 mod activity_read {
7735 include!("activity_read_tests.rs");
7736 }
7737
7738 mod lifecycle_activity {
7739 include!("control_lifecycle_tests.rs");
7740 }
7741 #[cfg(unix)]
7742 #[tokio::test]
7743 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7744 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7745 let path = dir.join("subc.jsonc");
7746 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7747 "program":"/bin/sleep","args":["60"],"protocol":"none",
7748 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7749 }}}).to_string()).unwrap();
7750 let mut configured = crate::daemon_config::load(&path)
7751 .unwrap()
7752 .unwrap()
7753 .modules
7754 .pop()
7755 .unwrap();
7756 let registry = std::sync::Arc::new(crate::Registry::default());
7757 let handle = crate::SupervisorHandle::new();
7758 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7759 .with_handle(handle.clone());
7760 let module = supervisor
7761 .supervise_configured_with_health(
7762 configured.module_spec(),
7763 true,
7764 configured.health.clone(),
7765 None,
7766 configured.restart,
7767 )
7768 .unwrap();
7769 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7770 let before = module.status().unwrap().pid;
7771 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7772 configured.health.cadence = std::time::Duration::from_secs(3600);
7773 let health_only = handler
7774 .reconcile_supervised_modules(
7775 &supervisor,
7776 vec![configured.clone()],
7777 false,
7778 Principal::Direct,
7779 )
7780 .await
7781 .unwrap();
7782 assert!(
7783 health_only.changed_pending_reload.is_empty(),
7784 "health policy is already applied live"
7785 );
7786 assert_eq!(module.status().unwrap().pid, before);
7787 assert_eq!(
7788 module.configuration().unwrap().1.http,
7789 configured.health.http
7790 );
7791 configured.args = vec!["61".into()];
7792 let launch = handler
7793 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7794 .await
7795 .unwrap();
7796 assert_eq!(launch.changed_pending_reload, ["stock"]);
7797 assert_eq!(
7798 module.status().unwrap().pid,
7799 before,
7800 "a launch edit is stored until reload"
7801 );
7802 module.drain().await.unwrap();
7803 }
7804 use cortexkit_test_support::ScratchDir;
7805 use std::{
7806 collections::BTreeMap,
7807 fmt,
7808 path::PathBuf,
7809 sync::{Arc, Mutex},
7810 time::Duration,
7811 };
7812
7813 use serde_json::{json, Value};
7814 use subc_protocol::{
7815 manifest::{
7816 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7817 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7818 },
7819 session::HealthStatus,
7820 FrameType,
7821 };
7822
7823 use super::*;
7824 use crate::{
7825 forwarding::{DataRoute, DataRouteState},
7826 registry::ChannelState,
7827 router::FrameSink,
7828 stderr_tail::DEFAULT_MAX_LINE_BYTES,
7829 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7830 RouteCtx, Router,
7831 };
7832 use tokio::{
7833 sync::mpsc,
7834 time::{sleep, Instant},
7835 };
7836 use tracing::{
7837 field::{Field, Visit},
7838 Event, Subscriber,
7839 };
7840 use tracing_subscriber::{layer::Context, prelude::*, Layer};
7841
7842 fn fake_aft_stub_path() -> PathBuf {
7861 let mut path = std::env::current_exe().expect("current_exe available in tests");
7862 path.pop(); path.pop(); path.push(if cfg!(windows) {
7865 "fake-aft-stub.exe"
7866 } else {
7867 "fake-aft-stub"
7868 });
7869 assert!(
7870 path.exists(),
7871 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7872 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7873 path.display()
7874 );
7875 path
7876 }
7877
7878 fn client_retries(code: &str) -> bool {
7883 subc_protocol::error_codes::is_retryable_route_open(code)
7884 }
7885
7886 #[test]
7897 fn retryability_of_forwarding_codes_matches_the_failure() {
7898 let transient = [
7901 ForwardingError::NoModuleConnection,
7902 ForwardingError::ModuleReloading {
7903 module_id: "m".into(),
7904 },
7905 ForwardingError::StaleModuleEndpoint,
7906 ForwardingError::UnknownReservation {
7907 client_channel: 1,
7908 module_channel: 1,
7909 },
7910 ForwardingError::ConnectionClosing {
7911 connection_id: ConnectionId::new(1),
7912 },
7913 ForwardingError::ClientEgressClosed {
7914 connection_id: ConnectionId::new(1),
7915 },
7916 ForwardingError::ModuleEgressUnavailable {
7917 connection_id: ConnectionId::new(1),
7918 },
7919 ];
7920 for err in transient {
7921 let code = forwarding_error_code(&err);
7922 assert!(
7923 client_retries(code),
7924 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7925 );
7926 }
7927
7928 let permanent = [
7933 ForwardingError::ConnectionRoleConflict {
7934 connection_id: ConnectionId::new(1),
7935 },
7936 ForwardingError::ClientRouteChannelExhausted {
7937 connection_id: ConnectionId::new(1),
7938 },
7939 ForwardingError::ModuleRouteChannelExhausted {
7940 endpoint: ModuleEndpointId {
7941 connection_id: ConnectionId::new(1),
7942 generation: 1,
7943 },
7944 },
7945 ForwardingError::RelayCorrelationExhausted,
7946 ForwardingError::RouteOpenBuild("x".into()),
7947 ForwardingError::Poisoned,
7948 ];
7949 for err in permanent {
7950 let code = forwarding_error_code(&err);
7951 assert!(
7952 !client_retries(code),
7953 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7954 );
7955 }
7956 }
7957
7958 #[tokio::test]
7986 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7987 let handler = ControlHandler::default();
7988 let frame =
7989 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7990
7991 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7995 assert_eq!(
7996 stamped,
7997 Principal::Direct,
7998 "a caller that proved nothing must not be stamped as a supervised module"
7999 );
8000
8001 let forged = handler
8006 .route_open_principal(
8007 &frame,
8008 Some(ConsumerIdentity {
8009 module_id: "aft".to_string(),
8010 launch_nonce: "not-a-real-nonce".to_string(),
8011 }),
8012 )
8013 .unwrap();
8014 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8015 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8016 }
8017
8018 #[test]
8035 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8036 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8037 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8038 let ClientControlRequest::RouteOpen {
8039 consumer_identity, ..
8040 } = parsed
8041 else {
8042 panic!("route.open body must parse as RouteOpen");
8043 };
8044 assert_eq!(
8045 consumer_identity,
8046 Some(ConsumerIdentity {
8047 module_id: "aft".to_string(),
8048 launch_nonce: "n".to_string(),
8049 }),
8050 "the wire field name must reach the value route_open_principal reads"
8051 );
8052 }
8053
8054 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8055 ModuleManifest::builder(module_id, "0.1.0")
8056 .protocol_ver(protocol_ver)
8057 .provides(vec![ProviderRole::ToolProvider {
8058 tools: vec![Tool {
8059 name: "read".to_string(),
8060 description: None,
8061 execution_mode: ExecutionMode::Pure,
8062 schema: json!({"type": "object"}),
8063 }],
8064 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8065 concurrency: Concurrency::ModuleManaged,
8066 emits_push: true,
8067 sub_supervises: true,
8068 }])
8069 .build()
8070 }
8071
8072 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8073 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8074 }
8075
8076 fn hello_frame_with_control_ops(
8077 module_id: &str,
8078 protocol_ver: u8,
8079 corr: u64,
8080 control_ops: Option<Vec<String>>,
8081 ) -> Frame {
8082 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8083 }
8084
8085 fn hello_frame_with_nonce(
8086 module_id: &str,
8087 protocol_ver: u8,
8088 corr: u64,
8089 launch_nonce: Option<&str>,
8090 ) -> Frame {
8091 hello_frame_full(
8092 module_id,
8093 protocol_ver,
8094 corr,
8095 None,
8096 launch_nonce.map(ToOwned::to_owned),
8097 )
8098 }
8099
8100 fn hello_frame_full(
8101 module_id: &str,
8102 protocol_ver: u8,
8103 corr: u64,
8104 control_ops: Option<Vec<String>>,
8105 launch_nonce: Option<String>,
8106 ) -> Frame {
8107 let body = serde_json::to_vec(&ModuleHelloBody {
8108 manifest: manifest(module_id, protocol_ver),
8109 protocol_ver,
8110 control_ops,
8111 launch_nonce,
8112 })
8113 .unwrap();
8114 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8115 }
8116
8117 fn non_routable_hello_frame_with_control_ops(
8118 module_id: &str,
8119 corr: u64,
8120 control_ops: Option<Vec<String>>,
8121 ) -> Frame {
8122 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8123 manifest.provides.clear();
8124 let body = serde_json::to_vec(&ModuleHelloBody {
8125 manifest,
8126 protocol_ver: PROTOCOL_VERSION,
8127 control_ops,
8128 launch_nonce: None,
8129 })
8130 .unwrap();
8131 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8132 }
8133
8134 fn capability_grammar_hello_frame(
8135 capabilities: Value,
8136 runtime_computed: Option<Value>,
8137 corr: u64,
8138 ) -> Frame {
8139 let mut body = serde_json::to_value(ModuleHelloBody {
8140 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8141 protocol_ver: PROTOCOL_VERSION,
8142 control_ops: None,
8143 launch_nonce: None,
8144 })
8145 .expect("HELLO body serializes");
8146 body["manifest"]["capabilities"] = capabilities;
8147 if let Some(runtime_computed) = runtime_computed {
8148 body["runtime_computed"] = runtime_computed;
8149 }
8150 Frame::build(
8151 FrameType::Hello,
8152 control_flags(),
8153 0,
8154 0,
8155 corr,
8156 serde_json::to_vec(&body).expect("HELLO body reserializes"),
8157 )
8158 .expect("HELLO frame builds")
8159 }
8160
8161 fn channel_request(channel: u16, corr: u64) -> Frame {
8162 Frame::build(
8163 FrameType::Request,
8164 Flags::new(true, Priority::Interactive, false),
8165 channel,
8166 0,
8167 corr,
8168 b"opaque".to_vec(),
8169 )
8170 .unwrap()
8171 }
8172
8173 fn route_ctx(
8174 connection_id: ConnectionId,
8175 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8176 let (tx, rx) = mpsc::channel(8);
8177 (
8178 RouteCtx {
8179 connection_id,
8180 egress: FrameSink::new(tx),
8181 },
8182 rx,
8183 )
8184 }
8185
8186 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8187 serde_json::from_slice(&frame.body).unwrap()
8188 }
8189
8190 async fn hello_via_sink(
8195 handler: &ControlHandler,
8196 ctx: &RouteCtx,
8197 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8198 hello: Frame,
8199 ) -> Frame {
8200 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8201 assert!(
8202 replies.is_empty(),
8203 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8204 );
8205 let ack = rx
8206 .try_recv()
8207 .expect("HELLO_ACK is queued on the module sink")
8208 .frame;
8209 assert_eq!(ack.header.ty, FrameType::HelloAck);
8210 ack
8211 }
8212
8213 fn parse_error(frame: &Frame) -> Value {
8214 serde_json::from_slice(&frame.body).unwrap()
8215 }
8216
8217 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8218 serde_json::from_slice(&frame.body).unwrap()
8219 }
8220
8221 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8222 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8223 route_channel,
8224 route_epoch: 0,
8225 kind,
8226 })
8227 .unwrap();
8228 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8229 }
8230
8231 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8232 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8233 module_id: module_id.to_string(),
8234 })
8235 .unwrap();
8236 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8237 }
8238
8239 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8240 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8241 }
8242
8243 fn route_open_frame_with_consumer_capabilities(
8244 corr: u64,
8245 module_id: &str,
8246 project_root: ScratchDir,
8247 consumer_capabilities: Option<Vec<String>>,
8248 ) -> Frame {
8249 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8250 target: RouteTarget::ToolProvider {
8251 module_id: module_id.to_string(),
8252 },
8253 identity: BindIdentity::new(
8254 project_root.path().to_path_buf(),
8255 "unit".to_string(),
8256 "session".to_string(),
8257 ),
8258 consumer_identity: None,
8259 consumer_capabilities,
8260 role_versions: None,
8261 admission_facts: None,
8262 scope: None,
8263 })
8264 .unwrap();
8265 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8266 }
8267
8268 fn route_open_frame_with_role_versions(
8269 corr: u64,
8270 module_id: &str,
8271 project_root: ScratchDir,
8272 role_versions: Option<BTreeMap<String, String>>,
8273 ) -> Frame {
8274 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8275 target: RouteTarget::ToolProvider {
8276 module_id: module_id.to_string(),
8277 },
8278 identity: BindIdentity::new(
8279 project_root.path().to_path_buf(),
8280 "unit".to_string(),
8281 format!("session-{corr}"),
8282 ),
8283 consumer_identity: None,
8284 consumer_capabilities: None,
8285 role_versions,
8286 admission_facts: None,
8287 scope: None,
8288 })
8289 .unwrap();
8290 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8291 }
8292
8293 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8294 entries
8295 .iter()
8296 .map(|(role, version)| (role.to_string(), version.to_string()))
8297 .collect()
8298 }
8299
8300 fn route_open_frame_with_admission_facts(
8301 corr: u64,
8302 module_id: &str,
8303 project_root: ScratchDir,
8304 consumer_identity: Option<subc_control::ConsumerIdentity>,
8305 facts: Option<Value>,
8306 ) -> Frame {
8307 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8308 target: RouteTarget::ToolProvider {
8309 module_id: module_id.to_string(),
8310 },
8311 identity: BindIdentity::new(
8312 project_root.path().to_path_buf(),
8313 "unit".to_string(),
8314 format!("session-{corr}"),
8315 ),
8316 consumer_identity,
8317 consumer_capabilities: None,
8318 role_versions: None,
8319 admission_facts: facts,
8320 scope: None,
8321 })
8322 .unwrap();
8323 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8324 }
8325
8326 #[derive(Clone, Default)]
8327 struct EventCapture {
8328 events: Arc<Mutex<Vec<CapturedEvent>>>,
8329 }
8330
8331 #[derive(Clone, Debug)]
8332 struct CapturedEvent {
8333 target: String,
8334 level: tracing::Level,
8335 fields: BTreeMap<String, String>,
8336 }
8337
8338 impl EventCapture {
8339 fn events(&self) -> Vec<CapturedEvent> {
8340 self.events.lock().unwrap().clone()
8341 }
8342 }
8343
8344 impl<S> Layer<S> for EventCapture
8345 where
8346 S: Subscriber,
8347 {
8348 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8349 let mut visitor = EventFieldVisitor::default();
8350 event.record(&mut visitor);
8351 self.events.lock().unwrap().push(CapturedEvent {
8352 target: event.metadata().target().to_string(),
8353 level: *event.metadata().level(),
8354 fields: visitor.fields,
8355 });
8356 }
8357 }
8358
8359 #[derive(Default)]
8360 struct EventFieldVisitor {
8361 fields: BTreeMap<String, String>,
8362 }
8363
8364 impl Visit for EventFieldVisitor {
8365 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8366 self.fields
8367 .insert(field.name().to_string(), format!("{value:?}"));
8368 }
8369 }
8370
8371 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8372 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8373 status,
8374 detail: Some("warming".to_string()),
8375 metrics: Some(json!({"queue_depth": 3})),
8376 })
8377 .unwrap();
8378 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8379 }
8380
8381 fn route_bind_ack(corr: u64) -> Frame {
8382 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8383 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8384 }
8385
8386 fn unique_project_root(label: &str) -> ScratchDir {
8387 ScratchDir::new(label)
8388 }
8389
8390 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8391 match parse_route_poll(frame) {
8392 ClientControlResponse::RoutePoll {
8393 status: None,
8394 live: Some(live),
8395 ..
8396 } => assert_eq!(live, expected_live),
8397 other => panic!("unexpected route.poll response: {other:?}"),
8398 }
8399 }
8400
8401 fn bind_liveness_route(
8402 registry: &Registry,
8403 forwarding: &ForwardingTable,
8404 module_id: &str,
8405 ) -> (RouteCtx, u16, u32) {
8406 let module_connection = ConnectionId::new(101);
8407 let client_connection = ConnectionId::new(202);
8408 let registration = registry
8409 .register_with_control_ops(
8410 manifest(module_id, PROTOCOL_VERSION),
8411 PROTOCOL_VERSION,
8412 module_connection,
8413 module_baseline_control_ops(),
8414 )
8415 .unwrap();
8416 let (module_tx, _module_rx) = mpsc::channel(8);
8417 let endpoint = forwarding
8418 .register_module_connection(
8419 module_connection,
8420 module_id.to_string(),
8421 PROTOCOL_VERSION,
8422 manifest_concurrency(®istration.manifest),
8423 FrameSink::new(module_tx),
8424 )
8425 .unwrap();
8426 let (client_ctx, _client_rx) = route_ctx(client_connection);
8427 let pending = forwarding
8428 .begin_route_bind_relay_for_test(
8429 client_connection,
8430 client_ctx.egress.clone(),
8431 1,
8432 module_id,
8433 )
8434 .unwrap();
8435 assert_eq!(pending.endpoint, endpoint);
8436 let route_channel = pending.client_channel;
8437 let route_epoch = pending.client_epoch;
8438 forwarding
8439 .complete_pending_relay(
8440 module_connection,
8441 pending.corr,
8442 RouteBindRelayOutcome::Accepted,
8443 )
8444 .unwrap();
8445 (client_ctx, route_channel, route_epoch)
8446 }
8447
8448 struct FakeProcessLiveness {
8449 live: Option<bool>,
8450 }
8451
8452 impl ModuleProcessLiveness for FakeProcessLiveness {
8453 fn process_live(&self, _module_id: &str) -> Option<bool> {
8454 self.live
8455 }
8456 }
8457
8458 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8459 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8460 {
8461 let registry = Arc::new(Registry::default());
8462 let supervisor_handle = SupervisorHandle::new();
8463 let supervisor = Supervisor::new_for_test(
8464 Arc::clone(®istry),
8465 RestartPolicy::new(1, Duration::from_millis(10)),
8466 )
8467 .with_handle(supervisor_handle.clone());
8468 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8469 let module = supervisor
8470 .spawn(ModuleSpec {
8471 module_id: "stderr-tail-wire".to_string(),
8472 program: fake_aft_stub_path(),
8473 args: Vec::new(),
8474 env: vec![
8475 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8476 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8477 ],
8478 reserved: false,
8479 reserved_prefixes: Vec::new(),
8480 protocol: ModuleProtocol::Subc,
8481 overlap: Default::default(),
8482 })
8483 .unwrap();
8484
8485 let deadline = Instant::now() + Duration::from_secs(5);
8486 loop {
8487 let tail = module.stderr_tail(None, None);
8488 if tail
8489 .entries
8490 .iter()
8491 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8492 && tail.entries.iter().any(|entry| {
8493 matches!(
8494 entry,
8495 TailEntry::Line {
8496 truncated: true,
8497 ..
8498 }
8499 )
8500 })
8501 {
8502 break;
8503 }
8504 assert!(
8505 Instant::now() < deadline,
8506 "module did not produce a truncated line and restart boundary: {tail:?}"
8507 );
8508 sleep(Duration::from_millis(10)).await;
8509 }
8510
8511 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8512 let request = ClientControlRequest::SupervisorStderrTail {
8513 module_id: "stderr-tail-wire".to_string(),
8514 max_lines: None,
8515 max_bytes: None,
8516 };
8517 let frame = Frame::build(
8518 FrameType::Request,
8519 control_flags(),
8520 0,
8521 0,
8522 1,
8523 serde_json::to_vec(&request).unwrap(),
8524 )
8525 .unwrap();
8526 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8527 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8528 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8529 serde_json::from_slice(&responses[0].body).unwrap()
8530 else {
8531 panic!("expected supervisor.stderr_tail response");
8532 };
8533
8534 assert!(
8535 tail.entries
8536 .iter()
8537 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8538 "the control response lost the restart boundary"
8539 );
8540 let Some(StderrTailEntry::Line {
8541 text,
8542 truncated,
8543 at_ms,
8544 }) = tail.entries.iter().find(|entry| {
8545 matches!(
8546 entry,
8547 StderrTailEntry::Line {
8548 truncated: true,
8549 ..
8550 }
8551 )
8552 })
8553 else {
8554 panic!("the control response lost the truncated line");
8555 };
8556 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8557 assert!(*truncated);
8558 assert!(
8559 at_ms.is_some(),
8560 "the control response lost the line's capture time"
8561 );
8562 }
8563
8564 #[tokio::test(flavor = "current_thread")]
8569 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8570 let dir = ScratchDir::new("terminals-off-worker");
8571 let journal_path = dir.join("terminals.jsonl");
8572 let registry = Arc::new(Registry::default());
8573 let supervisor_handle = SupervisorHandle::new();
8574 let supervisor =
8575 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8576 .with_handle(supervisor_handle.clone())
8577 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8578 let module = supervisor
8579 .spawn(ModuleSpec {
8580 module_id: "terminal-off-worker".to_string(),
8581 program: fake_aft_stub_path(),
8582 args: Vec::new(),
8583 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8584 reserved: false,
8585 reserved_prefixes: Vec::new(),
8586 protocol: ModuleProtocol::Subc,
8587 overlap: Default::default(),
8588 })
8589 .unwrap();
8590 let deadline = Instant::now() + Duration::from_secs(5);
8591 while module.terminal_history().entries.len() != 2 {
8592 assert!(Instant::now() < deadline, "module did not record two exits");
8593 sleep(Duration::from_millis(10)).await;
8594 }
8595
8596 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8597 let handler =
8598 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8599 let frame = Frame::build(
8600 FrameType::Request,
8601 control_flags(),
8602 0,
8603 0,
8604 1,
8605 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8606 module_id: "terminal-off-worker".to_string(),
8607 })
8608 .unwrap(),
8609 )
8610 .unwrap();
8611 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8612 let spawned_at = std::time::Instant::now();
8613 let read = tokio::spawn({
8614 let handler = Arc::clone(&handler);
8615 async move { handler.handle_control_frame(&ctx, frame).await }
8616 });
8617 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8620 .await
8621 .unwrap()
8622 .expect("the history read reached its pause");
8623 let elapsed = spawned_at.elapsed();
8624 assert!(
8625 elapsed < Duration::from_secs(2) && !read.is_finished(),
8626 "this task could not run while the history read was paused \
8627 (resumed after {elapsed:?}, read finished: {})",
8628 read.is_finished()
8629 );
8630
8631 drop(release);
8632 let responses = read.await.unwrap().unwrap();
8633 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8634 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8635 panic!("expected supervisor.terminals response");
8636 };
8637 assert_eq!(terminals.entries.len(), 2);
8638 assert_eq!(terminals.journal_skipped_lines, 0);
8639 assert_eq!(terminals.journal_read_errors, 0);
8640 }
8641
8642 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8643 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8644 let registry = Arc::new(Registry::default());
8645 let supervisor_handle = SupervisorHandle::new();
8646 let supervisor =
8647 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8648 .with_handle(supervisor_handle.clone());
8649 let module = supervisor
8650 .spawn(ModuleSpec {
8651 module_id: "terminal-golden".to_string(),
8652 program: fake_aft_stub_path(),
8653 args: Vec::new(),
8654 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8655 reserved: false,
8656 reserved_prefixes: Vec::new(),
8657 protocol: ModuleProtocol::Subc,
8658 overlap: Default::default(),
8659 })
8660 .unwrap();
8661
8662 let deadline = Instant::now() + Duration::from_secs(5);
8663 while module.terminal_history().entries.len() != 2 {
8664 assert!(
8665 Instant::now() < deadline,
8666 "module did not retain two terminal exits: {:?}",
8667 module.terminal_history()
8668 );
8669 sleep(Duration::from_millis(10)).await;
8670 }
8671
8672 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8673 let request = ClientControlRequest::SupervisorTerminals {
8674 module_id: "terminal-golden".to_string(),
8675 };
8676 let frame = Frame::build(
8677 FrameType::Request,
8678 control_flags(),
8679 0,
8680 0,
8681 1,
8682 serde_json::to_vec(&request).unwrap(),
8683 )
8684 .unwrap();
8685 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8686 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8687 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8688 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8689 panic!("expected supervisor.terminals response");
8690 };
8691 assert_eq!(terminals.entries.len(), 2);
8692 assert_eq!(terminals.dropped, 0);
8693
8694 let mut rendered = serde_json::to_value(response).unwrap();
8695 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8698 for (index, entry) in rendered["entries"]
8699 .as_array_mut()
8700 .expect("terminal response entries array")
8701 .iter_mut()
8702 .enumerate()
8703 {
8704 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8705 }
8706
8707 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8708 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8709 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8710 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8711 std::fs::write(&golden_path, &serialized).unwrap();
8712 }
8713 let expected: Value =
8714 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8715 assert_eq!(rendered, expected);
8716 }
8717
8718 #[test]
8719 fn hello_registers_manifest_and_returns_ack() {
8720 let registry = Arc::new(Registry::default());
8721 let handler = ControlHandler::new(Arc::clone(®istry));
8722 let conn = ConnectionId::new(1);
8723
8724 let responses = handler
8725 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8726 .unwrap();
8727
8728 assert_eq!(responses.len(), 1);
8729 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8730 assert_eq!(responses[0].header.channel, 0);
8731 assert_eq!(responses[0].header.corr, 7);
8732 let ack = parse_ack(&responses[0]);
8733 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8734 assert!(ack
8735 .subc_capabilities
8736 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8737 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8738 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8739 assert!(ack
8740 .subc_ops
8741 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8742 assert!(ack
8743 .subc_ops
8744 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8745
8746 let registration = registry.get_module("aft").unwrap().unwrap();
8747 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8748 assert_eq!(registration.state, ChannelState::Active);
8749 assert_eq!(registration.connection_id, conn);
8750 assert_eq!(registration.control_ops, module_baseline_control_ops());
8751 }
8752
8753 #[test]
8754 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8755 let invalid_identifiers = [
8756 ("case_change", "credentials-Provider/v1"),
8757 ("leading_zero", "credentials-provider/v01"),
8758 ("trailing_hyphen", "credentials-provider-/v1"),
8759 ("consecutive_hyphens", "credentials--provider/v1"),
8760 ("uppercase", "Credentials-provider/v1"),
8761 ("missing_v", "credentials-provider/1"),
8762 ("whitespace", "credentials provider/v1"),
8763 ("zero_version", "credentials-provider/v0"),
8764 ("out_of_range_version", "credentials-provider/v4294967296"),
8765 (
8766 "overlength_name",
8767 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8768 ),
8769 ];
8770 let mut cases = invalid_identifiers
8771 .into_iter()
8772 .map(|(name, identifier)| {
8773 (
8774 format!("identifier_{name}"),
8775 "capabilities.provides[0]".to_string(),
8776 identifier.to_string(),
8777 json!({ "provides": [identifier] }),
8778 None,
8779 )
8780 })
8781 .collect::<Vec<_>>();
8782 cases.extend([
8783 (
8784 "unknown_need".to_string(),
8785 "capabilities.requires[0].need".to_string(),
8786 "deferred".to_string(),
8787 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8788 None,
8789 ),
8790 (
8791 "duplicate_provides".to_string(),
8792 "capabilities.provides[1]".to_string(),
8793 "credentials-provider/v1".to_string(),
8794 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8795 None,
8796 ),
8797 (
8798 "duplicate_must_never_reach".to_string(),
8799 "capabilities.must_never_reach[1]".to_string(),
8800 "credentials-provider/v1".to_string(),
8801 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8802 None,
8803 ),
8804 (
8805 "duplicate_requires_same_need".to_string(),
8806 "capabilities.requires[1]".to_string(),
8807 "credentials-provider/v1".to_string(),
8808 json!({ "requires": [
8809 { "capability": "credentials-provider/v1", "need": "required" },
8810 { "capability": "credentials-provider/v1", "need": "required" }
8811 ] }),
8812 None,
8813 ),
8814 (
8815 "duplicate_requires_conflicting_need".to_string(),
8816 "capabilities.requires[1]".to_string(),
8817 "credentials-provider/v1".to_string(),
8818 json!({ "requires": [
8819 { "capability": "credentials-provider/v1", "need": "required" },
8820 { "capability": "credentials-provider/v1", "need": "optional" }
8821 ] }),
8822 None,
8823 ),
8824 (
8825 "capabilities_root_pointer".to_string(),
8826 "runtime_computed[0]".to_string(),
8827 "/capabilities".to_string(),
8828 json!({}),
8829 Some(json!(["/capabilities"])),
8830 ),
8831 (
8832 "capabilities_descendant_pointer".to_string(),
8833 "runtime_computed[0]".to_string(),
8834 "/capabilities/provides".to_string(),
8835 json!({}),
8836 Some(json!(["/capabilities/provides"])),
8837 ),
8838 (
8839 "malformed_pointer_without_leading_slash".to_string(),
8840 "runtime_computed[0]".to_string(),
8841 "capabilities".to_string(),
8842 json!({}),
8843 Some(json!(["capabilities"])),
8844 ),
8845 (
8846 "malformed_pointer_escape".to_string(),
8847 "runtime_computed[0]".to_string(),
8848 "/roles/~2/tools".to_string(),
8849 json!({}),
8850 Some(json!(["/roles/~2/tools"])),
8851 ),
8852 (
8853 "unknown_capabilities_field".to_string(),
8854 "capabilities.future".to_string(),
8855 "<array>".to_string(),
8856 json!({ "future": [] }),
8857 None,
8858 ),
8859 ]);
8860
8861 for (index, (name, field, value, capabilities, runtime_computed)) in
8862 cases.into_iter().enumerate()
8863 {
8864 let registry = Arc::new(Registry::default());
8865 let handler = ControlHandler::new(Arc::clone(®istry));
8866 let response = handler
8867 .handle_control(
8868 ConnectionId::new((index + 1) as u64),
8869 capability_grammar_hello_frame(
8870 capabilities,
8871 runtime_computed,
8872 index as u64 + 1,
8873 ),
8874 )
8875 .expect("invalid HELLO returns a refusal");
8876
8877 assert_eq!(response.len(), 1, "{name} must emit one refusal");
8878 let error = parse_error(&response[0]);
8879 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8880 let message = error["message"]
8881 .as_str()
8882 .expect("error message is a string");
8883 assert!(
8884 message.contains(&field),
8885 "{name}: field missing from {message}"
8886 );
8887 assert!(
8888 message.contains(&value),
8889 "{name}: value missing from {message}"
8890 );
8891 assert_eq!(
8892 registry
8893 .active_registration_count()
8894 .expect("registry reads"),
8895 0,
8896 "{name}: refused HELLO must not create a catalog entry"
8897 );
8898 }
8899 }
8900
8901 #[test]
8902 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8903 let registry = Arc::new(Registry::default());
8904 let handler = ControlHandler::new(Arc::clone(®istry));
8905 let capabilities = json!({
8906 "provides": ["credentials-provider/v1"],
8907 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8908 "must_never_reach": ["federation-transport/v1"]
8909 });
8910 let response = handler
8911 .handle_control(
8912 ConnectionId::new(99),
8913 capability_grammar_hello_frame(
8914 capabilities.clone(),
8915 Some(json!(["/roles/0/tools"])),
8916 99,
8917 ),
8918 )
8919 .expect("valid HELLO registers");
8920 assert_eq!(response[0].header.ty, FrameType::HelloAck);
8921
8922 let request = Frame::build(
8923 FrameType::Request,
8924 control_flags(),
8925 0,
8926 0,
8927 100,
8928 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8929 .expect("catalog request serializes"),
8930 )
8931 .expect("catalog request frame builds");
8932 let response = handler
8933 .handle_catalog_list(request, None)
8934 .expect("catalog list succeeds");
8935 let ClientControlResponse::CatalogList { modules, .. } =
8936 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8937 else {
8938 panic!("catalog request must return catalog.list");
8939 };
8940 assert_eq!(modules.len(), 1);
8941 assert_eq!(
8942 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8943 capabilities
8944 );
8945 }
8946
8947 #[test]
8948 fn catalog_list_mirrors_management_operation_description() {
8949 let registry = Arc::new(Registry::default());
8950 let handler = ControlHandler::new(Arc::clone(®istry));
8951 let description = "List managed records and return their identifiers and metadata.";
8952 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8953 manifest.provides = vec![ProviderRole::ManagementSurface {
8954 operations: vec![ManagementOperation {
8955 name: "records.list".to_string(),
8956 kind: ManagementOperationKind::Query,
8957 description: Some(description.to_string()),
8958 }],
8959 config_schema: json!({"type": "object"}),
8960 observability: vec![ObservabilitySurface {
8961 name: "records.stats".to_string(),
8962 kind: ObservabilityKind::Snapshot,
8963 }],
8964 identity_scope: vec![IdentityScope::Project],
8965 concurrency: Concurrency::ModuleManaged,
8966 }];
8967 registry
8968 .register_with_control_ops(
8969 manifest,
8970 PROTOCOL_VERSION,
8971 ConnectionId::new(99),
8972 Vec::new(),
8973 )
8974 .expect("described management manifest registers");
8975
8976 let request = Frame::build(
8977 FrameType::Request,
8978 control_flags(),
8979 0,
8980 0,
8981 100,
8982 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8983 .expect("catalog request serializes"),
8984 )
8985 .expect("catalog request frame builds");
8986 let response = handler
8987 .handle_catalog_list(request, None)
8988 .expect("catalog list succeeds");
8989 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8990 assert_eq!(
8991 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8992 "catalog.list must preserve the declared operation description verbatim"
8993 );
8994 }
8995
8996 #[test]
8997 fn catalog_list_mirrors_registered_operation_delivery() {
8998 let registry = Arc::new(Registry::default());
8999 let handler = ControlHandler::new(Arc::clone(®istry));
9000 let delivery =
9001 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9002 let mut body = serde_json::to_value(ModuleHelloBody {
9003 manifest: manifest("watch-management", PROTOCOL_VERSION),
9004 protocol_ver: PROTOCOL_VERSION,
9005 control_ops: None,
9006 launch_nonce: None,
9007 })
9008 .unwrap();
9009 body["manifest"]["provides"] = json!([{
9010 "role": "management_surface",
9011 "operations": [
9012 {"name": "status", "kind": "query"},
9013 {"name": "future", "kind": "query"},
9014 {"name": "reset", "kind": "mutate"}
9015 ],
9016 "config_schema": {}, "observability": [], "identity_scope": []
9017 }]);
9018 body["manifest"]["operation_delivery"] = delivery.clone();
9019 let hello = Frame::build(
9020 FrameType::Hello,
9021 control_flags(),
9022 0,
9023 0,
9024 99,
9025 serde_json::to_vec(&body).unwrap(),
9026 )
9027 .unwrap();
9028 let response = handler
9029 .handle_control(ConnectionId::new(99), hello)
9030 .unwrap();
9031 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9032
9033 let (_, registrations) = registry.list_modules().unwrap();
9034 let registered = ®istrations[0].manifest;
9035 assert!(registered.is_watch_operation("status"));
9036 assert!(!registered.is_watch_operation("future"));
9037 assert!(!registered.is_watch_operation("reset"));
9038 assert!(!registered.is_watch_operation("missing"));
9039
9040 let request = Frame::build(
9041 FrameType::Request,
9042 control_flags(),
9043 0,
9044 0,
9045 100,
9046 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9047 )
9048 .unwrap();
9049 let response = handler.handle_catalog_list(request, None).unwrap();
9050 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9051 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9052 let ClientControlResponse::CatalogList { modules, .. } =
9053 serde_json::from_slice(&response[0].body).unwrap()
9054 else {
9055 panic!("expected catalog.list");
9056 };
9057 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9058 }
9059
9060 #[test]
9061 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
9062 let registry = Arc::new(Registry::default());
9063 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9064 [("vault".to_string(), true), ("squatter".to_string(), true)],
9065 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9066 );
9067 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
9068 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9069 provides: vec!["credentials-provider/v1".to_string()],
9070 requires: Vec::new(),
9071 must_never_reach: Vec::new(),
9072 });
9073 let frame = Frame::build(
9074 FrameType::Hello,
9075 control_flags(),
9076 0,
9077 0,
9078 77,
9079 serde_json::to_vec(&ModuleHelloBody {
9080 manifest: squatter,
9081 protocol_ver: PROTOCOL_VERSION,
9082 control_ops: None,
9083 launch_nonce: None,
9084 })
9085 .expect("HELLO serializes"),
9086 )
9087 .expect("HELLO frame builds");
9088 let response = handler
9089 .handle_control(ConnectionId::new(77), frame)
9090 .expect("reserved claim receives a typed refusal");
9091 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9092 assert_eq!(
9093 registry
9094 .active_registration_count()
9095 .expect("registry reads"),
9096 0,
9097 "a reserved capability refusal must not leave a catalog entry"
9098 );
9099 }
9100
9101 #[test]
9102 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
9103 for settlement in ["timeout", "inconclusive", "drop"] {
9104 let breakers = RouteBindBreakers::default();
9105 let RouteBindAdmission::Admitted {
9106 guard: mut old,
9107 probe: false,
9108 } = breakers.admit("prov")
9109 else {
9110 panic!("ordinary relay admitted")
9111 };
9112 let RouteBindAdmission::Admitted {
9113 guard: mut opener, ..
9114 } = breakers.admit("prov")
9115 else {
9116 panic!("second relay admitted")
9117 };
9118 assert!(
9119 !opener
9120 .record_timeout(1, Duration::ZERO)
9121 .unwrap()
9122 .reopened_after_probe
9123 );
9124 let RouteBindAdmission::Admitted {
9125 guard: mut probe,
9126 probe: true,
9127 } = breakers.admit("prov")
9128 else {
9129 panic!("one cooldown probe admitted")
9130 };
9131 match settlement {
9132 "timeout" => assert!(
9133 !old.record_timeout(1, Duration::ZERO)
9134 .unwrap()
9135 .reopened_after_probe
9136 ),
9137 "inconclusive" => old.record_inconclusive(),
9138 "drop" => drop(old),
9139 _ => unreachable!(),
9140 }
9141 assert!(
9142 matches!(
9143 breakers.admit("prov"),
9144 RouteBindAdmission::Refused {
9145 probe_in_flight: true,
9146 ..
9147 }
9148 ),
9149 "{settlement} of a pre-open relay cannot release the real probe"
9150 );
9151 assert!(
9152 probe
9153 .record_timeout(1, Duration::ZERO)
9154 .unwrap()
9155 .reopened_after_probe
9156 );
9157 assert!(matches!(
9158 breakers.admit("prov"),
9159 RouteBindAdmission::Admitted { probe: true, .. }
9160 ));
9161 }
9162 let breakers = RouteBindBreakers::default();
9163 let admit = || match breakers.admit("prov") {
9164 RouteBindAdmission::Admitted { guard, .. } => guard,
9165 _ => panic!("relay admitted"),
9166 };
9167 admit().record_timeout(1, Duration::ZERO);
9168 let mut old_probe = admit();
9169 breakers.reset_for_new_module_connection("prov");
9170 admit().record_timeout(1, Duration::ZERO);
9171 let _new_probe = admit();
9172 old_probe.record_inconclusive();
9173 assert!(matches!(
9174 breakers.admit("prov"),
9175 RouteBindAdmission::Refused {
9176 probe_in_flight: true,
9177 ..
9178 }
9179 ));
9180 }
9181
9182 #[tokio::test]
9183 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9184 for candidate in [false, true] {
9185 let registry = Arc::new(Registry::default());
9186 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9187 [("vault".to_string(), true), ("squatter".to_string(), true)],
9188 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9189 );
9190 let conn = ConnectionId::new(77);
9191 let (ctx, mut rx) = route_ctx(conn);
9192 let initial = capability_manifest("squatter", &[], &[]);
9193 if candidate {
9194 registry
9195 .register_candidate_with_control_ops(
9196 initial.clone(),
9197 PROTOCOL_VERSION,
9198 conn,
9199 module_baseline_control_ops(),
9200 )
9201 .unwrap();
9202 handler
9203 .forwarding
9204 .register_candidate_module_connection(
9205 conn,
9206 "squatter".to_string(),
9207 PROTOCOL_VERSION,
9208 manifest_concurrency(&initial),
9209 ctx.egress.clone(),
9210 )
9211 .unwrap();
9212 } else {
9213 hello_via_sink(
9214 &handler,
9215 &ctx,
9216 &mut rx,
9217 hello_frame_with_manifest(initial.clone(), 1),
9218 )
9219 .await;
9220 }
9221 let response = handler
9222 .handle_control_frame(
9223 &ctx,
9224 catalog_update_with_capabilities_frame(
9225 2,
9226 capability_manifest("squatter", &["credentials-provider/v1"], &[])
9227 .capabilities
9228 .unwrap(),
9229 ),
9230 )
9231 .await
9232 .unwrap();
9233 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9234 assert_eq!(
9235 registry
9236 .get_module_by_connection(conn)
9237 .unwrap()
9238 .unwrap()
9239 .manifest,
9240 initial
9241 );
9242 }
9243 }
9244
9245 #[test]
9246 fn server_describe_surfaces_required_capability_verdict_fields() {
9247 let registry = Arc::new(Registry::default());
9248 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9249 [
9250 ("consumer".to_string(), true),
9251 ("provider".to_string(), false),
9252 ],
9253 BTreeMap::new(),
9254 );
9255 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9256 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9257 provides: Vec::new(),
9258 requires: vec![subc_protocol::manifest::CapabilityRequirement {
9259 capability: "credentials-provider/v1".to_string(),
9260 need: subc_protocol::manifest::CapabilityNeed::Required,
9261 }],
9262 must_never_reach: Vec::new(),
9263 });
9264 let hello = Frame::build(
9265 FrameType::Hello,
9266 control_flags(),
9267 0,
9268 0,
9269 78,
9270 serde_json::to_vec(&ModuleHelloBody {
9271 manifest: consumer,
9272 protocol_ver: PROTOCOL_VERSION,
9273 control_ops: None,
9274 launch_nonce: None,
9275 })
9276 .expect("HELLO serializes"),
9277 )
9278 .expect("HELLO frame builds");
9279 handler
9280 .handle_control(ConnectionId::new(78), hello)
9281 .expect("consumer registers");
9282 let describe = Frame::build(
9283 FrameType::Request,
9284 control_flags(),
9285 0,
9286 0,
9287 79,
9288 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
9289 .expect("request serializes"),
9290 )
9291 .expect("describe frame builds");
9292 let response = handler
9293 .handle_server_describe(describe)
9294 .expect("server.describe succeeds");
9295 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
9296 let requirement = &rendered["capability_requirements"][0];
9297 assert_eq!(requirement["consumer"], "consumer");
9298 assert_eq!(requirement["verdict"], "never_provided");
9299 assert_eq!(requirement["episode_seq"], 1);
9300 assert_eq!(requirement["config_satisfiable"], false);
9301 assert_eq!(requirement["runtime_available"], false);
9302 assert!(requirement["detail"]
9303 .as_str()
9304 .expect("detail string")
9305 .contains("credentials-provider/v1"));
9306 }
9307
9308 fn deregister_required_provider(
9314 daemon_shutdown_begun: bool,
9315 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
9316 const CAPABILITY: &str = "project-identity/v1";
9317 let registry = Arc::new(Registry::default());
9318 let daemon_shutdown = DaemonShutdownFlag::default();
9319 let handler = ControlHandler::new(Arc::clone(®istry))
9320 .with_capability_config(
9321 [
9322 ("consumer".to_string(), true),
9323 ("provider".to_string(), true),
9324 ],
9325 BTreeMap::new(),
9326 )
9327 .with_daemon_shutdown_flag(daemon_shutdown.clone());
9328 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9329 consumer.capabilities = Some(CapabilityDeclarations {
9330 provides: Vec::new(),
9331 requires: vec![subc_protocol::manifest::CapabilityRequirement {
9332 capability: CAPABILITY.to_string(),
9333 need: subc_protocol::manifest::CapabilityNeed::Required,
9334 }],
9335 must_never_reach: Vec::new(),
9336 });
9337 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
9338 let provider_conn = ConnectionId::new(182);
9339 for (conn, manifest, corr) in [
9340 (ConnectionId::new(181), consumer, 181),
9341 (provider_conn, provider, 182),
9342 ] {
9343 let hello = handler
9344 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
9345 .expect("module registers");
9346 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9347 }
9348 assert_eq!(
9349 handler.capability_evaluator.verdict("consumer", CAPABILITY),
9350 Some(CapabilityVerdict::Provided),
9351 "the registered provider must satisfy the requirement first"
9352 );
9353 if daemon_shutdown_begun {
9354 daemon_shutdown.set_for_test();
9355 }
9356
9357 let capture = EventCapture::default();
9358 let _subscriber =
9359 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9360 handler
9361 .cleanup_connection(provider_conn)
9362 .expect("provider deregisters");
9363 let events = capture
9364 .events()
9365 .into_iter()
9366 .filter(|event| {
9367 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
9368 })
9369 .collect();
9370 (
9371 events,
9372 handler.capability_evaluator.verdict("consumer", CAPABILITY),
9373 )
9374 }
9375
9376 #[test]
9377 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
9378 let (events, verdict) = deregister_required_provider(true);
9379 assert_eq!(
9380 verdict,
9381 Some(CapabilityVerdict::NeverProvided),
9382 "shutdown must not change the verdict route.open refusals read"
9383 );
9384 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9385 let event = &events[0];
9386 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
9387 assert_eq!(
9388 event.fields.get("cause").map(String::as_str),
9389 Some("\"daemon_shutdown\""),
9390 "{event:?}"
9391 );
9392 assert_eq!(
9393 event.fields.get("verdict").map(String::as_str),
9394 Some("\"never_provided\""),
9395 "{event:?}"
9396 );
9397 }
9398
9399 #[test]
9400 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
9401 let (events, verdict) = deregister_required_provider(false);
9402 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
9403 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9404 let event = &events[0];
9405 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
9406 assert_eq!(event.fields.get("cause"), None, "{event:?}");
9407 assert_eq!(
9408 event.fields.get("verdict").map(String::as_str),
9409 Some("\"never_provided\""),
9410 "{event:?}"
9411 );
9412 }
9413
9414 #[test]
9415 fn catalog_list_omits_capabilities_for_legacy_manifest() {
9416 let registry = Arc::new(Registry::default());
9417 let handler = ControlHandler::new(Arc::clone(®istry));
9418 let hello = handler
9419 .handle_control(
9420 ConnectionId::new(101),
9421 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
9422 )
9423 .expect("legacy HELLO registers");
9424 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9425
9426 let request = Frame::build(
9427 FrameType::Request,
9428 control_flags(),
9429 0,
9430 0,
9431 102,
9432 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9433 .expect("catalog request serializes"),
9434 )
9435 .expect("catalog request frame builds");
9436 let response = handler
9437 .handle_catalog_list(request, None)
9438 .expect("catalog list succeeds");
9439 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9440 assert!(
9441 body["modules"][0].get("capabilities").is_none(),
9442 "legacy manifest must retain an absent capabilities field on catalog.list"
9443 );
9444 }
9445
9446 #[test]
9447 fn hello_ack_omits_storage_when_no_storage_config() {
9448 let registry = Arc::new(Registry::default());
9449 let handler = ControlHandler::new(Arc::clone(®istry));
9450 let responses = handler
9451 .handle_control(
9452 ConnectionId::new(1),
9453 hello_frame("aft", PROTOCOL_VERSION, 7),
9454 )
9455 .unwrap();
9456 let ack = parse_ack(&responses[0]);
9457 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
9458 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
9459 }
9460
9461 #[tokio::test]
9462 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
9463 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
9464 let registry = Arc::new(Registry::default());
9465 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
9466 let responses = handler
9467 .handle_control(
9468 ConnectionId::new(1),
9469 hello_frame("aft", PROTOCOL_VERSION, 7),
9470 )
9471 .unwrap();
9472 let ack = parse_ack(&responses[0]);
9473 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
9474
9475 let described = handler
9476 .handle_control_frame(
9477 &route_ctx(ConnectionId::new(2)).0,
9478 Frame::build(
9479 FrameType::Request,
9480 control_flags(),
9481 0,
9482 0,
9483 9,
9484 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9485 )
9486 .unwrap(),
9487 )
9488 .await
9489 .unwrap();
9490 let ClientControlResponse::ServerDescribe { machine_id, .. } =
9491 serde_json::from_slice(&described[0].body).unwrap()
9492 else {
9493 panic!("server.describe answered with another shape");
9494 };
9495 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
9496 }
9497
9498 #[test]
9499 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
9500 let registry = Arc::new(Registry::default());
9503 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
9504 crate::daemon_config::StorageConfig::Sqlite {
9505 data_home: std::path::PathBuf::from("/data"),
9506 },
9507 ));
9508
9509 let responses = handler
9510 .handle_control(
9511 ConnectionId::new(1),
9512 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
9513 )
9514 .unwrap();
9515 let ack = parse_ack(&responses[0]);
9516 assert_eq!(
9517 ack.storage,
9518 Some(serde_json::json!({
9519 "module_id": "alfonso-routing",
9520 "storage_namespace": "default",
9521 "isolation": { "kind": "module" },
9522 "backend": {
9523 "backend": "sqlite",
9524 "path": "/data/cortexkit/alfonso-routing/store.db"
9525 }
9526 })),
9527 "the delivered descriptor is the module's own sqlite store path"
9528 );
9529 }
9530
9531 #[test]
9532 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
9533 let registry = Arc::new(Registry::default());
9534 let handler = ControlHandler::new(Arc::clone(®istry));
9535 let conn = ConnectionId::new(1);
9536 let responses = handler
9537 .handle_control(
9538 conn,
9539 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9540 )
9541 .unwrap();
9542 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9543 let registration = registry.get_module("aft").unwrap().unwrap();
9544 assert_eq!(registration.control_ops, module_baseline_control_ops());
9545
9546 let frame =
9547 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
9548 assert!(handler
9549 .guard_module_control_op(&frame, "aft", "route.bind")
9550 .unwrap()
9551 .is_none());
9552 let error = handler
9553 .guard_module_control_op(&frame, "aft", "test.synthetic")
9554 .unwrap()
9555 .expect("synthetic ungranted op should be rejected");
9556 assert_eq!(error.header.ty, FrameType::Error);
9557 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
9558 }
9559
9560 #[test]
9561 fn hello_control_ops_some_adds_optional_grants() {
9562 let registry = Arc::new(Registry::default());
9563 let handler = ControlHandler::new(Arc::clone(®istry));
9564 handler
9565 .handle_control(
9566 ConnectionId::new(1),
9567 hello_frame_with_control_ops(
9568 "aft",
9569 PROTOCOL_VERSION,
9570 7,
9571 Some(vec![
9572 "future.synthetic".to_string(),
9573 "route.bind".to_string(),
9574 ]),
9575 ),
9576 )
9577 .unwrap();
9578 let registration = registry.get_module("aft").unwrap().unwrap();
9579 assert_eq!(
9580 registration.control_ops,
9581 vec![
9582 "route.bind".to_string(),
9583 "route.status".to_string(),
9584 "future.synthetic".to_string(),
9585 ]
9586 );
9587 let frame =
9588 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9589 assert!(handler
9590 .guard_module_control_op(&frame, "aft", "future.synthetic")
9591 .unwrap()
9592 .is_none());
9593 }
9594
9595 #[tokio::test]
9596 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9597 let registry = Arc::new(Registry::default());
9598 let forwarding = Arc::new(ForwardingTable::default());
9599 let handler =
9600 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9601 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9602 hello_via_sink(
9603 &handler,
9604 &module_ctx,
9605 &mut module_rx,
9606 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9607 )
9608 .await;
9609
9610 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9611 let responses = handler
9612 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9613 .await
9614 .unwrap();
9615 assert_eq!(responses.len(), 1);
9616 assert_eq!(responses[0].header.ty, FrameType::Error);
9617 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9618 assert!(module_rx.try_recv().is_err());
9619 }
9620
9621 #[tokio::test]
9622 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9623 let registry = Arc::new(Registry::default());
9624 let forwarding = Arc::new(ForwardingTable::default());
9625 let handler =
9626 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9627 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9628 hello_via_sink(
9629 &handler,
9630 &module_ctx,
9631 &mut module_rx,
9632 hello_frame_with_control_ops(
9633 "aft",
9634 PROTOCOL_VERSION,
9635 7,
9636 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9637 ),
9638 )
9639 .await;
9640
9641 let project_root = unique_project_root("demux");
9642 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9643 let route_handler = handler.clone();
9644 let route_task = tokio::spawn(async move {
9645 route_handler
9646 .handle_control_frame(
9647 &route_client_ctx,
9648 route_open_frame(100, "aft", project_root),
9649 )
9650 .await
9651 .unwrap()
9652 });
9653 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9654 .await
9655 .unwrap()
9656 .unwrap();
9657 assert!(matches!(
9658 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
9659 ModuleControlRequest::RouteBind { .. }
9660 ));
9661
9662 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
9663 let health_handler = handler.clone();
9664 let health_task = tokio::spawn(async move {
9665 health_handler
9666 .handle_control_frame(
9667 &health_client_ctx,
9668 supervisor_health_probe_frame(101, "aft"),
9669 )
9670 .await
9671 .unwrap()
9672 });
9673 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9674 .await
9675 .unwrap()
9676 .unwrap();
9677 assert_eq!(
9678 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9679 ModuleControlRequest::HealthCheck {}
9680 );
9681
9682 handler
9683 .handle_control_frame(
9684 &module_ctx,
9685 health_response(health_frame.header.corr, HealthStatus::Degraded),
9686 )
9687 .await
9688 .unwrap();
9689 let health_response = health_task.await.unwrap();
9690 assert_eq!(health_response.len(), 1);
9691 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9692 ClientControlResponse::SupervisorHealthProbe {
9693 module_id,
9694 status,
9695 detail,
9696 metrics,
9697 } => {
9698 assert_eq!(module_id, "aft");
9699 assert_eq!(status, HealthStatus::Degraded);
9700 assert_eq!(detail.as_deref(), Some("warming"));
9701 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
9702 }
9703 other => panic!("unexpected health response: {other:?}"),
9704 }
9705
9706 handler
9707 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9708 .await
9709 .unwrap();
9710 let route_response = route_task.await.unwrap();
9711 assert!(route_response.is_empty());
9712 let published = route_client_rx.recv().await.unwrap();
9713 assert!(matches!(
9714 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9715 ClientControlResponse::RouteOpen { .. }
9716 ));
9717 }
9718
9719 async fn relay_route_open(
9724 handler: &ControlHandler,
9725 client_connection: ConnectionId,
9726 client_egress: &FrameSink,
9727 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9728 corr: u64,
9729 module_id: &str,
9730 project_root_label: &str,
9731 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
9732 let ctx = RouteCtx {
9733 connection_id: client_connection,
9734 egress: client_egress.clone(),
9735 };
9736 let handler = handler.clone();
9737 let project_root = unique_project_root(project_root_label);
9738 let module_id = module_id.to_string();
9739 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
9740 let task = tokio::spawn(async move {
9741 let _guard = tracing::dispatcher::set_default(&dispatch);
9742 handler
9743 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
9744 .await
9745 .unwrap()
9746 });
9747 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
9748 .await
9749 .expect("module receives the relayed route.bind")
9750 .expect("module egress is open");
9751 (task, bind.frame)
9752 }
9753
9754 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
9755 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
9756 ModuleControlRequest::RouteBind {
9757 route_channel,
9758 epoch,
9759 ..
9760 } => (route_channel, epoch),
9761 other => panic!("expected a route.bind request, got {other:?}"),
9762 }
9763 }
9764
9765 fn published_route(frame: &Frame) -> (u16, u32) {
9766 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
9767 ClientControlResponse::RouteOpen {
9768 route_channel,
9769 route_epoch,
9770 } => (route_channel, route_epoch),
9771 other => panic!("expected a route.open response, got {other:?}"),
9772 }
9773 }
9774
9775 #[tokio::test]
9795 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
9796 let registry = Arc::new(Registry::default());
9797 let forwarding = Arc::new(ForwardingTable::default());
9798 let handler =
9799 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9800
9801 let module_connection = ConnectionId::new(30);
9802 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9803 hello_via_sink(
9804 &handler,
9805 &module_ctx,
9806 &mut module_rx,
9807 hello_frame("aft", PROTOCOL_VERSION, 7),
9808 )
9809 .await;
9810
9811 let dying_client = ConnectionId::new(31);
9812 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
9813
9814 let (first_task, first_bind) = relay_route_open(
9817 &handler,
9818 dying_client,
9819 &dying_ctx.egress,
9820 &mut module_rx,
9821 100,
9822 "aft",
9823 "closing-first",
9824 )
9825 .await;
9826 handler
9827 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
9828 .await
9829 .unwrap();
9830 assert!(first_task.await.unwrap().is_empty());
9831 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
9832
9833 let (second_task, second_bind) = relay_route_open(
9835 &handler,
9836 dying_client,
9837 &dying_ctx.egress,
9838 &mut module_rx,
9839 101,
9840 "aft",
9841 "closing-second",
9842 )
9843 .await;
9844 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9845
9846 assert!(forwarding
9849 .escalate_client_delivery_failure(
9850 dying_client,
9851 first_channel,
9852 first_epoch,
9853 CloseReason::new(
9854 "module_to_client_delivery_failed",
9855 "client egress refused a module frame",
9856 ),
9857 crate::forwarding::UndeliveredFrame {
9858 module_id: None,
9859 sink: &dying_ctx.egress,
9860 },
9861 )
9862 .unwrap());
9863 assert!(!dying_ctx.egress.is_closed());
9864
9865 let ack = handler
9867 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9868 .await;
9869 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9870 if module_loop_error.is_some() {
9871 handler.cleanup_connection(module_connection).unwrap();
9875 }
9876 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9880 .await
9881 .ok()
9882 .flatten();
9883
9884 assert!(
9886 registry
9887 .get_module_by_connection(module_connection)
9888 .unwrap()
9889 .is_some(),
9890 "one client's closing connection ended the shared module connection: \
9891 {module_loop_error:?}"
9892 );
9893 let cotenant = ConnectionId::new(32);
9895 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9896 let (cotenant_task, cotenant_bind) = relay_route_open(
9897 &handler,
9898 cotenant,
9899 &cotenant_ctx.egress,
9900 &mut module_rx,
9901 102,
9902 "aft",
9903 "closing-cotenant",
9904 )
9905 .await;
9906 handler
9907 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9908 .await
9909 .unwrap();
9910 assert!(cotenant_task.await.unwrap().is_empty());
9911 let (cotenant_channel, cotenant_epoch) =
9912 published_route(&cotenant_rx.recv().await.unwrap());
9913 assert!(matches!(
9914 forwarding
9915 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9916 .unwrap(),
9917 DataRoute::Client(DataRouteState::Bound(_))
9918 ));
9919
9920 let goodbye = post_ack_module_frame
9923 .expect("module receives a GOODBYE for the abandoned route channel");
9924 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9925 assert_eq!(goodbye.header.channel, abandoned_channel);
9926 assert_eq!(goodbye.header.epoch, abandoned_epoch);
9927
9928 assert!(dying_rx.try_recv().is_err());
9932 let second_response = second_task.await.unwrap();
9933 assert_eq!(second_response.len(), 1);
9934 assert_eq!(
9935 parse_error(&second_response[0])["code"],
9936 "target_unavailable"
9937 );
9938 }
9939
9940 async fn catalog_snapshot(
9943 handler: &ControlHandler,
9944 client_ctx: &RouteCtx,
9945 corr: u64,
9946 ) -> (u64, Vec<(String, bool)>) {
9947 let body =
9948 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
9949 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
9950 let reply = handler
9951 .handle_control_frame(client_ctx, request)
9952 .await
9953 .unwrap()
9954 .pop()
9955 .unwrap();
9956 let ClientControlResponse::CatalogList {
9957 generation,
9958 modules,
9959 ..
9960 } = serde_json::from_slice(&reply.body).unwrap()
9961 else {
9962 panic!("not a catalog.list reply");
9963 };
9964 (
9965 generation,
9966 modules
9967 .into_iter()
9968 .map(|entry| (entry.module_id, entry.ready))
9969 .collect(),
9970 )
9971 }
9972
9973 fn hello_held_before_forwarding() -> (
9979 ControlHandler,
9980 Arc<std::sync::Barrier>,
9981 std::thread::JoinHandle<Vec<Frame>>,
9982 RouteCtx,
9983 mpsc::Receiver<crate::router::OutboundFrame>,
9984 ) {
9985 let hold = Arc::new(std::sync::Barrier::new(2));
9986 let handler = ControlHandler::with_forwarding(
9987 Arc::new(Registry::default()),
9988 Arc::new(ForwardingTable::default()),
9989 )
9990 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
9991 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9992 let hello = {
9993 let handler = handler.clone();
9994 let module_ctx = module_ctx.clone();
9995 std::thread::spawn(move || {
9996 handler
9997 .handle_hello(
9998 module_ctx.connection_id,
9999 Some(module_ctx.egress.clone()),
10000 hello_frame("aft", PROTOCOL_VERSION, 7),
10001 )
10002 .unwrap()
10003 })
10004 };
10005 (handler, hold, hello, module_ctx, module_rx)
10006 }
10007
10008 #[tokio::test]
10017 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
10018 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
10019 let client = ConnectionId::new(41);
10020 let (client_ctx, mut client_rx) = route_ctx(client);
10021
10022 hold.wait();
10023 assert!(handler.registry.get_module("aft").unwrap().is_some());
10025 assert!(!handler
10026 .forwarding
10027 .has_live_module_connection("aft")
10028 .unwrap());
10029 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
10030 assert!(
10031 listed.is_empty(),
10032 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
10033 );
10034 let refused = handler
10035 .handle_control_frame(
10036 &client_ctx,
10037 route_open_frame(301, "aft", unique_project_root("listed-window")),
10038 )
10039 .await
10040 .unwrap();
10041 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
10042 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
10043 let duplicate = handler
10044 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
10045 .await
10046 .unwrap();
10047 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
10048
10049 hold.wait();
10050 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10051 let ack = module_rx.try_recv().unwrap().frame;
10052 assert_eq!(ack.header.ty, FrameType::HelloAck);
10053 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10054 assert_eq!(listed, vec![("aft".to_string(), true)]);
10055
10056 let (open_task, bind) = relay_route_open(
10057 &handler,
10058 client,
10059 &client_ctx.egress,
10060 &mut module_rx,
10061 303,
10062 "aft",
10063 "listed-after",
10064 )
10065 .await;
10066 assert!(handler
10067 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10068 .await
10069 .unwrap()
10070 .is_empty());
10071 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
10072 published_route(&client_rx.try_recv().unwrap().frame);
10073 }
10074
10075 #[tokio::test]
10082 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
10083 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
10084 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10085 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10086
10087 hold.wait();
10088 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
10089 assert!(listed.is_empty());
10090 assert_eq!(
10091 during, before,
10092 "the generation moved while the module was not listed"
10093 );
10094
10095 hold.wait();
10096 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
10097 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
10098 assert_eq!(listed, vec![("aft".to_string(), true)]);
10099 assert_eq!(
10100 after,
10101 before + 1,
10102 "the module appeared without exactly one generation bump"
10103 );
10104 }
10105
10106 #[tokio::test]
10111 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10112 let registry = Arc::new(Registry::default());
10113 let handler = ControlHandler::with_forwarding(
10114 Arc::clone(®istry),
10115 Arc::new(ForwardingTable::default()),
10116 );
10117 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10118 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10119 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10120 drop(module_rx);
10121
10122 let replies = handler
10123 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10124 .await
10125 .unwrap();
10126 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10127 assert!(registry.get_module("aft").unwrap().is_none());
10128 assert!(registry
10129 .get_module_by_connection(module_ctx.connection_id)
10130 .unwrap()
10131 .is_none());
10132 assert_eq!(
10133 catalog_snapshot(&handler, &client_ctx, 301).await,
10134 (before, Vec::new())
10135 );
10136 }
10137
10138 async fn route_open_held_after_bind_send() -> (
10142 ControlHandler,
10143 Arc<tokio::sync::Barrier>,
10144 RouteCtx,
10145 mpsc::Receiver<crate::router::OutboundFrame>,
10146 ) {
10147 let hold = Arc::new(tokio::sync::Barrier::new(2));
10148 let handler = ControlHandler::with_forwarding(
10149 Arc::new(Registry::default()),
10150 Arc::new(ForwardingTable::default()),
10151 )
10152 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10153 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10154 hello_via_sink(
10155 &handler,
10156 &module_ctx,
10157 &mut module_rx,
10158 hello_frame("aft", PROTOCOL_VERSION, 7),
10159 )
10160 .await;
10161 (handler, hold, module_ctx, module_rx)
10162 }
10163
10164 #[tokio::test]
10171 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10172 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10173 let client = ConnectionId::new(41);
10174 let (client_ctx, mut client_rx) = route_ctx(client);
10175
10176 let (open_task, bind) = relay_route_open(
10177 &handler,
10178 client,
10179 &client_ctx.egress,
10180 &mut module_rx,
10181 200,
10182 "aft",
10183 "acked-before-mark",
10184 )
10185 .await;
10186 let (module_channel, module_epoch) = route_bind_channel(&bind);
10187 hold.wait().await;
10189
10190 assert!(handler
10192 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10193 .await
10194 .unwrap()
10195 .is_empty());
10196 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10197
10198 hold.wait().await;
10199 assert!(
10200 open_task.await.unwrap().is_empty(),
10201 "route.open answers accepted"
10202 );
10203
10204 match module_rx.try_recv() {
10207 Err(mpsc::error::TryRecvError::Empty) => {}
10208 Ok(outbound) => panic!(
10209 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10210 outbound.frame.header.ty,
10211 outbound.frame.header.channel,
10212 outbound.frame.header.epoch
10213 ),
10214 Err(err) => panic!("module egress closed: {err}"),
10215 }
10216
10217 let router = Router::with_control_handler(Arc::new(handler.clone()));
10220 let request = Frame::build(
10221 FrameType::Request,
10222 Flags::new(true, Priority::Interactive, false),
10223 client_channel,
10224 client_epoch,
10225 201,
10226 b"first request".to_vec(),
10227 )
10228 .unwrap();
10229 router
10230 .route_for_connection(&client_ctx, request)
10231 .await
10232 .unwrap();
10233 let delivered = module_rx.try_recv().unwrap().frame;
10234 assert_eq!(delivered.header.ty, FrameType::Request);
10235 assert_eq!(
10236 (delivered.header.channel, delivered.header.epoch),
10237 (module_channel, module_epoch)
10238 );
10239 assert_eq!(delivered.body, b"first request");
10240 }
10241
10242 #[tokio::test]
10248 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10249 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10250 let client = ConnectionId::new(42);
10251 let (client_ctx, _client_rx) = route_ctx(client);
10252
10253 let (open_task, bind) = relay_route_open(
10254 &handler,
10255 client,
10256 &client_ctx.egress,
10257 &mut module_rx,
10258 300,
10259 "aft",
10260 "released-before-mark",
10261 )
10262 .await;
10263 let (module_channel, module_epoch) = route_bind_channel(&bind);
10264 hold.wait().await;
10265
10266 handler.cleanup_connection(client).unwrap();
10267 assert!(
10268 module_rx.try_recv().is_err(),
10269 "client teardown must not GOODBYE a bind route.open is still queueing"
10270 );
10271
10272 hold.wait().await;
10273 let response = open_task.await.unwrap();
10274 assert_eq!(response.len(), 1);
10275 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10276
10277 let goodbye = module_rx
10278 .try_recv()
10279 .expect("module is told to drop the binding the daemon abandoned")
10280 .frame;
10281 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10282 assert_eq!(
10283 (goodbye.header.channel, goodbye.header.epoch),
10284 (module_channel, module_epoch)
10285 );
10286 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10287 }
10288
10289 #[tokio::test]
10294 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
10295 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10296 let client = ConnectionId::new(43);
10297 let (client_ctx, client_rx) = route_ctx(client);
10298
10299 let (open_task, bind) = relay_route_open(
10300 &handler,
10301 client,
10302 &client_ctx.egress,
10303 &mut module_rx,
10304 400,
10305 "aft",
10306 "gone-client-before-mark",
10307 )
10308 .await;
10309 let (module_channel, module_epoch) = route_bind_channel(&bind);
10310 hold.wait().await;
10311
10312 drop(client_rx);
10313 assert!(client_ctx.egress.is_closed());
10314 handler
10315 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10316 .await
10317 .unwrap();
10318
10319 hold.wait().await;
10320 let response = open_task.await.unwrap();
10321 assert_eq!(response.len(), 1);
10322 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10323
10324 let goodbye = module_rx
10325 .try_recv()
10326 .expect("module is told to drop the binding it acked")
10327 .frame;
10328 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10329 assert_eq!(
10330 (goodbye.header.channel, goodbye.header.epoch),
10331 (module_channel, module_epoch)
10332 );
10333 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10334 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
10335 }
10336
10337 #[test]
10344 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
10345 let handler = ControlHandler::default();
10346 let module_connection = ConnectionId::new(30);
10347 let client_connection = ConnectionId::new(31);
10348
10349 handler
10350 .refuse_to_end_module_connection_for_a_client(
10351 module_connection,
10352 77,
10353 ForwardingError::ConnectionClosing {
10354 connection_id: client_connection,
10355 },
10356 )
10357 .expect("a closing client must never end the module connection");
10358
10359 assert!(matches!(
10360 handler.refuse_to_end_module_connection_for_a_client(
10361 module_connection,
10362 78,
10363 ForwardingError::ConnectionClosing {
10364 connection_id: module_connection,
10365 },
10366 ),
10367 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
10368 connection_id
10369 })) if connection_id == module_connection
10370 ));
10371 assert!(matches!(
10372 handler.refuse_to_end_module_connection_for_a_client(
10373 module_connection,
10374 79,
10375 ForwardingError::Poisoned,
10376 ),
10377 Err(RouterError::Forwarding(ForwardingError::Poisoned))
10378 ));
10379 assert!(matches!(
10380 handler.refuse_to_end_module_connection_for_a_client(
10381 module_connection,
10382 80,
10383 ForwardingError::StaleModuleEndpoint,
10384 ),
10385 Err(RouterError::Forwarding(
10386 ForwardingError::StaleModuleEndpoint
10387 ))
10388 ));
10389 }
10390
10391 #[tokio::test]
10404 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
10405 let registry = Arc::new(Registry::default());
10406 let forwarding = Arc::new(ForwardingTable::default());
10407 let supervisor = SupervisorHandle::new();
10408 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10409 let handler =
10410 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10411 .with_supervisor(supervisor);
10412
10413 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10414 hello_via_sink(
10415 &handler,
10416 &target_ctx,
10417 &mut target_rx,
10418 hello_frame("target", PROTOCOL_VERSION, 1),
10419 )
10420 .await;
10421
10422 let wrong_nonce = handler
10426 .handle_control_frame(
10427 &route_ctx(ConnectionId::new(91)).0,
10428 route_open_frame_with_admission_facts(
10429 20,
10430 "target",
10431 unique_project_root("admission-facts"),
10432 Some(subc_control::ConsumerIdentity {
10433 module_id: "fed".to_string(),
10434 launch_nonce: "not-the-real-nonce".to_string(),
10435 }),
10436 None,
10437 ),
10438 )
10439 .await
10440 .unwrap();
10441 assert_eq!(
10442 parse_error(&wrong_nonce[0])["code"],
10443 "bad_consumer_identity",
10444 "a mismatched launch nonce must be refused, not stamped Reserved"
10445 );
10446
10447 let never_spawned = handler
10451 .handle_control_frame(
10452 &route_ctx(ConnectionId::new(92)).0,
10453 route_open_frame_with_admission_facts(
10454 21,
10455 "target",
10456 unique_project_root("admission-facts"),
10457 Some(subc_control::ConsumerIdentity {
10458 module_id: "never-spawned".to_string(),
10459 launch_nonce: "any-nonce".to_string(),
10460 }),
10461 None,
10462 ),
10463 )
10464 .await
10465 .unwrap();
10466 assert_eq!(
10467 parse_error(&never_spawned[0])["code"],
10468 "bad_consumer_identity",
10469 "an unspawned module_id must be refused rather than accepted for lack of a record"
10470 );
10471 }
10472
10473 #[tokio::test]
10496 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
10497 let registry = Arc::new(Registry::default());
10498 let forwarding = Arc::new(ForwardingTable::default());
10499 let supervisor = SupervisorHandle::new();
10500 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10501 let handler =
10502 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10503 .with_supervisor(supervisor);
10504
10505 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
10506 hello_via_sink(
10507 &handler,
10508 &target_ctx,
10509 &mut target_rx,
10510 hello_frame("target", PROTOCOL_VERSION, 1),
10511 )
10512 .await;
10513
10514 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
10515 let route_handler = handler.clone();
10516 let route_task = tokio::spawn(async move {
10517 route_handler
10518 .handle_control_frame(
10519 &client_ctx,
10520 route_open_frame_with_admission_facts(
10521 30,
10522 "target",
10523 unique_project_root("admission-facts"),
10524 Some(subc_control::ConsumerIdentity {
10525 module_id: "fed".to_string(),
10526 launch_nonce: "fed-nonce".to_string(),
10527 }),
10528 None,
10529 ),
10530 )
10531 .await
10532 .unwrap()
10533 });
10534
10535 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10542 .await
10543 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
10544 .expect("module control channel closed before route.bind");
10545 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10546 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10547 panic!("expected route.bind")
10548 };
10549 assert_eq!(
10550 principal,
10551 Some(Principal::Reserved {
10552 module_id: "fed".to_string()
10553 }),
10554 "a correctly attested consumer must be stamped Reserved for its own id"
10555 );
10556
10557 handler
10558 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10559 .await
10560 .unwrap();
10561 assert!(route_task.await.unwrap().is_empty());
10562 assert!(
10563 matches!(
10564 serde_json::from_slice::<ClientControlResponse>(
10565 &client_rx.recv().await.unwrap().body
10566 )
10567 .unwrap(),
10568 ClientControlResponse::RouteOpen { .. }
10569 ),
10570 "the route must actually open, not merely avoid an error"
10571 );
10572 }
10573
10574 #[tokio::test(start_paused = true)]
10575 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10576 let registry = Arc::new(Registry::default());
10577 let forwarding = Arc::new(ForwardingTable::default());
10578 let supervisor = SupervisorHandle::new();
10579 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10580 let handler =
10581 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10582 .with_supervisor(supervisor);
10583
10584 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10585 hello_via_sink(
10586 &handler,
10587 &target_ctx,
10588 &mut target_rx,
10589 hello_frame("target", PROTOCOL_VERSION, 1),
10590 )
10591 .await;
10592
10593 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10594 let direct_handler = handler.clone();
10595 let direct_open = tokio::spawn(async move {
10596 direct_handler
10597 .handle_control_frame(
10598 &direct_ctx,
10599 route_open_frame(2, "target", unique_project_root("route-census-direct")),
10600 )
10601 .await
10602 .unwrap()
10603 });
10604 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10605 .await
10606 .expect("no direct route.bind within 5s")
10607 .expect("target control channel closed before direct route.bind");
10608 handler
10609 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10610 .await
10611 .unwrap();
10612 assert!(direct_open.await.unwrap().is_empty());
10613 let _ = direct_rx.recv().await.unwrap();
10614
10615 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10616 let reserved_handler = handler.clone();
10617 let reserved_open = tokio::spawn(async move {
10618 reserved_handler
10619 .handle_control_frame(
10620 &reserved_ctx,
10621 route_open_frame_with_admission_facts(
10622 3,
10623 "target",
10624 unique_project_root("admission-facts"),
10625 Some(ConsumerIdentity {
10626 module_id: "fed".to_string(),
10627 launch_nonce: "fed-nonce".to_string(),
10628 }),
10629 None,
10630 ),
10631 )
10632 .await
10633 .unwrap()
10634 });
10635 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10636 .await
10637 .expect("no reserved route.bind within 5s")
10638 .expect("target control channel closed before reserved route.bind");
10639 handler
10640 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10641 .await
10642 .unwrap();
10643 assert!(reserved_open.await.unwrap().is_empty());
10644 let _ = reserved_rx.recv().await.unwrap();
10645
10646 forwarding
10647 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10648 .unwrap();
10649 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
10650 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
10651 module_id: Some("target".to_string()),
10652 })
10653 .unwrap();
10654 let census_frame =
10655 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
10656 let response = handler
10657 .handle_control_frame(&census_ctx, census_frame)
10658 .await
10659 .unwrap()
10660 .pop()
10661 .unwrap();
10662 let actual: Value = serde_json::from_slice(&response.body).unwrap();
10663 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
10664 assert!(matches!(
10665 decoded,
10666 ClientControlResponse::SupervisorRoutes { .. }
10667 ));
10668 let routes = actual["modules"][0]["routes"].as_array().unwrap();
10669 assert_eq!(routes.len(), 2);
10670 assert!(routes.iter().all(|route| route["draining"] == true));
10671 assert!(
10674 routes.iter().all(|route| route["drain_reason"] == "reload"),
10675 "draining routes must name the drain's reason: {routes:?}"
10676 );
10677 assert!(routes.iter().any(|route| {
10678 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
10679 }));
10680 assert!(routes.iter().any(|route| {
10681 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
10682 }));
10683
10684 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10685 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
10686 if std::env::var_os("UPDATE_GOLDEN").is_some() {
10687 std::fs::write(
10688 &golden_path,
10689 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
10690 )
10691 .unwrap();
10692 }
10693 let expected: Value =
10694 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
10695 assert_eq!(actual, expected);
10696 }
10697
10698 async fn query_live_roots(
10699 handler: &ControlHandler,
10700 module_ctx: &RouteCtx,
10701 ) -> ModuleControlResponseToModule {
10702 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
10703 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
10704 let response = handler
10705 .handle_control_frame(module_ctx, frame)
10706 .await
10707 .unwrap()
10708 .pop()
10709 .unwrap();
10710 serde_json::from_slice(&response.body).unwrap()
10711 }
10712
10713 #[tokio::test(start_paused = true)]
10714 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
10715 let registry = Arc::new(Registry::default());
10716 let forwarding = Arc::new(ForwardingTable::default());
10717 let handler = ControlHandler::with_forwarding(registry, forwarding);
10718 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
10719 hello_via_sink(
10720 &handler,
10721 &target_ctx,
10722 &mut target_rx,
10723 hello_frame("target", PROTOCOL_VERSION, 1),
10724 )
10725 .await;
10726 let root = unique_project_root("live-roots-known");
10727 let path = ProjectRootId::from_path_allowing_missing(root.path())
10728 .unwrap()
10729 .as_path()
10730 .to_path_buf();
10731 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
10732 let open_handler = handler.clone();
10733 let opened = tokio::spawn(async move {
10734 open_handler
10735 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
10736 .await
10737 .unwrap()
10738 });
10739 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10740 .await
10741 .unwrap()
10742 .unwrap();
10743 handler
10744 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
10745 .await
10746 .unwrap();
10747 assert!(opened.await.unwrap().is_empty());
10748 let _ = client_rx.recv().await.unwrap();
10749
10750 let root = unique_project_root("live-roots-pending");
10751 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
10752 .unwrap()
10753 .as_path()
10754 .to_path_buf();
10755 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
10756 let open_handler = handler.clone();
10757 let pending = tokio::spawn(async move {
10758 open_handler
10759 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
10760 .await
10761 .unwrap()
10762 });
10763 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10764 .await
10765 .unwrap()
10766 .unwrap();
10767 let actual = query_live_roots(&handler, &target_ctx).await;
10768 let ModuleControlResponseToModule::LiveRoots {
10769 roots,
10770 unknown_root_bindings,
10771 total_bindings,
10772 } = actual
10773 else {
10774 panic!("expected live roots")
10775 };
10776 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
10777 assert_eq!(unknown_root_bindings, 0);
10778 assert_eq!(
10779 roots.len(),
10780 2,
10781 "root-known arm must retain each canonical root"
10782 );
10783 assert_eq!(
10784 total_bindings,
10785 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10786 );
10787 let counts = roots
10788 .iter()
10789 .map(|root| (root.project_root.clone(), root.bound, root.pending))
10790 .collect::<Vec<_>>();
10791 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
10792 expected.sort_by(|a, b| a.0.cmp(&b.0));
10793 assert_eq!(
10794 counts, expected,
10795 "roots must sort by path and count pending separately"
10796 );
10797 handler
10798 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
10799 .await
10800 .unwrap();
10801 assert!(pending.await.unwrap().is_empty());
10802 }
10803
10804 #[tokio::test(start_paused = true)]
10805 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
10806 let forwarding = Arc::new(ForwardingTable::default());
10807 let handler =
10808 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10809 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
10810 hello_via_sink(
10811 &handler,
10812 &target_ctx,
10813 &mut target_rx,
10814 hello_frame("target", PROTOCOL_VERSION, 1),
10815 )
10816 .await;
10817 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
10818 let pending = forwarding
10819 .begin_route_bind_relay_for_test(
10820 client_ctx.connection_id,
10821 client_ctx.egress.clone(),
10822 2,
10823 "target",
10824 )
10825 .unwrap();
10826 forwarding
10827 .complete_pending_relay(
10828 target_ctx.connection_id,
10829 pending.corr,
10830 RouteBindRelayOutcome::Accepted,
10831 )
10832 .unwrap();
10833 let actual = query_live_roots(&handler, &target_ctx).await;
10834 let ModuleControlResponseToModule::LiveRoots {
10835 roots,
10836 unknown_root_bindings,
10837 total_bindings,
10838 } = actual
10839 else {
10840 panic!("expected live roots")
10841 };
10842 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
10843 assert_eq!(
10844 unknown_root_bindings, 1,
10845 "unknown-root arm must not read as no bindings"
10846 );
10847 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
10848 assert_eq!(
10849 total_bindings,
10850 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10851 );
10852 }
10853
10854 #[tokio::test(start_paused = true)]
10862 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
10863 let forwarding = Arc::new(ForwardingTable::default());
10864 let handler =
10865 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10866 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
10867 let replies = handler
10868 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
10869 .await
10870 .unwrap();
10871 let queued_by_hello = module_rx.len();
10872
10873 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
10874 let open_handler = handler.clone();
10875 let open = tokio::spawn(async move {
10876 open_handler
10877 .handle_control_frame(
10878 &client_ctx,
10879 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
10880 )
10881 .await
10882 .unwrap()
10883 });
10884 let mut spins = 0;
10886 while module_rx.len() == queued_by_hello {
10887 spins += 1;
10888 assert!(spins < 10_000, "route.open never queued a route.bind");
10889 tokio::task::yield_now().await;
10890 }
10891
10892 for reply in replies {
10894 module_ctx.egress.send(reply).await.unwrap();
10895 }
10896
10897 let first = module_rx.recv().await.unwrap().frame;
10898 assert_eq!(
10899 first.header.ty,
10900 FrameType::HelloAck,
10901 "the first frame a registering module reads must be its HELLO_ACK"
10902 );
10903 assert_eq!(first.header.corr, 7);
10904 let second = module_rx.recv().await.unwrap().frame;
10905 assert_eq!(second.header.ty, FrameType::Request);
10906 assert!(
10907 matches!(
10908 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
10909 ModuleControlRequest::RouteBind { .. }
10910 ),
10911 "the route.bind follows the ack"
10912 );
10913 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
10914
10915 handler
10916 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
10917 .await
10918 .unwrap();
10919 assert!(open.await.unwrap().is_empty());
10920 let _ = client_rx.recv().await.unwrap();
10921 }
10922
10923 #[tokio::test(start_paused = true)]
10924 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
10925 let handler = ControlHandler::with_forwarding(
10926 Arc::new(Registry::default()),
10927 Arc::new(ForwardingTable::default()),
10928 );
10929 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
10930 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
10931 hello_via_sink(
10932 &handler,
10933 &first_ctx,
10934 &mut first_rx,
10935 hello_frame("first", PROTOCOL_VERSION, 1),
10936 )
10937 .await;
10938 hello_via_sink(
10939 &handler,
10940 &second_ctx,
10941 &mut second_rx,
10942 hello_frame("second", PROTOCOL_VERSION, 2),
10943 )
10944 .await;
10945 let root = unique_project_root("second-only");
10946 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
10947 let cloned = handler.clone();
10948 let open = tokio::spawn(async move {
10949 cloned
10950 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
10951 .await
10952 .unwrap()
10953 });
10954 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
10955 .await
10956 .unwrap()
10957 .unwrap();
10958 let first = query_live_roots(&handler, &first_ctx).await;
10959 let second = query_live_roots(&handler, &second_ctx).await;
10960 assert!(
10961 matches!(
10962 first,
10963 ModuleControlResponseToModule::LiveRoots {
10964 total_bindings: 0,
10965 ..
10966 }
10967 ),
10968 "cross-module scope must not expose another module's roots"
10969 );
10970 assert!(
10971 matches!(
10972 second,
10973 ModuleControlResponseToModule::LiveRoots {
10974 total_bindings: 1,
10975 ..
10976 }
10977 ),
10978 "second module must see its pending route"
10979 );
10980 handler
10981 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
10982 .await
10983 .unwrap();
10984 assert!(open.await.unwrap().is_empty());
10985 }
10986
10987 #[tokio::test(start_paused = true)]
10988 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
10989 let handler = ControlHandler::with_forwarding(
10990 Arc::new(Registry::default()),
10991 Arc::new(ForwardingTable::default()),
10992 );
10993 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
10994 hello_via_sink(
10995 &handler,
10996 &target_ctx,
10997 &mut target_rx,
10998 hello_frame("target", PROTOCOL_VERSION, 1),
10999 )
11000 .await;
11001 let actual = query_live_roots(&handler, &target_ctx).await;
11002 let ModuleControlResponseToModule::LiveRoots {
11003 roots,
11004 unknown_root_bindings,
11005 total_bindings,
11006 } = actual
11007 else {
11008 panic!("expected live roots")
11009 };
11010 assert!(roots.is_empty());
11011 assert_eq!(unknown_root_bindings, 0);
11012 assert_eq!(total_bindings, 0);
11013 assert_eq!(
11014 total_bindings,
11015 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11016 );
11017 }
11018
11019 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
11026 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11027 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
11028 let text = std::fs::read_to_string(&path)
11029 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
11030 let vectors: Vec<(String, Value)> = text
11031 .lines()
11032 .filter(|line| !line.trim().is_empty())
11033 .map(|line| {
11034 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
11035 let id = entry["corpus_id"]
11036 .as_str()
11037 .expect("every vector carries a corpus_id")
11038 .to_string();
11039 (id, entry["package"].clone())
11040 })
11041 .collect();
11042 assert_eq!(
11046 vectors.len(),
11047 3,
11048 "vendored fed corpus changed size; re-sync from subc-federation"
11049 );
11050
11051 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
11066 let richest = vectors
11067 .iter()
11068 .filter_map(|(_, package)| package.as_object())
11069 .map(|object| {
11070 object
11071 .keys()
11072 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
11073 .count()
11074 })
11075 .max()
11076 .unwrap_or(0);
11077 assert!(
11078 richest >= 2,
11079 "vendored corpus no longer carries a package with unmodelled fields, \
11080 so the relay test can no longer distinguish a verbatim relay from a lossy one"
11081 );
11082
11083 vectors
11084 }
11085
11086 #[tokio::test]
11094 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
11095 for (corpus_id, package) in fed_admission_facts_vectors() {
11096 let registry = Arc::new(Registry::default());
11097 let forwarding = Arc::new(ForwardingTable::default());
11098 let supervisor = SupervisorHandle::new();
11099 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11100 let handler =
11101 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11102 .with_supervisor(supervisor)
11103 .with_admission_facts_config(
11104 Some("fed".to_string()),
11105 Some(vec!["target".to_string()]),
11106 );
11107
11108 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11109 hello_via_sink(
11110 &handler,
11111 &target_ctx,
11112 &mut target_rx,
11113 hello_frame("target", PROTOCOL_VERSION, 1),
11114 )
11115 .await;
11116
11117 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11118 let route_handler = handler.clone();
11119 let expected = package.clone();
11120 let route_task = tokio::spawn(async move {
11121 route_handler
11122 .handle_control_frame(
11123 &client_ctx,
11124 route_open_frame_with_admission_facts(
11125 20,
11126 "target",
11127 unique_project_root("admission-facts"),
11128 Some(subc_control::ConsumerIdentity {
11129 module_id: "fed".to_string(),
11130 launch_nonce: "fed-nonce".to_string(),
11131 }),
11132 Some(package),
11133 ),
11134 )
11135 .await
11136 .unwrap()
11137 });
11138
11139 let bind_frame = target_rx.recv().await.unwrap();
11140 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11141 let ModuleControlRequest::RouteBind {
11142 admission_facts, ..
11143 } = bind
11144 else {
11145 panic!("{corpus_id}: expected route.bind")
11146 };
11147 assert_eq!(
11148 admission_facts,
11149 Some(expected),
11150 "{corpus_id}: relay must not add, drop or reshape any field"
11151 );
11152
11153 handler
11154 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11155 .await
11156 .unwrap();
11157 route_task.await.unwrap();
11158 }
11159 }
11160
11161 #[tokio::test]
11162 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11163 let registry = Arc::new(Registry::default());
11164 let forwarding = Arc::new(ForwardingTable::default());
11165 let supervisor = SupervisorHandle::new();
11166 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11167 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11168 let handler =
11169 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11170 .with_supervisor(supervisor)
11171 .with_admission_facts_config(
11172 Some("fed".to_string()),
11173 Some(vec!["target".to_string()]),
11174 );
11175
11176 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11177 hello_via_sink(
11178 &handler,
11179 &target_ctx,
11180 &mut target_rx,
11181 hello_frame("target", PROTOCOL_VERSION, 1),
11182 )
11183 .await;
11184 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11185 hello_via_sink(
11186 &handler,
11187 &other_ctx,
11188 &mut other_rx,
11189 hello_frame("other", PROTOCOL_VERSION, 2),
11190 )
11191 .await;
11192
11193 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11194 let expected_facts = facts.clone();
11195 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11196 let route_handler = handler.clone();
11197 let route_task = tokio::spawn(async move {
11198 route_handler
11199 .handle_control_frame(
11200 &client_ctx,
11201 route_open_frame_with_admission_facts(
11202 10,
11203 "target",
11204 unique_project_root("admission-facts"),
11205 Some(subc_control::ConsumerIdentity {
11206 module_id: "fed".to_string(),
11207 launch_nonce: "fed-nonce".to_string(),
11208 }),
11209 Some(facts.clone()),
11210 ),
11211 )
11212 .await
11213 .unwrap()
11214 });
11215 let bind_frame = target_rx.recv().await.unwrap();
11216 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11217 let ModuleControlRequest::RouteBind {
11218 admission_facts, ..
11219 } = bind
11220 else {
11221 panic!("expected route.bind")
11222 };
11223 assert_eq!(admission_facts, Some(expected_facts));
11224 handler
11225 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11226 .await
11227 .unwrap();
11228 assert!(route_task.await.unwrap().is_empty());
11229 assert!(matches!(
11230 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11231 .unwrap(),
11232 ClientControlResponse::RouteOpen { .. }
11233 ));
11234
11235 let direct = handler
11236 .handle_control_frame(
11237 &route_ctx(ConnectionId::new(73)).0,
11238 route_open_frame_with_admission_facts(
11239 11,
11240 "target",
11241 unique_project_root("admission-facts"),
11242 None,
11243 Some(json!({"x": 1})),
11244 ),
11245 )
11246 .await
11247 .unwrap();
11248 assert_eq!(
11249 parse_error(&direct[0])["code"],
11250 "admission_facts_not_permitted"
11251 );
11252
11253 let different_reserved = handler
11254 .handle_control_frame(
11255 &route_ctx(ConnectionId::new(77)).0,
11256 route_open_frame_with_admission_facts(
11257 15,
11258 "target",
11259 unique_project_root("admission-facts"),
11260 Some(subc_control::ConsumerIdentity {
11261 module_id: "other".to_string(),
11262 launch_nonce: "other-nonce".to_string(),
11263 }),
11264 Some(json!({"x": 1})),
11265 ),
11266 )
11267 .await
11268 .unwrap();
11269 assert_eq!(
11270 parse_error(&different_reserved[0])["code"],
11271 "admission_facts_not_permitted"
11272 );
11273
11274 let other_target = handler
11275 .handle_control_frame(
11276 &route_ctx(ConnectionId::new(74)).0,
11277 route_open_frame_with_admission_facts(
11278 12,
11279 "other",
11280 unique_project_root("admission-facts"),
11281 Some(subc_control::ConsumerIdentity {
11282 module_id: "fed".to_string(),
11283 launch_nonce: "fed-nonce".to_string(),
11284 }),
11285 Some(json!({"x": 1})),
11286 ),
11287 )
11288 .await
11289 .unwrap();
11290 assert_eq!(
11291 parse_error(&other_target[0])["code"],
11292 "admission_facts_target_not_allowed"
11293 );
11294
11295 let nonexistent = handler
11296 .handle_control_frame(
11297 &route_ctx(ConnectionId::new(75)).0,
11298 route_open_frame_with_admission_facts(
11299 13,
11300 "missing",
11301 unique_project_root("admission-facts"),
11302 None,
11303 Some(json!({"x": 1})),
11304 ),
11305 )
11306 .await
11307 .unwrap();
11308 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
11309
11310 let described = handler
11311 .handle_control_frame(
11312 &route_ctx(ConnectionId::new(76)).0,
11313 Frame::build(
11314 FrameType::Request,
11315 control_flags(),
11316 0,
11317 0,
11318 14,
11319 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11320 )
11321 .unwrap(),
11322 )
11323 .await
11324 .unwrap();
11325 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11326 serde_json::from_slice(&described[0].body).unwrap()
11327 else {
11328 panic!("expected server.describe response")
11329 };
11330 assert!(capabilities
11331 .iter()
11332 .any(|cap| cap == "admission_facts_relay_v1"));
11333 }
11334
11335 #[tokio::test]
11336 async fn admission_facts_without_configured_carrier_are_rejected() {
11337 let registry = Arc::new(Registry::default());
11338 let forwarding = Arc::new(ForwardingTable::default());
11339 let handler = ControlHandler::with_forwarding(registry, forwarding);
11340 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
11341 hello_via_sink(
11342 &handler,
11343 &target_ctx,
11344 &mut target_rx,
11345 hello_frame("target", PROTOCOL_VERSION, 1),
11346 )
11347 .await;
11348
11349 let responses = handler
11350 .handle_control_frame(
11351 &route_ctx(ConnectionId::new(79)).0,
11352 route_open_frame_with_admission_facts(
11353 16,
11354 "target",
11355 unique_project_root("admission-facts"),
11356 None,
11357 Some(json!({"x": 1})),
11358 ),
11359 )
11360 .await
11361 .unwrap();
11362 assert_eq!(
11363 parse_error(&responses[0])["code"],
11364 "admission_facts_not_permitted"
11365 );
11366 }
11367
11368 #[tokio::test]
11369 async fn route_open_relays_consumer_capabilities_verbatim() {
11370 let registry = Arc::new(Registry::default());
11371 let forwarding = Arc::new(ForwardingTable::default());
11372 let handler =
11373 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11374 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
11375 hello_via_sink(
11376 &handler,
11377 &module_ctx,
11378 &mut module_rx,
11379 hello_frame("aft", PROTOCOL_VERSION, 7),
11380 )
11381 .await;
11382
11383 let expected = vec!["elicitation".to_string(), "roots".to_string()];
11384 let expected_for_request = expected.clone();
11385 let project_root = unique_project_root("consumer-capabilities-present");
11386 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
11387 let route_handler = handler.clone();
11388 let route_task = tokio::spawn(async move {
11389 route_handler
11390 .handle_control_frame(
11391 &client_ctx,
11392 route_open_frame_with_consumer_capabilities(
11393 401,
11394 "aft",
11395 project_root,
11396 Some(expected_for_request),
11397 ),
11398 )
11399 .await
11400 .unwrap()
11401 });
11402 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11403 .await
11404 .unwrap()
11405 .unwrap();
11406 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11407 let ModuleControlRequest::RouteBind {
11408 consumer_capabilities,
11409 ..
11410 } = bind
11411 else {
11412 panic!("expected route.bind request, got {bind:?}");
11413 };
11414 assert_eq!(consumer_capabilities, Some(expected.clone()));
11415
11416 handler
11417 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11418 .await
11419 .unwrap();
11420 let route_response = route_task.await.unwrap();
11421 assert!(route_response.is_empty());
11422 let published = client_rx.recv().await.unwrap();
11423 assert!(matches!(
11424 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11425 ClientControlResponse::RouteOpen { .. }
11426 ));
11427 }
11428
11429 #[tokio::test]
11430 async fn route_open_without_consumer_capabilities_relays_none() {
11431 let registry = Arc::new(Registry::default());
11432 let forwarding = Arc::new(ForwardingTable::default());
11433 let handler =
11434 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11435 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
11436 hello_via_sink(
11437 &handler,
11438 &module_ctx,
11439 &mut module_rx,
11440 hello_frame("aft", PROTOCOL_VERSION, 7),
11441 )
11442 .await;
11443
11444 let project_root = unique_project_root("consumer-capabilities-absent");
11445 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
11446 let route_handler = handler.clone();
11447 let route_task = tokio::spawn(async move {
11448 route_handler
11449 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
11450 .await
11451 .unwrap()
11452 });
11453 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11454 .await
11455 .unwrap()
11456 .unwrap();
11457 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11458 let ModuleControlRequest::RouteBind {
11459 consumer_capabilities,
11460 ..
11461 } = bind
11462 else {
11463 panic!("expected route.bind request, got {bind:?}");
11464 };
11465 assert_eq!(consumer_capabilities, None);
11466
11467 handler
11468 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11469 .await
11470 .unwrap();
11471 let route_response = route_task.await.unwrap();
11472 assert!(route_response.is_empty());
11473 let published = client_rx.recv().await.unwrap();
11474 assert!(matches!(
11475 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11476 ClientControlResponse::RouteOpen { .. }
11477 ));
11478 }
11479
11480 async fn bind_role_versions_for(
11484 sent: Option<BTreeMap<String, String>>,
11485 connection: u64,
11486 ) -> Option<BTreeMap<String, String>> {
11487 let registry = Arc::new(Registry::default());
11488 let forwarding = Arc::new(ForwardingTable::default());
11489 let handler =
11490 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11491 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
11492 hello_via_sink(
11493 &handler,
11494 &module_ctx,
11495 &mut module_rx,
11496 hello_frame("aft", PROTOCOL_VERSION, 7),
11497 )
11498 .await;
11499 let project_root = unique_project_root("role-versions");
11500 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
11501 let route_handler = handler.clone();
11502 let route_task = tokio::spawn(async move {
11503 route_handler
11504 .handle_control_frame(
11505 &client_ctx,
11506 route_open_frame_with_role_versions(403, "aft", project_root, sent),
11507 )
11508 .await
11509 .unwrap()
11510 });
11511 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11512 .await
11513 .expect("a well-formed route.open reaches the module as a bind")
11514 .unwrap();
11515 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11516 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
11517 panic!("expected route.bind request, got {bind:?}");
11518 };
11519 handler
11520 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11521 .await
11522 .unwrap();
11523 assert!(route_task.await.unwrap().is_empty());
11524 let published = client_rx.recv().await.unwrap();
11525 assert!(matches!(
11526 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11527 ClientControlResponse::RouteOpen { .. }
11528 ));
11529 role_versions
11530 }
11531
11532 #[tokio::test]
11533 async fn route_open_relays_role_versions_verbatim() {
11534 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
11535 assert_eq!(
11536 bind_role_versions_for(Some(sent.clone()), 141).await,
11537 Some(sent)
11538 );
11539 }
11540
11541 #[tokio::test]
11544 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
11545 assert_eq!(bind_role_versions_for(None, 143).await, None);
11546 assert_eq!(
11547 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
11548 None
11549 );
11550 }
11551
11552 #[tokio::test]
11553 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
11554 let registry = Arc::new(Registry::default());
11555 let forwarding = Arc::new(ForwardingTable::default());
11556 let handler =
11557 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11558 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
11559 hello_via_sink(
11560 &handler,
11561 &module_ctx,
11562 &mut module_rx,
11563 hello_frame("aft", PROTOCOL_VERSION, 7),
11564 )
11565 .await;
11566
11567 let nine: BTreeMap<String, String> = (0..9)
11568 .map(|index| (format!("role-{index}"), "v1".to_string()))
11569 .collect();
11570 for (label, malformed) in [
11571 (
11572 "invalid role name",
11573 role_versions(&[("Tool_Provider", "v1")]),
11574 ),
11575 ("invalid version", role_versions(&[("tool-provider", "v0")])),
11576 ("nine entries", nine),
11577 ] {
11578 let responses = tokio::time::timeout(
11581 Duration::from_secs(1),
11582 handler.handle_control_frame(
11583 &route_ctx(ConnectionId::new(148)).0,
11584 route_open_frame_with_role_versions(
11585 404,
11586 "aft",
11587 unique_project_root("role-versions-malformed"),
11588 Some(malformed),
11589 ),
11590 ),
11591 )
11592 .await
11593 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11594 .unwrap();
11595 assert_eq!(responses.len(), 1, "{label}");
11596 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11597 let error = parse_error(&responses[0]);
11598 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11599 assert_eq!(
11600 error["detail"]["field"], "role_versions",
11601 "{label}: {error}"
11602 );
11603 assert!(
11604 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11605 "{label}: a malformed declaration is terminal"
11606 );
11607 assert!(
11608 module_rx.try_recv().is_err(),
11609 "{label}: the module must never see a bind"
11610 );
11611 }
11612 }
11613
11614 #[tokio::test]
11618 async fn route_role_versions_capability_is_advertised() {
11619 let handler = ControlHandler::new(Arc::new(Registry::default()));
11620 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11621 let ack = hello_via_sink(
11622 &handler,
11623 &ctx,
11624 &mut rx,
11625 hello_frame("m", PROTOCOL_VERSION, 1),
11626 )
11627 .await;
11628 let ack = parse_ack(&ack);
11629 assert!(
11630 ack.subc_capabilities
11631 .iter()
11632 .any(|c| c == "route-role-versions/v1"),
11633 "{:?}",
11634 ack.subc_capabilities
11635 );
11636
11637 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11638 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11639 let reply = handler
11640 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11641 .await
11642 .unwrap()
11643 .pop()
11644 .unwrap();
11645 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11646 serde_json::from_slice(&reply.body).unwrap()
11647 else {
11648 panic!("not a server.describe reply");
11649 };
11650 assert!(
11651 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
11652 "{capabilities:?}"
11653 );
11654 }
11655
11656 #[tokio::test]
11660 async fn catalog_events_capability_is_advertised() {
11661 let handler = ControlHandler::new(Arc::new(Registry::default()));
11662 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11663 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11664 let reply = handler
11665 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
11666 .await
11667 .unwrap()
11668 .pop()
11669 .unwrap();
11670 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11671 serde_json::from_slice(&reply.body).unwrap()
11672 else {
11673 panic!("not a server.describe reply");
11674 };
11675 assert!(
11676 capabilities
11677 .iter()
11678 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
11679 "{capabilities:?}"
11680 );
11681 }
11682
11683 #[tokio::test]
11684 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
11685 let handler = ControlHandler::new(Arc::new(Registry::default()));
11686 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
11687 let ack = hello_via_sink(
11688 &handler,
11689 &module,
11690 &mut module_rx,
11691 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
11692 )
11693 .await;
11694 let ack = parse_ack(&ack);
11695 assert!(
11696 ack.subc_capabilities
11697 .iter()
11698 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11699 "{:?}",
11700 ack.subc_capabilities
11701 );
11702 assert!(ack
11703 .subc_capabilities
11704 .iter()
11705 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11706
11707 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
11708 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11709 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11710 let reply = handler
11711 .handle_control_frame(&client, frame)
11712 .await
11713 .unwrap()
11714 .pop()
11715 .unwrap();
11716 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11717 serde_json::from_slice(&reply.body).unwrap()
11718 else {
11719 panic!("not a server.describe reply");
11720 };
11721 assert!(
11722 capabilities
11723 .iter()
11724 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11725 "{capabilities:?}"
11726 );
11727 assert!(capabilities
11728 .iter()
11729 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11730 }
11731
11732 #[tokio::test]
11733 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
11734 let handler = ControlHandler::new(Arc::new(Registry::default()));
11735 let (module, mut rx) = route_ctx(ConnectionId::new(156));
11736 let ack = hello_via_sink(
11737 &handler,
11738 &module,
11739 &mut rx,
11740 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
11741 )
11742 .await;
11743 assert!(parse_ack(&ack)
11744 .subc_capabilities
11745 .iter()
11746 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
11747 assert!(parse_ack(&ack)
11748 .subc_capabilities
11749 .iter()
11750 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
11751 let (client, _rx) = route_ctx(ConnectionId::new(157));
11752 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11753 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11754 let reply = handler
11755 .handle_control_frame(&client, frame)
11756 .await
11757 .unwrap()
11758 .pop()
11759 .unwrap();
11760 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11761 serde_json::from_slice(&reply.body).unwrap()
11762 else {
11763 panic!("server.describe reply")
11764 };
11765 assert!(capabilities
11766 .iter()
11767 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
11768 assert!(capabilities
11769 .iter()
11770 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
11771 }
11772
11773 #[tokio::test]
11774 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
11775 let handler = ControlHandler::new(Arc::new(Registry::default()));
11776 let (module, mut rx) = route_ctx(ConnectionId::new(154));
11777 let ack = hello_via_sink(
11778 &handler,
11779 &module,
11780 &mut rx,
11781 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
11782 )
11783 .await;
11784 assert!(parse_ack(&ack)
11785 .subc_capabilities
11786 .iter()
11787 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11788 let (client, _rx) = route_ctx(ConnectionId::new(155));
11789 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11790 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11791 let reply = handler
11792 .handle_control_frame(&client, frame)
11793 .await
11794 .unwrap()
11795 .pop()
11796 .unwrap();
11797 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11798 serde_json::from_slice(&reply.body).unwrap()
11799 else {
11800 panic!("not a server.describe reply")
11801 };
11802 assert!(capabilities
11803 .iter()
11804 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11805 }
11806
11807 #[tokio::test]
11808 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
11809 let registry = Arc::new(Registry::default());
11810 let forwarding = Arc::new(ForwardingTable::default());
11811 let handler =
11812 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11813 .with_health_probe_timeout(Duration::from_secs(5));
11814 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
11815 hello_via_sink(
11816 &handler,
11817 &module_ctx,
11818 &mut module_rx,
11819 non_routable_hello_frame_with_control_ops(
11820 "mcp",
11821 300,
11822 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11823 ),
11824 )
11825 .await;
11826 assert!(registry
11827 .get_module("mcp")
11828 .unwrap()
11829 .unwrap()
11830 .manifest
11831 .provides
11832 .is_empty());
11833
11834 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
11835 let route_response = handler
11836 .handle_control_frame(
11837 &route_client_ctx,
11838 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
11839 )
11840 .await
11841 .unwrap();
11842 assert_eq!(route_response[0].header.ty, FrameType::Error);
11843 assert_eq!(
11844 parse_error(&route_response[0])["code"],
11845 "target_unavailable"
11846 );
11847 assert!(parse_error(&route_response[0])["message"]
11848 .as_str()
11849 .unwrap()
11850 .contains("does not provide the requested target"));
11851 assert!(module_rx.try_recv().is_err());
11852
11853 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
11854 let health_handler = handler.clone();
11855 let health_task = tokio::spawn(async move {
11856 health_handler
11857 .handle_control_frame(
11858 &health_client_ctx,
11859 supervisor_health_probe_frame(302, "mcp"),
11860 )
11861 .await
11862 .unwrap()
11863 });
11864 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11865 .await
11866 .unwrap()
11867 .unwrap();
11868 assert_eq!(
11869 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11870 ModuleControlRequest::HealthCheck {}
11871 );
11872 handler
11873 .handle_control_frame(
11874 &module_ctx,
11875 health_response(health_frame.header.corr, HealthStatus::Ok),
11876 )
11877 .await
11878 .unwrap();
11879 let health_response = health_task.await.unwrap();
11880 assert_eq!(health_response[0].header.ty, FrameType::Response);
11881 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11882 ClientControlResponse::SupervisorHealthProbe {
11883 module_id, status, ..
11884 } => {
11885 assert_eq!(module_id, "mcp");
11886 assert_eq!(status, HealthStatus::Ok);
11887 }
11888 other => panic!("unexpected health response: {other:?}"),
11889 }
11890
11891 forwarding
11896 .cleanup_connection(module_ctx.connection_id)
11897 .unwrap();
11898 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
11899 let cleanup_response = tokio::time::timeout(
11900 Duration::from_millis(200),
11901 handler.handle_control_frame(
11902 &cleanup_probe_ctx,
11903 supervisor_health_probe_frame(303, "mcp"),
11904 ),
11905 )
11906 .await
11907 .expect("probe should fail immediately when the control lane is gone")
11908 .unwrap();
11909 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
11910 assert_eq!(
11911 parse_error(&cleanup_response[0])["code"],
11912 "target_unavailable"
11913 );
11914 assert!(parse_error(&cleanup_response[0])["message"]
11915 .as_str()
11916 .unwrap()
11917 .contains("no module connection"));
11918
11919 handler
11920 .cleanup_connection(module_ctx.connection_id)
11921 .unwrap();
11922 }
11923
11924 #[tokio::test]
11925 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
11926 let registry = Arc::new(Registry::default());
11927 let supervisor_handle = SupervisorHandle::new();
11928 let supervisor =
11929 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11930 .with_handle(supervisor_handle.clone())
11931 .with_connection_file_path(
11932 std::env::temp_dir()
11933 .join(format!("subc-route-open-warming-{}", std::process::id())),
11934 );
11935 let module = supervisor
11936 .supervise_configured(
11937 ModuleSpec {
11938 module_id: "warming".to_string(),
11939 program: fake_aft_stub_path(),
11940 args: Vec::new(),
11941 env: Vec::new(),
11942 reserved: false,
11943 reserved_prefixes: Vec::new(),
11944 protocol: ModuleProtocol::Subc,
11945 overlap: Default::default(),
11946 },
11947 true,
11948 )
11949 .unwrap();
11950 assert_eq!(module.state().unwrap(), ModuleState::Running);
11951
11952 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11953 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
11954 let response = handler
11955 .handle_control_frame(
11956 &ctx,
11957 route_open_frame(304, "warming", unique_project_root("warming")),
11958 )
11959 .await
11960 .unwrap();
11961 module.stop().await.unwrap();
11962
11963 assert_eq!(response[0].header.ty, FrameType::Error);
11964 let error = parse_error(&response[0]);
11965 assert_eq!(error["code"], "module_warming");
11966 assert!(error["message"]
11967 .as_str()
11968 .unwrap()
11969 .contains("state=running, enabled=true, live=false"));
11970 }
11971
11972 #[test]
11973 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
11974 let handler = ControlHandler::new(Arc::new(Registry::default()));
11975 let capture = EventCapture::default();
11976 let _subscriber =
11977 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11978 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
11979 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
11980 let pending = (0..limit).collect::<Vec<_>>();
11981 let response = handler
11982 .route_open_capacity_refusal(
11983 &ctx,
11984 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
11985 "busy",
11986 pending.len(),
11987 limit,
11988 )
11989 .unwrap();
11990 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11991 let event = capture
11992 .events()
11993 .into_iter()
11994 .find(|event| {
11995 event.target == "control"
11996 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
11997 })
11998 .expect("connection admission refusal event");
11999 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12000 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12001 }
12002
12003 #[test]
12004 fn route_open_target_cap_logs_admission_reason_and_capacity() {
12005 let handler = ControlHandler::new(Arc::new(Registry::default()));
12006 let capture = EventCapture::default();
12007 let _subscriber =
12008 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12009 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
12010 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
12011 let guards = (0..limit)
12012 .map(|_| {
12013 handler
12014 .route_bind_concurrency
12015 .try_admit("busy", limit)
12016 .unwrap()
12017 })
12018 .collect::<Vec<_>>();
12019 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
12020 Err(in_flight) => in_flight,
12021 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
12022 };
12023 let response = handler
12024 .route_open_target_capacity_refusal(
12025 &ctx,
12026 &route_open_frame(397, "busy", unique_project_root("target-cap")),
12027 "busy",
12028 in_flight,
12029 )
12030 .unwrap();
12031 assert_eq!(parse_error(&response)["code"], "target_unavailable");
12032 let event = capture
12033 .events()
12034 .into_iter()
12035 .find(|event| {
12036 event.target == "control"
12037 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
12038 })
12039 .expect("target admission refusal event");
12040 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
12041 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
12042 drop(guards);
12043 }
12044
12045 #[tokio::test(flavor = "current_thread")]
12046 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
12047 let handler = ControlHandler::new(Arc::new(Registry::default()));
12048 let capture = EventCapture::default();
12049 let _subscriber =
12050 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12051 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
12052 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12053
12054 let _ = handler
12055 .handle_client_control_request(
12056 &ctx,
12057 frame,
12058 ClientControlRequest::SupervisorSetEnabled {
12059 module_id: "broca".to_string(),
12060 enabled: false,
12061 },
12062 &mut None,
12063 )
12064 .await
12065 .unwrap();
12066
12067 let events = capture
12068 .events()
12069 .into_iter()
12070 .filter(|event| {
12071 event.target == "control"
12072 && event.fields.get("message").map(String::as_str)
12073 == Some("supervisor request received")
12074 })
12075 .collect::<Vec<_>>();
12076 assert_eq!(events.len(), 1, "one supervisor request log line");
12077 let event = &events[0];
12078 assert_eq!(event.level, tracing::Level::INFO);
12079 assert_eq!(
12080 event.fields.get("op"),
12081 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
12082 );
12083 assert_eq!(
12084 event.fields.get("module_id"),
12085 Some(&"\"broca\"".to_string())
12086 );
12087 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
12088 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
12089 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
12090 }
12091
12092 #[tokio::test(flavor = "current_thread")]
12093 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
12094 let registry = Arc::new(Registry::default());
12095 let module_connection = ConnectionId::new(303);
12096 registry
12097 .register_with_control_ops(
12098 manifest("aft", PROTOCOL_VERSION),
12099 PROTOCOL_VERSION,
12100 module_connection,
12101 module_baseline_control_ops(),
12102 )
12103 .unwrap();
12104 let handler = ControlHandler::new(registry);
12105 let capture = EventCapture::default();
12106 let _subscriber =
12107 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12108 let (ctx, _rx) = route_ctx(module_connection);
12109 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
12110
12111 let _ = handler
12112 .handle_client_control_request(
12113 &ctx,
12114 frame,
12115 ClientControlRequest::SupervisorSetEnabled {
12116 module_id: "broca".to_string(),
12117 enabled: false,
12118 },
12119 &mut None,
12120 )
12121 .await
12122 .unwrap();
12123
12124 let callers = capture
12125 .events()
12126 .into_iter()
12127 .filter(|event| {
12128 event.fields.get("message").map(String::as_str)
12129 == Some("supervisor request received")
12130 })
12131 .map(|event| event.fields.get("caller").cloned())
12132 .collect::<Vec<_>>();
12133 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
12134 }
12135
12136 #[tokio::test(flavor = "current_thread")]
12137 async fn supervisor_list_does_not_log_request_received() {
12138 let handler = ControlHandler::new(Arc::new(Registry::default()));
12139 let capture = EventCapture::default();
12140 let _subscriber =
12141 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12142 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
12143 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12144
12145 handler
12146 .handle_client_control_request(
12147 &ctx,
12148 frame,
12149 ClientControlRequest::SupervisorList {},
12150 &mut None,
12151 )
12152 .await
12153 .unwrap();
12154
12155 assert!(capture.events().into_iter().all(|event| {
12156 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12157 }));
12158 }
12159
12160 #[tokio::test(flavor = "current_thread")]
12161 async fn supervisor_rescan_preview_does_not_log_request_received() {
12162 let handler = ControlHandler::new(Arc::new(Registry::default()));
12163 let capture = EventCapture::default();
12164 let _subscriber =
12165 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12166 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
12167 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12168
12169 handler
12170 .handle_client_control_request(
12171 &ctx,
12172 frame,
12173 ClientControlRequest::SupervisorRescan { preview: true },
12174 &mut None,
12175 )
12176 .await
12177 .unwrap();
12178
12179 assert!(capture.events().into_iter().all(|event| {
12180 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12181 }));
12182 }
12183
12184 struct ApproveOperator;
12185 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
12186 fn prompt(
12187 &self,
12188 _: &str,
12189 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
12190 ) -> crate::operator_confirm::ProviderResult {
12191 crate::operator_confirm::ProviderResult::Approved
12192 }
12193 }
12194
12195 #[tokio::test]
12196 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
12197 assert_eq!(
12198 ControlHandler::default().privileged_operator_prompt,
12199 PrivilegedOperatorPrompt::Log
12200 );
12201 for (initial, changed, running) in [
12202 ("log", "enforce", PrivilegedOperatorPrompt::Log),
12203 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
12204 ] {
12205 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
12206 let path = dir.join("subc.jsonc");
12207 std::fs::write(
12208 &path,
12209 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
12210 )
12211 .unwrap();
12212 let config = crate::daemon_config::load(&path).unwrap().unwrap();
12213 let registry = Arc::new(Registry::default());
12214 let handle = SupervisorHandle::new();
12215 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
12216 .with_handle(handle.clone());
12217 let handler = ControlHandler::new(registry)
12218 .with_supervisor(handle)
12219 .with_privileged_operator_prompt(config.privileged_operator_prompt)
12220 .with_supervisor_rescan(supervisor, &path, config.port);
12221 handler.forwarding.operator_confirms().configure(
12222 Arc::new(ApproveOperator),
12223 crate::operator_confirm::OperatorLimits::default(),
12224 );
12225 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
12226
12227 for (disk_mode, needs_restart) in [
12228 (Some(initial), false),
12229 (Some(changed), true),
12230 (Some(changed), true),
12231 (None, running == PrivilegedOperatorPrompt::Enforce),
12232 (Some(initial), false),
12233 ] {
12234 let mut doc = json!({"version":1});
12235 if let Some(mode) = disk_mode {
12236 doc["privileged_operator_prompt"] = json!(mode);
12237 }
12238 std::fs::write(&path, doc.to_string()).unwrap();
12239 for preview in [true, false] {
12240 let request = ClientControlRequest::SupervisorRescan { preview };
12241 let frame = Frame::build(
12242 FrameType::Request,
12243 control_flags(),
12244 0,
12245 0,
12246 1,
12247 serde_json::to_vec(&request).unwrap(),
12248 )
12249 .unwrap();
12250 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12251 assert_eq!(frames.len(), 1);
12252 let ClientControlResponse::SupervisorRescan { result } =
12253 serde_json::from_slice(&frames[0].body).unwrap()
12254 else {
12255 panic!("expected a rescan response");
12256 };
12257 let expected = if needs_restart {
12258 vec!["privileged_operator_prompt"]
12259 } else {
12260 Vec::new()
12261 };
12262 assert_eq!(result.restart_required, expected);
12263 assert_eq!(handler.privileged_operator_prompt, running);
12264 assert_eq!(
12265 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
12266 running
12267 );
12268 }
12269 }
12270 }
12271 }
12272
12273 #[tokio::test]
12274 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
12275 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
12276 let path = dir.join("subc.jsonc");
12277 let registry = Arc::new(Registry::default());
12278 let handle = SupervisorHandle::new();
12279 let supervisor =
12280 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
12281 let handler = ControlHandler::new(registry)
12282 .with_supervisor(handle)
12283 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
12284 .with_supervisor_rescan(supervisor, &path, None);
12285 handler.forwarding.operator_confirms().configure(
12286 Arc::new(ApproveOperator),
12287 crate::operator_confirm::OperatorLimits::default(),
12288 );
12289 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
12290 for preview in [true, false] {
12291 std::fs::write(
12292 &path,
12293 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
12294 )
12295 .unwrap();
12296 let request = ClientControlRequest::SupervisorRescan { preview };
12297 let frame = Frame::build(
12298 FrameType::Request,
12299 control_flags(),
12300 0,
12301 0,
12302 1,
12303 serde_json::to_vec(&request).unwrap(),
12304 )
12305 .unwrap();
12306 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12307 assert_eq!(frames.len(), 1);
12308 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
12309 assert_eq!(
12310 handler.privileged_operator_prompt,
12311 PrivilegedOperatorPrompt::Enforce
12312 );
12313 }
12314 }
12315
12316 #[tokio::test]
12321 async fn route_open_refusal_names_the_check_that_refused() {
12322 let handler = ControlHandler::new(Arc::new(Registry::default()));
12323 let capture = EventCapture::default();
12324 let _subscriber =
12325 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12326 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12327 let response = handler
12328 .handle_control_frame(
12329 &ctx,
12330 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
12331 )
12332 .await
12333 .unwrap();
12334
12335 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12336 let event = capture
12337 .events()
12338 .into_iter()
12339 .find(|event| {
12340 event.target == "control"
12341 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12342 })
12343 .expect("route.open refusal event");
12344 assert_eq!(
12345 event.fields.get("reason"),
12346 Some(&"\"not_registered\"".to_string())
12347 );
12348 }
12349
12350 #[tokio::test]
12351 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
12352 let registry = Arc::new(Registry::default());
12353 let supervisor_handle = SupervisorHandle::new();
12354 let supervisor =
12355 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12356 .with_handle(supervisor_handle.clone())
12357 .with_connection_file_path(std::env::temp_dir().join(format!(
12358 "subc-route-open-refusal-info-{}",
12359 std::process::id()
12360 )));
12361 let module = supervisor
12362 .supervise_configured(
12363 ModuleSpec {
12364 module_id: "warming".to_string(),
12365 program: fake_aft_stub_path(),
12366 args: Vec::new(),
12367 env: Vec::new(),
12368 reserved: false,
12369 reserved_prefixes: Vec::new(),
12370 protocol: ModuleProtocol::Subc,
12371 overlap: Default::default(),
12372 },
12373 true,
12374 )
12375 .unwrap();
12376 assert_eq!(module.state().unwrap(), ModuleState::Running);
12377
12378 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12379 assert!(handler
12380 .counters()
12381 .snapshot()
12382 .get("route_open_refused_by_code")
12383 .is_none());
12384 let capture = EventCapture::default();
12385 let _subscriber =
12386 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12387 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
12388 let response = handler
12389 .handle_control_frame(
12390 &ctx,
12391 route_open_frame(394, "warming", unique_project_root("refusal-info")),
12392 )
12393 .await
12394 .unwrap();
12395 module.stop().await.unwrap();
12396
12397 assert_eq!(parse_error(&response[0])["code"], "module_warming");
12398 let event = capture
12399 .events()
12400 .into_iter()
12401 .find(|event| {
12402 event.target == "control"
12403 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
12404 })
12405 .expect("route.open refusal event");
12406 assert_eq!(
12407 event.fields.get("module_id"),
12408 Some(&"\"warming\"".to_string())
12409 );
12410 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
12411 assert_eq!(
12412 event.fields.get("reason"),
12413 Some(&"\"supervised_not_registered\"".to_string())
12414 );
12415 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
12416 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
12417 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
12418 assert_eq!(
12419 handler.counters().snapshot()["route_open_refused_by_code"],
12420 json!({ "module_warming": 1 })
12421 );
12422 }
12423
12424 const OUTAGE_START: &str = "route.open refusing module: not serving";
12425 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
12426
12427 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
12428 capture
12429 .events()
12430 .into_iter()
12431 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
12432 .collect()
12433 }
12434
12435 fn supervise_stub(
12436 registry: &Arc<Registry>,
12437 module_id: &str,
12438 enabled: bool,
12439 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
12440 let supervisor_handle = SupervisorHandle::new();
12441 let supervisor =
12442 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
12443 .with_handle(supervisor_handle.clone())
12444 .with_connection_file_path(std::env::temp_dir().join(format!(
12445 "subc-route-outage-{module_id}-{}",
12446 std::process::id()
12447 )));
12448 let module = supervisor
12449 .supervise_configured(
12450 ModuleSpec {
12451 module_id: module_id.to_string(),
12452 program: fake_aft_stub_path(),
12453 args: Vec::new(),
12454 env: Vec::new(),
12455 reserved: false,
12456 reserved_prefixes: Vec::new(),
12457 protocol: ModuleProtocol::Subc,
12458 overlap: Default::default(),
12459 },
12460 enabled,
12461 )
12462 .unwrap();
12463 (supervisor_handle, module)
12464 }
12465
12466 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
12467 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
12468 module_id: module_id.to_string(),
12469 drain_timeout_ms: Some(50),
12470 })
12471 .unwrap();
12472 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
12473 }
12474
12475 #[test]
12479 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
12480 let registry = Arc::new(Registry::default());
12481 let forwarding = Arc::new(ForwardingTable::default());
12482 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12483 let second = ControlHandler::with_forwarding(registry, forwarding);
12484 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
12485 }
12486
12487 #[tokio::test(flavor = "current_thread")]
12491 async fn route_open_unknown_module_refusals_add_no_outage_state() {
12492 let handler = ControlHandler::new(Arc::new(Registry::default()));
12493 let capture = EventCapture::default();
12494 let _subscriber =
12495 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12496 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
12497 for corr in 0..8 {
12498 let response = handler
12499 .handle_control_frame(
12500 &ctx,
12501 route_open_frame(
12502 380 + corr,
12503 &format!("nobody-{corr}"),
12504 unique_project_root("outage-unknown"),
12505 ),
12506 )
12507 .await
12508 .unwrap();
12509 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12510 }
12511
12512 assert_eq!(handler.route_outages.tracked_module_count(), 0);
12513 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
12514 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
12515 }
12516
12517 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12522 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
12523 let registry = Arc::new(Registry::default());
12524 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
12525 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12526 let capture = EventCapture::default();
12527 let _subscriber =
12528 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12529 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
12530 handler.route_outages.record_accepted("outage-restart");
12533
12534 let response = handler
12535 .handle_control_frame(
12536 &ctx,
12537 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
12538 )
12539 .await
12540 .unwrap();
12541 assert_eq!(response[0].header.ty, FrameType::Error);
12542 let starts = outage_lines(&capture, OUTAGE_START);
12543 assert_eq!(starts.len(), 1, "{starts:?}");
12544 assert_eq!(starts[0].level, tracing::Level::WARN);
12545 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
12546 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
12547 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
12548 handler.route_outages.record_accepted("outage-restart");
12549 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
12550
12551 let restart = handler
12552 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
12553 .await
12554 .unwrap();
12555 assert_eq!(
12556 restart[0].header.ty,
12557 FrameType::Response,
12558 "{:?}",
12559 parse_error(&restart[0])
12560 );
12561 handler
12562 .handle_control_frame(
12563 &ctx,
12564 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
12565 )
12566 .await
12567 .unwrap();
12568 module.stop().await.unwrap();
12569
12570 let starts = outage_lines(&capture, OUTAGE_START);
12571 assert_eq!(starts.len(), 2, "{starts:?}");
12572 assert_eq!(starts[1].level, tracing::Level::INFO);
12573 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
12574 }
12575
12576 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12580 async fn failed_operator_restart_leaves_no_operator_mark() {
12581 let registry = Arc::new(Registry::default());
12582 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
12583 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12584 let capture = EventCapture::default();
12585 let _subscriber =
12586 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12587 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
12588 handler.route_outages.record_accepted("outage-disabled");
12589
12590 let restart = handler
12591 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
12592 .await
12593 .unwrap();
12594 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
12595 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
12596
12597 handler
12598 .handle_control_frame(
12599 &ctx,
12600 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
12601 )
12602 .await
12603 .unwrap();
12604 let starts = outage_lines(&capture, OUTAGE_START);
12605 assert_eq!(starts.len(), 1, "{starts:?}");
12606 assert_eq!(starts[0].level, tracing::Level::WARN);
12607 }
12608
12609 #[tokio::test(flavor = "current_thread")]
12610 async fn route_open_unknown_module_escapes_target_module_id() {
12611 let handler = ControlHandler::new(Arc::new(Registry::default()));
12612 let capture = EventCapture::default();
12613 let _subscriber =
12614 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12615 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
12616 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12617 let response = handler
12618 .handle_control_frame(
12619 &ctx,
12620 route_open_frame(
12621 395,
12622 hostile_module_id,
12623 unique_project_root("hostile-target-module-id"),
12624 ),
12625 )
12626 .await
12627 .unwrap();
12628
12629 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12630 let event = capture
12631 .events()
12632 .into_iter()
12633 .find(|event| {
12634 event.target == "control"
12635 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12636 })
12637 .expect("route.open unknown-module refusal event");
12638 let logged = event.fields.get("module_id").expect("module_id field");
12639 assert!(!logged.bytes().any(|byte| byte < 0x20));
12640 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12641 }
12642
12643 #[tokio::test(flavor = "current_thread")]
12644 async fn route_open_module_rejection_uses_daemon_counter_key() {
12645 let registry = Arc::new(Registry::default());
12646 let forwarding = Arc::new(ForwardingTable::default());
12647 let handler =
12648 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12649 let module_connection = ConnectionId::new(95);
12650 let (module_ctx, mut module_rx) = route_ctx(module_connection);
12651 hello_via_sink(
12652 &handler,
12653 &module_ctx,
12654 &mut module_rx,
12655 hello_frame("aft", PROTOCOL_VERSION, 395),
12656 )
12657 .await;
12658
12659 let client_connection = ConnectionId::new(96);
12660 let (client_ctx, _client_rx) = route_ctx(client_connection);
12661 let capture = EventCapture::default();
12662 let _subscriber =
12663 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12664 let (route_task, bind) = relay_route_open(
12665 &handler,
12666 client_connection,
12667 &client_ctx.egress,
12668 &mut module_rx,
12669 396,
12670 "aft",
12671 "hostile-module-code",
12672 )
12673 .await;
12674 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
12675 let rejection = Frame::build(
12676 FrameType::Error,
12677 control_flags(),
12678 0,
12679 0,
12680 bind.header.corr,
12681 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
12682 )
12683 .unwrap();
12684 handler
12685 .handle_control_frame(&module_ctx, rejection)
12686 .await
12687 .unwrap();
12688
12689 let response = route_task.await.unwrap();
12690 assert_eq!(parse_error(&response[0])["code"], hostile_code);
12691 let counters = handler.counters().snapshot();
12692 assert_eq!(
12693 counters["route_open_refused_by_code"],
12694 json!({ "module_rejected": 1 })
12695 );
12696 assert!(counters["route_open_refused_by_code"]
12697 .get(hostile_code)
12698 .is_none());
12699
12700 let event = capture
12701 .events()
12702 .into_iter()
12703 .find(|event| {
12704 event.target == "control"
12705 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
12706 })
12707 .expect("route.open module-rejection refusal event");
12708 let logged = event.fields.get("module_code").expect("module_code field");
12709 assert!(!logged.bytes().any(|byte| byte < 0x20));
12710 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12711 }
12712
12713 #[tokio::test]
12714 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
12715 let registry = Arc::new(Registry::default());
12716 let supervisor_handle = SupervisorHandle::new();
12717 let missing_program = std::env::temp_dir().join(format!(
12718 "subc-route-open-missing-program-{}",
12719 std::process::id()
12720 ));
12721 let supervisor =
12722 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12723 .with_handle(supervisor_handle.clone());
12724 let module = supervisor
12725 .supervise_configured(
12726 ModuleSpec {
12727 module_id: "failed".to_string(),
12728 program: missing_program,
12729 args: Vec::new(),
12730 env: Vec::new(),
12731 reserved: false,
12732 reserved_prefixes: Vec::new(),
12733 protocol: ModuleProtocol::Subc,
12734 overlap: Default::default(),
12735 },
12736 true,
12737 )
12738 .unwrap();
12739 assert_eq!(module.state().unwrap(), ModuleState::Failed);
12740
12741 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12742 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
12743 let response = handler
12744 .handle_control_frame(
12745 &ctx,
12746 route_open_frame(305, "failed", unique_project_root("failed")),
12747 )
12748 .await
12749 .unwrap();
12750
12751 assert_eq!(response[0].header.ty, FrameType::Error);
12752 let error = parse_error(&response[0]);
12753 assert_eq!(error["code"], "target_unavailable");
12754 assert!(error["message"]
12755 .as_str()
12756 .unwrap()
12757 .contains("state=failed, enabled=true, live=false"));
12758 }
12759
12760 #[tokio::test]
12761 async fn route_open_role_mismatch_remains_target_unavailable() {
12762 let registry = Arc::new(Registry::default());
12763 let handler = ControlHandler::new(Arc::clone(®istry));
12764 handler
12765 .handle_control(
12766 ConnectionId::new(41),
12767 non_routable_hello_frame_with_control_ops("health-only", 306, None),
12768 )
12769 .unwrap();
12770
12771 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
12772 let response = handler
12773 .handle_control_frame(
12774 &ctx,
12775 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
12776 )
12777 .await
12778 .unwrap();
12779
12780 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12781 assert!(parse_error(&response[0])["message"]
12782 .as_str()
12783 .unwrap()
12784 .contains("does not provide the requested target"));
12785 }
12786
12787 #[tokio::test]
12788 async fn route_open_inactive_registration_remains_target_unavailable() {
12789 let registry = Arc::new(Registry::default());
12790 let handler = ControlHandler::new(Arc::clone(®istry));
12791 handler
12792 .handle_control(
12793 ConnectionId::new(43),
12794 hello_frame("inactive", PROTOCOL_VERSION, 308),
12795 )
12796 .unwrap();
12797 assert!(registry
12798 .set_module_state_for_test("inactive", ChannelState::Closed)
12799 .unwrap());
12800
12801 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
12802 let response = handler
12803 .handle_control_frame(
12804 &ctx,
12805 route_open_frame(309, "inactive", unique_project_root("inactive")),
12806 )
12807 .await
12808 .unwrap();
12809
12810 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12811 assert!(parse_error(&response[0])["message"]
12812 .as_str()
12813 .unwrap()
12814 .contains("is not active"));
12815 }
12816
12817 #[tokio::test]
12818 async fn late_health_reply_is_recorded_through_the_module_response_path() {
12819 let registry = Arc::new(Registry::default());
12820 let forwarding = Arc::new(ForwardingTable::default());
12821 let supervisor_handle = SupervisorHandle::new();
12822 let supervisor =
12823 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
12824 .with_forwarding(Arc::clone(&forwarding))
12825 .with_handle(supervisor_handle.clone());
12826 let module = supervisor
12827 .supervise_configured(
12828 crate::ModuleSpec {
12829 module_id: "late-health-response".to_string(),
12830 program: PathBuf::from("disabled-module"),
12831 args: Vec::new(),
12832 env: Vec::new(),
12833 reserved: false,
12834 reserved_prefixes: Vec::new(),
12835 protocol: ModuleProtocol::Subc,
12836 overlap: Default::default(),
12837 },
12838 false,
12839 )
12840 .unwrap();
12841 let handler =
12842 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12843 .with_supervisor(supervisor_handle);
12844 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
12845 handler
12846 .handle_control_frame(
12847 &module_ctx,
12848 hello_frame_with_control_ops(
12849 "late-health-response",
12850 PROTOCOL_VERSION,
12851 7,
12852 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12853 ),
12854 )
12855 .await
12856 .unwrap();
12857 let probe_started_at = Instant::now() - Duration::from_millis(80);
12858 let pending = forwarding
12859 .begin_health_probe_rpc_for(
12860 "late-health-response",
12861 MODULE_CONTROL_OP_HEALTH_CHECK,
12862 probe_started_at,
12863 Instant::now() - Duration::from_millis(1),
12864 )
12865 .unwrap();
12866 assert!(forwarding
12867 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
12868 .unwrap());
12869
12870 let responses = handler
12871 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
12872 .await
12873 .unwrap();
12874
12875 assert!(responses.is_empty());
12876 let health = module.status().unwrap().health;
12877 assert_eq!(health.late_answer_count, 1);
12878 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
12879 }
12880
12881 #[tokio::test]
12882 async fn health_probe_timeout_and_module_death_are_typed() {
12883 let registry = Arc::new(Registry::default());
12884 let forwarding = Arc::new(ForwardingTable::default());
12885 let handler =
12886 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12887 .with_health_probe_timeout(Duration::from_millis(50));
12888 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12889 hello_via_sink(
12890 &handler,
12891 &module_ctx,
12892 &mut module_rx,
12893 hello_frame_with_control_ops(
12894 "aft",
12895 PROTOCOL_VERSION,
12896 7,
12897 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12898 ),
12899 )
12900 .await;
12901
12902 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12903 let responses = handler
12904 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
12905 .await
12906 .unwrap();
12907 assert_eq!(responses[0].header.ty, FrameType::Error);
12908 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
12909 let _ = module_rx.try_recv();
12910
12911 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
12912 let health_handler = handler.clone();
12913 let death_task = tokio::spawn(async move {
12914 health_handler
12915 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
12916 .await
12917 .unwrap()
12918 });
12919 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12920 .await
12921 .unwrap()
12922 .unwrap();
12923 handler
12924 .cleanup_connection(module_ctx.connection_id)
12925 .unwrap();
12926 let responses = death_task.await.unwrap();
12927 assert_eq!(responses[0].header.ty, FrameType::Error);
12928 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
12929 }
12930
12931 #[test]
12932 fn hello_requires_exact_protocol_version() {
12933 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
12934 let registry = Arc::new(Registry::default());
12935 let handler = ControlHandler::new(Arc::clone(®istry));
12936 let responses = handler
12937 .handle_control(
12938 ConnectionId::new(connection),
12939 hello_frame("aft", offered, 9),
12940 )
12941 .unwrap();
12942
12943 assert_eq!(responses.len(), 1);
12944 assert_eq!(responses[0].header.ty, FrameType::Error);
12945 let error = parse_error(&responses[0]);
12946 assert_eq!(error["code"], "version_unsupported");
12947 assert!(registry.get_module("aft").unwrap().is_none());
12948 assert_eq!(registry.active_registration_count().unwrap(), 0);
12949 }
12950 }
12951
12952 #[test]
12953 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
12954 let registry = Arc::new(Registry::default());
12955 let forwarding = Arc::new(ForwardingTable::default());
12956 let handler =
12957 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12958 let module_connection = ConnectionId::new(301);
12959 let registration = registry
12960 .register_with_control_ops(
12961 manifest("aft-push", PROTOCOL_VERSION),
12962 PROTOCOL_VERSION,
12963 module_connection,
12964 module_baseline_control_ops(),
12965 )
12966 .unwrap();
12967 let (module_tx, _module_rx) = mpsc::channel(8);
12968 let endpoint = forwarding
12969 .register_module_connection(
12970 module_connection,
12971 "aft-push".to_string(),
12972 PROTOCOL_VERSION,
12973 manifest_concurrency(®istration.manifest),
12974 FrameSink::new(module_tx),
12975 )
12976 .unwrap();
12977
12978 let unknown = Frame::build(
12980 FrameType::Push,
12981 control_flags(),
12982 0,
12983 0,
12984 5,
12985 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
12986 )
12987 .unwrap();
12988 let out = handler.handle_status_update(endpoint, unknown).unwrap();
12989 assert!(
12990 out.is_empty(),
12991 "unknown push op must be ignored, got {out:?}"
12992 );
12993
12994 let malformed = Frame::build(
12996 FrameType::Push,
12997 control_flags(),
12998 0,
12999 0,
13000 6,
13001 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
13002 )
13003 .unwrap();
13004 let out = handler.handle_status_update(endpoint, malformed).unwrap();
13005 assert_eq!(out.len(), 1);
13006 assert_eq!(out[0].header.ty, FrameType::Error);
13007 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
13008 }
13009
13010 #[test]
13011 fn hello_rejected_when_connection_already_owns_client_routes() {
13012 let registry = Arc::new(Registry::default());
13013 let forwarding = Arc::new(ForwardingTable::default());
13014 let handler =
13015 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13016 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
13018 let client_connection = ConnectionId::new(202);
13019
13020 let responses = handler
13023 .handle_control(
13024 client_connection,
13025 hello_frame("aft-second", PROTOCOL_VERSION, 9),
13026 )
13027 .unwrap();
13028 assert_eq!(responses[0].header.ty, FrameType::Error);
13029 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
13030 assert!(registry.get_module("aft-second").unwrap().is_none());
13031 }
13032
13033 #[tokio::test]
13034 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
13035 let registry = Arc::new(Registry::default());
13036 let forwarding = Arc::new(ForwardingTable::default());
13037 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
13038 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
13039 hello_via_sink(
13040 &handler,
13041 &module_ctx,
13042 &mut module_rx,
13043 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
13044 )
13045 .await;
13046 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
13047 let pending = forwarding
13048 .begin_route_bind_relay_for_test(
13049 client_ctx.connection_id,
13050 client_ctx.egress.clone(),
13051 2,
13052 "alpha",
13053 )
13054 .unwrap();
13055 forwarding
13056 .complete_pending_relay(
13057 module_ctx.connection_id,
13058 pending.corr,
13059 RouteBindRelayOutcome::Accepted,
13060 )
13061 .unwrap();
13062 client_rx.try_recv().unwrap();
13063 for module_id in ["beta", "alpha"] {
13064 let replies = handler
13065 .handle_control_frame(
13066 &module_ctx,
13067 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
13068 )
13069 .await
13070 .unwrap();
13071 assert_eq!(replies.len(), 1, "second HELLO must be refused");
13072 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
13073 }
13074 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
13075 assert!(registry.get_module("beta").unwrap().is_none());
13076 assert!(matches!(
13077 forwarding
13078 .lookup_data_route(
13079 client_ctx.connection_id,
13080 pending.client_channel,
13081 pending.client_epoch,
13082 )
13083 .unwrap(),
13084 DataRoute::Client(DataRouteState::Bound(_))
13085 ));
13086 assert!(handler
13087 .hello_launch_nonces
13088 .lock()
13089 .unwrap()
13090 .presented(module_ctx.connection_id, Some("alpha-nonce")));
13091 assert!(module_rx.try_recv().is_err());
13092 }
13093
13094 #[test]
13095 fn reserved_module_hello_requires_matching_launch_nonce() {
13096 let registry = Arc::new(Registry::default());
13097 let supervisor = SupervisorHandle::new();
13098 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
13101 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13102
13103 let no_nonce = handler
13105 .handle_control(
13106 ConnectionId::new(1),
13107 hello_frame("vault", PROTOCOL_VERSION, 1),
13108 )
13109 .unwrap();
13110 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
13111 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
13112 assert!(registry.get_module("vault").unwrap().is_none());
13113
13114 let wrong = handler
13116 .handle_control(
13117 ConnectionId::new(2),
13118 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
13119 )
13120 .unwrap();
13121 assert_eq!(wrong[0].header.ty, FrameType::Error);
13122 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
13123 assert!(registry.get_module("vault").unwrap().is_none());
13124
13125 let ok = handler
13127 .handle_control(
13128 ConnectionId::new(3),
13129 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
13130 )
13131 .unwrap();
13132 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
13133 assert!(registry.get_module("vault").unwrap().is_some());
13134 }
13135
13136 #[test]
13137 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
13138 let registry = Arc::new(Registry::default());
13139 let supervisor = SupervisorHandle::new();
13140 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13141 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13142 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13143
13144 let squat = handler
13145 .handle_control(
13146 ConnectionId::new(1),
13147 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
13148 )
13149 .unwrap();
13150 assert_eq!(squat[0].header.ty, FrameType::Error);
13151 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
13152 assert!(parse_error(&squat[0])["message"]
13153 .as_str()
13154 .unwrap()
13155 .contains("fed:"));
13156
13157 let accepted_peer = handler
13158 .handle_control(
13159 ConnectionId::new(2),
13160 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
13161 )
13162 .unwrap();
13163 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
13164
13165 let accepted_short = handler
13166 .handle_control(
13167 ConnectionId::new(3),
13168 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
13169 )
13170 .unwrap();
13171 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
13172
13173 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
13174 let response = handler
13175 .handle_control(
13176 ConnectionId::new(conn),
13177 hello_frame(module_id, PROTOCOL_VERSION, conn),
13178 )
13179 .unwrap();
13180 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
13181 }
13182 }
13183
13184 #[test]
13185 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
13186 let registry = Arc::new(Registry::default());
13187 let supervisor = SupervisorHandle::new();
13188 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13189 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13190 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
13191 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13192
13193 let owner_nonce = handler
13194 .handle_control(
13195 ConnectionId::new(1),
13196 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
13197 )
13198 .unwrap();
13199 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
13200 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
13201 assert!(registry.get_module("fed:special").unwrap().is_none());
13202
13203 let exact_nonce = handler
13204 .handle_control(
13205 ConnectionId::new(2),
13206 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
13207 )
13208 .unwrap();
13209 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
13210 assert!(registry.get_module("fed:special").unwrap().is_some());
13211 }
13212
13213 #[test]
13214 fn non_reserved_module_ignores_launch_nonce() {
13215 let registry = Arc::new(Registry::default());
13216 let handler = ControlHandler::new(Arc::clone(®istry));
13219 let no_nonce = handler
13220 .handle_control(
13221 ConnectionId::new(1),
13222 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
13223 )
13224 .unwrap();
13225 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
13226 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
13227
13228 let echoed_nonce = handler
13229 .handle_control(
13230 ConnectionId::new(2),
13231 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
13232 )
13233 .unwrap();
13234 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
13235 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
13236 }
13237
13238 #[test]
13239 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
13240 let handler = ControlHandler::default();
13241 let conn = ConnectionId::new(1);
13242 let malformed = Frame::build(
13243 FrameType::Hello,
13244 control_flags(),
13245 0,
13246 0,
13247 3,
13248 b"{not json".to_vec(),
13249 )
13250 .unwrap();
13251
13252 let error = handler.handle_control(conn, malformed).unwrap();
13253 assert_eq!(error[0].header.ty, FrameType::Error);
13254 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
13255
13256 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13257 let pong = handler.handle_control(conn, ping).unwrap();
13258 assert_eq!(pong[0].header.ty, FrameType::Pong);
13259 assert_eq!(pong[0].header.corr, 4);
13260 }
13261
13262 #[test]
13263 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
13264 let registry = Arc::new(Registry::default());
13265 let handler = ControlHandler::new(Arc::clone(®istry));
13266
13267 handler
13268 .handle_control(
13269 ConnectionId::new(1),
13270 hello_frame("aft", PROTOCOL_VERSION, 1),
13271 )
13272 .unwrap();
13273 let duplicate = handler
13274 .handle_control(
13275 ConnectionId::new(2),
13276 hello_frame("aft", PROTOCOL_VERSION, 2),
13277 )
13278 .unwrap();
13279
13280 assert_eq!(duplicate[0].header.ty, FrameType::Error);
13281 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13282 let registration = registry.get_module("aft").unwrap().unwrap();
13283 assert_eq!(registration.connection_id, ConnectionId::new(1));
13284 }
13285
13286 #[test]
13287 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
13288 let registry = Arc::new(Registry::default());
13289 let forwarding = Arc::new(ForwardingTable::default());
13290 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
13291 let handler =
13292 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13293 .with_process_liveness(process_liveness);
13294 let (ctx, route_channel, route_epoch) =
13295 bind_liveness_route(®istry, &forwarding, "aft-dead");
13296 let responses = handler
13297 .handle_route_poll(
13298 &ctx,
13299 route_poll_frame(41, PollKind::Liveness, route_channel),
13300 route_channel,
13301 route_epoch,
13302 PollKind::Liveness,
13303 )
13304 .unwrap();
13305
13306 assert_eq!(responses.len(), 1);
13307 assert_eq!(responses[0].header.ty, FrameType::Response);
13308 assert_route_poll_liveness(&responses[0], false);
13309 }
13310
13311 #[test]
13312 fn liveness_poll_without_process_source_uses_bound_route() {
13313 let registry = Arc::new(Registry::default());
13314 let forwarding = Arc::new(ForwardingTable::default());
13315 let handler =
13316 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13317 let (ctx, route_channel, route_epoch) =
13318 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
13319 let responses = handler
13320 .handle_route_poll(
13321 &ctx,
13322 route_poll_frame(42, PollKind::Liveness, route_channel),
13323 route_channel,
13324 route_epoch,
13325 PollKind::Liveness,
13326 )
13327 .unwrap();
13328
13329 assert_route_poll_liveness(&responses[0], true);
13330 }
13331
13332 #[test]
13333 fn liveness_poll_untracked_process_source_uses_bound_route() {
13334 let registry = Arc::new(Registry::default());
13335 let forwarding = Arc::new(ForwardingTable::default());
13336 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
13337 let handler =
13338 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13339 .with_process_liveness(process_liveness);
13340 let (ctx, route_channel, route_epoch) =
13341 bind_liveness_route(®istry, &forwarding, "aft-untracked");
13342 let responses = handler
13343 .handle_route_poll(
13344 &ctx,
13345 route_poll_frame(43, PollKind::Liveness, route_channel),
13346 route_channel,
13347 route_epoch,
13348 PollKind::Liveness,
13349 )
13350 .unwrap();
13351
13352 assert_route_poll_liveness(&responses[0], true);
13353 }
13354
13355 #[tokio::test]
13356 async fn unknown_op_returns_unknown_control_op() {
13357 let handler = ControlHandler::default();
13358 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
13359 let request = Frame::build(
13360 FrameType::Request,
13361 control_flags(),
13362 0,
13363 0,
13364 55,
13365 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
13366 )
13367 .unwrap();
13368
13369 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13370
13371 assert_eq!(response.len(), 1);
13372 assert_eq!(response[0].header.ty, FrameType::Error);
13373 assert_eq!(response[0].header.corr, 55);
13374 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
13375 }
13376
13377 #[tokio::test]
13378 async fn supervisor_provenance_rejects_unknown_exact_module() {
13379 let handler = ControlHandler::default();
13380 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
13381 let request = Frame::build(
13382 FrameType::Request,
13383 control_flags(),
13384 0,
13385 0,
13386 57,
13387 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
13388 )
13389 .unwrap();
13390
13391 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13392
13393 assert_eq!(response.len(), 1);
13394 assert_eq!(response[0].header.ty, FrameType::Error);
13395 assert_eq!(response[0].header.corr, 57);
13396 let error = parse_error(&response[0]);
13397 assert_eq!(error["code"], "unknown_module");
13398 assert_eq!(error["message"], "module_id 'missing' is not supervised");
13399 }
13400
13401 #[test]
13402 fn provenance_probe_override_keeps_handler_tests_deterministic() {
13403 let expected = subc_control::RunningImageAgreement::Unavailable {
13404 reason: subc_control::RunningImageUnavailableReason::HashFailed,
13405 };
13406 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
13407 assert_eq!(handler.provenance_probe_override, Some(expected));
13408 }
13409
13410 #[test]
13411 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
13412 let verdict = reload_verdict(
13413 std::path::Path::new("/bin/new"),
13414 Some(std::path::Path::new("/bin/old")),
13415 subc_control::RunningImageAgreement::Unavailable {
13416 reason: subc_control::RunningImageUnavailableReason::HashFailed,
13417 },
13418 );
13419 assert!(matches!(
13420 verdict.path,
13421 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
13422 if configured == std::path::Path::new("/bin/new")
13423 && spawned_from == std::path::Path::new("/bin/old")
13424 ));
13425 }
13426
13427 #[test]
13428 fn launch_spec_without_running_digest_is_unavailable_not_match() {
13429 use subc_control::{
13430 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
13431 };
13432 let configured = crate::supervise::ModuleSpec {
13433 module_id: "legacy".into(),
13434 program: "/module".into(),
13435 args: Vec::new(),
13436 env: Vec::new(),
13437 reserved: false,
13438 reserved_prefixes: Vec::new(),
13439 protocol: subc_control::ModuleProtocol::Subc,
13440 overlap: Default::default(),
13441 };
13442 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
13443 assert_eq!(
13444 launch_spec_agreement(&configured, false, Some(&digest)),
13445 Agreement::Unavailable {
13446 reason: Reason::NotRunning
13447 }
13448 );
13449 assert_eq!(
13450 launch_spec_agreement(&configured, true, None),
13451 Agreement::Unavailable {
13452 reason: Reason::SpawnedSpecUnavailable
13453 }
13454 );
13455 assert_eq!(
13456 launch_spec_agreement(&configured, true, Some(&digest)),
13457 Agreement::Match
13458 );
13459 }
13460
13461 #[cfg(windows)]
13462 #[test]
13463 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
13464 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
13465 let configured = dir.join("module.exe");
13466 std::fs::write(&configured, b"executable path fixture").unwrap();
13467 let spawned = std::fs::canonicalize(&configured).unwrap();
13468 assert_ne!(
13469 configured, spawned,
13470 "fixture must exercise different path spellings"
13471 );
13472 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
13473 let lower_case = configured.to_string_lossy().to_lowercase();
13474 for spelling in [
13475 &configured,
13476 Path::new(&forward_slashes),
13477 Path::new(&lower_case),
13478 ] {
13479 let verdict = reload_verdict(
13480 spelling,
13481 Some(&spawned),
13482 subc_control::RunningImageAgreement::Unavailable {
13483 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13484 },
13485 );
13486 assert_eq!(
13487 verdict.path,
13488 ReloadPathAgreement::Match,
13489 "configured {spelling:?}, spawned {spawned:?}"
13490 );
13491 }
13492 }
13493
13494 #[cfg(windows)]
13495 #[test]
13496 fn windows_reload_path_keeps_different_existing_program_pending() {
13497 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
13498 let old = dir.join("old.exe");
13499 let new = dir.join("new.exe");
13500 std::fs::write(&old, b"same executable content").unwrap();
13501 std::fs::copy(&old, &new).unwrap();
13502 let spawned = std::fs::canonicalize(&old).unwrap();
13503 let image = subc_control::RunningImageAgreement::Unavailable {
13504 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13505 };
13506 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
13507 assert!(
13508 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
13509 if configured == new && spawned_from == spawned)
13510 );
13511 let missing = dir.join("missing.exe");
13512 assert!(matches!(
13513 reload_verdict(&missing, Some(&spawned), image).path,
13514 ReloadPathAgreement::Mismatch { .. }
13515 ));
13516 }
13517
13518 #[test]
13519 fn reload_verdict_detects_replaced_image_at_same_path() {
13520 let image = subc_control::RunningImageAgreement::Mismatch {
13521 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
13522 digest: "old".into(),
13523 },
13524 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
13525 digest: "new".into(),
13526 },
13527 };
13528 let verdict = reload_verdict(
13529 std::path::Path::new("/bin/same"),
13530 Some(std::path::Path::new("/bin/same")),
13531 image.clone(),
13532 );
13533 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13534 assert_eq!(verdict.image, image);
13535 }
13536
13537 #[test]
13538 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
13539 let image = subc_control::RunningImageAgreement::Unavailable {
13540 reason: subc_control::RunningImageUnavailableReason::NotRunning,
13541 };
13542 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
13543 assert_eq!(
13544 verdict.path,
13545 subc_control::ReloadPathAgreement::Unavailable {
13546 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
13547 }
13548 );
13549 assert_eq!(verdict.image, image);
13550
13551 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
13552 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13553 };
13554 let verdict = reload_verdict(
13555 std::path::Path::new("/bin/same"),
13556 Some(std::path::Path::new("/bin/same")),
13557 unconfirmed.clone(),
13558 );
13559 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13560 assert_eq!(verdict.image, unconfirmed);
13561 }
13562
13563 #[test]
13564 fn reload_verdict_preserves_each_image_unavailability_reason() {
13565 use subc_control::RunningImageUnavailableReason as Reason;
13566
13567 for reason in [
13568 Reason::NotRunning,
13569 Reason::UnsupportedPlatform,
13570 Reason::RunningExecutableUnreadable,
13571 Reason::SpawnedPathUnreadable,
13572 Reason::HashFailed,
13573 Reason::ProcessIdentityUnconfirmed,
13574 Reason::Unknown("future_probe_reason".to_string()),
13575 ] {
13576 let image = subc_control::RunningImageAgreement::Unavailable {
13577 reason: reason.clone(),
13578 };
13579 let verdict = reload_verdict(
13580 std::path::Path::new("/bin/same"),
13581 Some(std::path::Path::new("/bin/same")),
13582 image.clone(),
13583 );
13584 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13585 assert_eq!(verdict.image, image, "{reason:?}");
13586 }
13587 }
13588
13589 #[tokio::test]
13590 async fn malformed_control_bodies_return_invalid_control_body() {
13591 let handler = ControlHandler::default();
13592 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
13593
13594 for (corr, body) in [
13595 (56, br#"{"route_channel":1}"#.as_slice()),
13596 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
13597 (
13598 58,
13599 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
13600 ),
13601 ] {
13602 let request = Frame::build(
13603 FrameType::Request,
13604 control_flags(),
13605 0,
13606 0,
13607 corr,
13608 body.to_vec(),
13609 )
13610 .unwrap();
13611 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13612
13613 assert_eq!(response.len(), 1);
13614 assert_eq!(response[0].header.ty, FrameType::Error);
13615 assert_eq!(response[0].header.corr, corr);
13616 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
13617 }
13618 }
13619
13620 #[tokio::test]
13621 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
13622 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13623 let registry = Arc::new(Registry::default());
13624 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13625 let router = Router::with_control_handler(Arc::clone(&control));
13626 let connection = router.begin_connection();
13627 let (ctx, mut rx) = route_ctx(connection.id());
13628
13629 router
13630 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
13631 .await
13632 .unwrap();
13633 let response = rx.recv().await.unwrap();
13634 let ack = parse_ack(&response);
13635 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13636 let channel = 1;
13637
13638 let goodbye =
13639 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
13640 router.route_for_connection(&ctx, goodbye).await.unwrap();
13641 assert!(rx.try_recv().is_err());
13642 assert!(registry.get_module("aft").unwrap().is_none());
13643
13644 router
13645 .route_for_connection(&ctx, channel_request(channel, 13))
13646 .await
13647 .unwrap();
13648 let error_frame = rx.recv().await.unwrap();
13649 assert_eq!(error_frame.header.ty, FrameType::Error);
13650 assert_eq!(error_frame.header.channel, channel);
13651 let captured = crate::router::test_log::captured_logs(&logs);
13652 assert_eq!(
13653 captured
13654 .lines()
13655 .filter(|line| {
13656 line.contains("module_id=aft")
13657 && line.contains("reason=explicit_goodbye")
13658 && line.contains("module registration ended")
13659 })
13660 .count(),
13661 1,
13662 "unexpected GOODBYE registry log: {captured}"
13663 );
13664 }
13665
13666 #[tokio::test]
13667 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
13668 let registry = Arc::new(Registry::default());
13669 let handler = ControlHandler::new(registry).with_capability_config(
13670 [("prov".to_string(), true), ("cons".to_string(), true)],
13671 BTreeMap::new(),
13672 );
13673 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
13674 register_capability_manifest(
13675 &handler,
13676 &provider_ctx,
13677 &mut provider_rx,
13678 capability_manifest("prov", &["thing/v1"], &[]),
13679 1,
13680 )
13681 .await;
13682 let mut consumer = capability_manifest("cons", &[], &[]);
13683 consumer.capabilities.as_mut().unwrap().requires.push(
13684 subc_protocol::manifest::CapabilityRequirement {
13685 capability: "thing/v1".to_string(),
13686 need: subc_protocol::manifest::CapabilityNeed::Required,
13687 },
13688 );
13689 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
13690 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
13691 assert_eq!(
13692 handler.capability_evaluator.verdict("cons", "thing/v1"),
13693 Some(CapabilityVerdict::Provided)
13694 );
13695 let (mut client_rx, _) = open_route_for_capability_test(
13696 &handler,
13697 &provider_ctx,
13698 &mut provider_rx,
13699 703,
13700 3,
13701 "prov",
13702 None,
13703 )
13704 .await;
13705 let goodbye =
13706 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13707 handler
13708 .handle_control_frame(&provider_ctx, goodbye)
13709 .await
13710 .unwrap();
13711 assert_eq!(
13712 handler.capability_evaluator.verdict("cons", "thing/v1"),
13713 Some(CapabilityVerdict::NeverProvided)
13714 );
13715 let closed = client_rx
13716 .try_recv()
13717 .expect("GOODBYE pushes route.closed before route GOODBYE");
13718 assert!(
13719 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
13720 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
13721 );
13722 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
13723 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
13724 }
13725
13726 struct RouteClosedFixture {
13727 handler: ControlHandler,
13728 #[cfg(unix)]
13729 supervisor: Supervisor,
13730 module_ctx: RouteCtx,
13731 #[cfg(unix)]
13732 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13733 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13734 route: Arc<crate::forwarding::RouteBinding>,
13735 }
13736
13737 async fn route_closed_fixture() -> RouteClosedFixture {
13738 let registry = Arc::new(Registry::default());
13739 let handler = ControlHandler::new(Arc::clone(®istry));
13740 #[cfg(unix)]
13741 let supervisor = Supervisor::new(registry, RestartPolicy::default())
13742 .with_forwarding(Arc::clone(&handler.forwarding));
13743 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
13744 register_capability_manifest(
13745 &handler,
13746 &module_ctx,
13747 &mut module_rx,
13748 capability_manifest("prov", &[], &[]),
13749 1,
13750 )
13751 .await;
13752 let (client_rx, bind) = open_route_for_capability_test(
13753 &handler,
13754 &module_ctx,
13755 &mut module_rx,
13756 702,
13757 2,
13758 "prov",
13759 None,
13760 )
13761 .await;
13762 let ModuleControlRequest::RouteBind {
13763 route_channel: channel,
13764 epoch: route_epoch,
13765 ..
13766 } = bind
13767 else {
13768 panic!("expected route.bind");
13769 };
13770 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
13771 handler
13772 .forwarding
13773 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
13774 .unwrap()
13775 else {
13776 panic!("the acknowledged route must be bound");
13777 };
13778 RouteClosedFixture {
13779 handler,
13780 #[cfg(unix)]
13781 supervisor,
13782 module_ctx,
13783 #[cfg(unix)]
13784 module_rx,
13785 client_rx,
13786 route,
13787 }
13788 }
13789
13790 impl RouteClosedFixture {
13791 #[cfg(unix)]
13792 async fn cut(&mut self) {
13793 let notices = async {
13794 let mut draining = self.module_rx.recv().await.unwrap();
13795 assert!(matches!(
13796 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
13797 &draining.body
13798 )
13799 .unwrap(),
13800 subc_protocol::session::ModuleControlCommand::Draining {
13801 reason: RouteCloseReason::Restart,
13802 ..
13803 }
13804 ));
13805 draining.flushed.take().unwrap().send(()).unwrap();
13806 let mut closing = self.client_rx.recv().await.unwrap();
13807 assert!(matches!(
13808 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
13809 ClientControlPush::RouteClosing {
13810 reason: RouteCloseReason::Restart,
13811 ..
13812 }
13813 ));
13814 closing.flushed.take().unwrap().send(()).unwrap();
13815 };
13816 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
13817 result.unwrap();
13818 }
13819
13820 fn disconnect(&mut self) -> ClientControlPush {
13821 self.handler
13822 .cleanup_connection(self.module_ctx.connection_id)
13823 .unwrap();
13824 let closed = self.client_rx.try_recv().unwrap();
13825 assert_eq!(closed.header.ty, FrameType::Push);
13826 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
13827 assert!(matches!(
13828 &push,
13829 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
13830 if channels == &[self.route.client_channel]
13831 ));
13832 assert_eq!(
13833 self.client_rx.try_recv().unwrap().header.ty,
13834 FrameType::Goodbye
13835 );
13836 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
13837 push
13838 }
13839 }
13840
13841 #[cfg(unix)]
13842 #[tokio::test(start_paused = true)]
13843 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
13844 let mut fixture = route_closed_fixture().await;
13845 fixture.cut().await;
13846 assert!(matches!(
13847 fixture.disconnect(),
13848 ClientControlPush::RouteClosed {
13849 reason: RouteCloseReason::Restart,
13850 drained: true,
13851 ..
13852 }
13853 ));
13854 }
13855
13856 #[cfg(unix)]
13857 #[tokio::test(start_paused = true)]
13858 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
13859 let mut fixture = route_closed_fixture().await;
13860 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
13861 let started = tokio::time::Instant::now();
13862 fixture.cut().await;
13863 assert!(started.elapsed() >= Duration::from_secs(2));
13864 assert_eq!(fixture.route.flow.in_flight(), 1);
13865 assert!(matches!(
13866 fixture.disconnect(),
13867 ClientControlPush::RouteClosed {
13868 reason: RouteCloseReason::Restart,
13869 drained: false,
13870 ..
13871 }
13872 ));
13873 }
13874
13875 #[tokio::test]
13876 async fn module_connection_crash_reports_crash_and_not_drained() {
13877 let mut fixture = route_closed_fixture().await;
13878 assert!(matches!(
13882 fixture.disconnect(),
13883 ClientControlPush::RouteClosed {
13884 reason: RouteCloseReason::Crash,
13885 drained: false,
13886 ..
13887 }
13888 ));
13889 }
13890
13891 #[tokio::test]
13892 async fn dropping_router_connection_releases_registration() {
13893 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13894 let registry = Arc::new(Registry::default());
13895 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13896 let router = Router::with_control_handler(Arc::clone(&control));
13897 let connection = router.begin_connection();
13898 let connection_id = connection.id();
13899 let (ctx, mut rx) = route_ctx(connection_id);
13900
13901 router
13902 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
13903 .await
13904 .unwrap();
13905 let response = rx.recv().await.unwrap();
13906 let ack = parse_ack(&response);
13907 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13908 assert!(registry.get_module("aft").unwrap().is_some());
13909
13910 drop(connection);
13911
13912 assert!(registry.get_module("aft").unwrap().is_none());
13913 assert_eq!(registry.active_registration_count().unwrap(), 0);
13914
13915 control
13916 .cleanup_connection(ConnectionId::new(u64::MAX))
13917 .unwrap();
13918 let captured = crate::router::test_log::captured_logs(&logs);
13919 println!("captured registry lifecycle logs:\n{captured}");
13920 let events: Vec<_> = captured
13921 .lines()
13922 .filter(|line| {
13923 line.contains("module registered module_id=aft ")
13924 || line.contains("module registration ended")
13925 })
13926 .collect();
13927 assert_eq!(
13928 events.len(),
13929 2,
13930 "unexpected registry lifecycle logs: {captured}"
13931 );
13932 assert!(events[0].contains("module registered module_id=aft "));
13933 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
13934 assert!(events[1].contains(&format!(
13935 "module_id=aft connection_id={}",
13936 connection_id.get()
13937 )));
13938 assert!(events[1].contains("reason=connection_closed"));
13939 assert!(events[1].contains("module registration ended"));
13940 assert_eq!(
13941 captured
13942 .lines()
13943 .filter(|line| line.contains("module registered module_id=aft "))
13944 .count(),
13945 1,
13946 "legacy registration admission line must appear once: {captured}"
13947 );
13948 }
13949
13950 #[tokio::test]
13951 async fn hello_registration_keeps_legacy_module_registered_line_once() {
13952 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13953 let registry = Arc::new(Registry::default());
13954 let control = ControlHandler::new(Arc::clone(®istry));
13955 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
13956 hello_via_sink(
13957 &control,
13958 &ctx,
13959 &mut rx,
13960 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
13961 )
13962 .await;
13963
13964 let captured = crate::router::test_log::captured_logs(&logs);
13965 let admissions: Vec<_> = captured
13966 .lines()
13967 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
13968 .collect();
13969 assert_eq!(
13970 admissions.len(),
13971 1,
13972 "legacy admission line must remain exactly once: {captured}"
13973 );
13974 assert!(admissions[0].contains("routable_provider=true"));
13975 assert!(admissions[0].contains("connection_id=777"));
13976 }
13977
13978 fn capability_manifest(
13979 module_id: &str,
13980 provides: &[&str],
13981 must_never_reach: &[&str],
13982 ) -> ModuleManifest {
13983 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
13984 manifest.capabilities = Some(CapabilityDeclarations {
13985 provides: provides
13986 .iter()
13987 .map(|capability| (*capability).to_string())
13988 .collect(),
13989 requires: Vec::new(),
13990 must_never_reach: must_never_reach
13991 .iter()
13992 .map(|capability| (*capability).to_string())
13993 .collect(),
13994 });
13995 manifest
13996 }
13997
13998 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
13999 Frame::build(
14000 FrameType::Hello,
14001 control_flags(),
14002 0,
14003 0,
14004 corr,
14005 serde_json::to_vec(&ModuleHelloBody {
14006 protocol_ver: manifest.protocol_ver,
14007 manifest,
14008 control_ops: None,
14009 launch_nonce: None,
14010 })
14011 .expect("capability test HELLO serializes"),
14012 )
14013 .expect("capability test HELLO frame builds")
14014 }
14015
14016 fn catalog_update_with_capabilities_frame(
14017 corr: u64,
14018 capabilities: CapabilityDeclarations,
14019 ) -> Frame {
14020 Frame::build(
14021 FrameType::Request,
14022 control_flags(),
14023 0,
14024 0,
14025 corr,
14026 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14027 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
14028 capabilities: Some(capabilities),
14029 ready: None,
14030 })
14031 .expect("capability catalog.update serializes"),
14032 )
14033 .expect("capability catalog.update frame builds")
14034 }
14035
14036 async fn register_capability_manifest(
14037 handler: &ControlHandler,
14038 ctx: &RouteCtx,
14039 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14040 manifest: ModuleManifest,
14041 corr: u64,
14042 ) {
14043 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
14044 }
14045
14046 async fn open_route_for_capability_test(
14047 handler: &ControlHandler,
14048 target_ctx: &RouteCtx,
14049 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
14050 client_connection_id: u64,
14051 corr: u64,
14052 target_module_id: &str,
14053 consumer_identity: Option<ConsumerIdentity>,
14054 ) -> (
14055 mpsc::Receiver<crate::router::OutboundFrame>,
14056 ModuleControlRequest,
14057 ) {
14058 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
14059 let route_handler = handler.clone();
14060 let target_module_id = target_module_id.to_string();
14061 let route_task = tokio::spawn(async move {
14062 route_handler
14063 .handle_control_frame(
14064 &client_ctx,
14065 route_open_frame_with_admission_facts(
14066 corr,
14067 &target_module_id,
14068 unique_project_root("admission-facts"),
14069 consumer_identity,
14070 None,
14071 ),
14072 )
14073 .await
14074 .expect("capability test route.open succeeds")
14075 });
14076 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
14077 .await
14078 .expect("capability test route.open must reach route.bind")
14079 .expect("target control receiver stays open");
14080 let bind_request: ModuleControlRequest =
14081 serde_json::from_slice(&bind.body).expect("route.bind decodes");
14082 handler
14083 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
14084 .await
14085 .expect("capability test route.bind ACK succeeds");
14086 assert!(route_task.await.expect("route.open task joins").is_empty());
14087 let opened = client_rx
14088 .recv()
14089 .await
14090 .expect("successful route.open publishes a response");
14091 assert!(matches!(
14092 serde_json::from_slice::<ClientControlResponse>(&opened.body),
14093 Ok(ClientControlResponse::RouteOpen { .. })
14094 ));
14095 (client_rx, bind_request)
14096 }
14097
14098 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
14099 assert_eq!(frame.header.ty, FrameType::Push);
14100 assert_eq!(frame.header.channel, 0);
14101 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
14102 .expect("route.closed control push decodes");
14103 let ClientControlPush::RouteClosed { channels, .. } = &push else {
14104 panic!("expected route.closed");
14105 };
14106 assert_eq!(channels.len(), 1, "exactly one violating route closed");
14107 let channels = channels.clone();
14108 assert_eq!(
14109 push,
14110 ClientControlPush::RouteClosed {
14111 module_id: target_module_id.to_string(),
14112 channels,
14113 reason: RouteCloseReason::CapabilityDenied,
14114 drained: false,
14115 abandoned: 0,
14116 excluded_subscriptions: 0,
14117 terminal: Some(false),
14118 }
14119 );
14120 }
14121
14122 #[tokio::test]
14123 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
14124 let registry = Arc::new(Registry::default());
14125 let forwarding = Arc::new(ForwardingTable::default());
14126 let supervisor = SupervisorHandle::new();
14127 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14128 let handler =
14129 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14130 .with_supervisor(supervisor);
14131 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
14132 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
14133 register_capability_manifest(
14134 &handler,
14135 &target_ctx,
14136 &mut target_rx,
14137 capability_manifest("target", &["credentials-provider/v1"], &[]),
14138 1,
14139 )
14140 .await;
14141 register_capability_manifest(
14142 &handler,
14143 &opener_ctx,
14144 &mut opener_rx,
14145 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14146 2,
14147 )
14148 .await;
14149
14150 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
14151 let replies = handler
14152 .handle_control_frame(
14153 &client_ctx,
14154 route_open_frame_with_admission_facts(
14155 3,
14156 "target",
14157 unique_project_root("admission-facts"),
14158 Some(ConsumerIdentity {
14159 module_id: "opener".to_string(),
14160 launch_nonce: "opener-nonce".to_string(),
14161 }),
14162 None,
14163 ),
14164 )
14165 .await
14166 .expect("denied route.open returns a typed frame");
14167 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14168 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14169 assert!(
14170 target_rx.try_recv().is_err(),
14171 "forbidden route.open must not relay route.bind"
14172 );
14173 }
14174
14175 #[tokio::test]
14176 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
14177 let registry = Arc::new(Registry::default());
14178 let forwarding = Arc::new(ForwardingTable::default());
14179 let supervisor = SupervisorHandle::new();
14180 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14181 let handler =
14182 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14183 .with_supervisor(supervisor);
14184 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
14185 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
14186 register_capability_manifest(
14187 &handler,
14188 &target_ctx,
14189 &mut target_rx,
14190 capability_manifest("target", &["credentials-provider/v1"], &[]),
14191 1,
14192 )
14193 .await;
14194 register_capability_manifest(
14195 &handler,
14196 &old_opener_ctx,
14197 &mut old_opener_rx,
14198 capability_manifest("opener", &[], &[]),
14199 2,
14200 )
14201 .await;
14202 let (mut client_rx, _) = open_route_for_capability_test(
14203 &handler,
14204 &target_ctx,
14205 &mut target_rx,
14206 712,
14207 3,
14208 "target",
14209 Some(ConsumerIdentity {
14210 module_id: "opener".to_string(),
14211 launch_nonce: "opener-nonce".to_string(),
14212 }),
14213 )
14214 .await;
14215 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14216
14217 handler
14218 .cleanup_connection(old_opener_ctx.connection_id)
14219 .expect("old opener registration cleans up");
14220 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
14221 register_capability_manifest(
14222 &handler,
14223 &new_opener_ctx,
14224 &mut new_opener_rx,
14225 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14226 4,
14227 )
14228 .await;
14229
14230 assert_capability_denied_push(
14231 client_rx
14232 .try_recv()
14233 .expect("HELLO deny addition must emit route.closed")
14234 .frame,
14235 "target",
14236 );
14237 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14238 assert!(matches!(
14239 target_rx.try_recv(),
14240 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14241 ));
14242 }
14243
14244 #[tokio::test]
14245 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
14246 let registry = Arc::new(Registry::default());
14247 let forwarding = Arc::new(ForwardingTable::default());
14248 let supervisor = SupervisorHandle::new();
14249 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14250 let handler =
14251 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14252 .with_supervisor(supervisor);
14253 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
14254 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
14255 register_capability_manifest(
14256 &handler,
14257 &target_ctx,
14258 &mut target_rx,
14259 capability_manifest("target", &[], &[]),
14260 1,
14261 )
14262 .await;
14263 register_capability_manifest(
14264 &handler,
14265 &opener_ctx,
14266 &mut opener_rx,
14267 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14268 2,
14269 )
14270 .await;
14271 let (mut client_rx, _) = open_route_for_capability_test(
14272 &handler,
14273 &target_ctx,
14274 &mut target_rx,
14275 722,
14276 3,
14277 "target",
14278 Some(ConsumerIdentity {
14279 module_id: "opener".to_string(),
14280 launch_nonce: "opener-nonce".to_string(),
14281 }),
14282 )
14283 .await;
14284 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14285
14286 let replies = handler
14287 .handle_control_frame(
14288 &target_ctx,
14289 catalog_update_with_capabilities_frame(
14290 4,
14291 CapabilityDeclarations {
14292 provides: vec!["credentials-provider/v1".to_string()],
14293 requires: Vec::new(),
14294 must_never_reach: Vec::new(),
14295 },
14296 ),
14297 )
14298 .await
14299 .expect("claim catalog.update succeeds");
14300 assert!(matches!(
14301 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
14302 Ok(ModuleControlResponseToModule::CatalogUpdate {})
14303 ));
14304 assert_capability_denied_push(
14305 client_rx
14306 .try_recv()
14307 .expect("claim addition must emit route.closed")
14308 .frame,
14309 "target",
14310 );
14311 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14312 assert!(matches!(
14313 target_rx.try_recv(),
14314 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14315 ));
14316 }
14317
14318 #[tokio::test]
14319 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
14320 let registry = Arc::new(Registry::default());
14321 let forwarding = Arc::new(ForwardingTable::default());
14322 let supervisor = SupervisorHandle::new();
14323 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14324 let handler =
14325 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14326 .with_supervisor(supervisor);
14327 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
14328 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
14329 register_capability_manifest(
14330 &handler,
14331 &target_ctx,
14332 &mut target_rx,
14333 capability_manifest("target", &["credentials-provider/v1"], &[]),
14334 1,
14335 )
14336 .await;
14337 register_capability_manifest(
14338 &handler,
14339 &opener_ctx,
14340 &mut opener_rx,
14341 capability_manifest("opener", &[], &[]),
14342 2,
14343 )
14344 .await;
14345 let (mut client_rx, _) = open_route_for_capability_test(
14346 &handler,
14347 &target_ctx,
14348 &mut target_rx,
14349 732,
14350 3,
14351 "target",
14352 Some(ConsumerIdentity {
14353 module_id: "opener".to_string(),
14354 launch_nonce: "opener-nonce".to_string(),
14355 }),
14356 )
14357 .await;
14358 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14359
14360 handler
14361 .handle_control_frame(
14362 &target_ctx,
14363 catalog_update_with_capabilities_frame(
14364 4,
14365 CapabilityDeclarations {
14366 provides: Vec::new(),
14367 requires: Vec::new(),
14368 must_never_reach: Vec::new(),
14369 },
14370 ),
14371 )
14372 .await
14373 .expect("claim removal catalog.update succeeds");
14374 assert_eq!(
14375 forwarding.active_binding_count().unwrap(),
14376 1,
14377 "removing an attested target claim must leave the route census unchanged"
14378 );
14379 assert!(
14380 client_rx.try_recv().is_err(),
14381 "claim removal must not emit route.closed capability_denied"
14382 );
14383 assert!(
14384 target_rx.try_recv().is_err(),
14385 "claim removal must not send the target a route GOODBYE"
14386 );
14387 }
14388
14389 #[tokio::test]
14392 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
14393 let registry = Arc::new(Registry::default());
14394 let forwarding = Arc::new(ForwardingTable::default());
14395 let supervisor = SupervisorHandle::new();
14396 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14397 let handler =
14398 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14399 .with_supervisor(supervisor);
14400 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
14401 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
14402 register_capability_manifest(
14403 &handler,
14404 &target_ctx,
14405 &mut target_rx,
14406 capability_manifest("target", &["credentials-provider/v1"], &[]),
14407 1,
14408 )
14409 .await;
14410 register_capability_manifest(
14411 &handler,
14412 &opener_ctx,
14413 &mut opener_rx,
14414 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14415 2,
14416 )
14417 .await;
14418
14419 let (_client_rx, bind) = open_route_for_capability_test(
14420 &handler,
14421 &target_ctx,
14422 &mut target_rx,
14423 742,
14424 3,
14425 "target",
14426 None,
14427 )
14428 .await;
14429 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14430 panic!("direct scope-honesty route must bind");
14431 };
14432 assert_eq!(principal, Some(Principal::Direct));
14433 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14434 }
14435
14436 #[tokio::test]
14439 async fn must_never_reach_self_route_is_capability_forbidden() {
14440 let registry = Arc::new(Registry::default());
14441 let forwarding = Arc::new(ForwardingTable::default());
14442 let supervisor = SupervisorHandle::new();
14443 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
14444 let handler =
14445 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14446 .with_supervisor(supervisor);
14447 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
14448 register_capability_manifest(
14449 &handler,
14450 &self_ctx,
14451 &mut self_rx,
14452 capability_manifest(
14453 "self-provider",
14454 &["credentials-provider/v1"],
14455 &["credentials-provider/v1"],
14456 ),
14457 1,
14458 )
14459 .await;
14460
14461 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
14462 let replies = handler
14463 .handle_control_frame(
14464 &client_ctx,
14465 route_open_frame_with_admission_facts(
14466 2,
14467 "self-provider",
14468 unique_project_root("admission-facts"),
14469 Some(ConsumerIdentity {
14470 module_id: "self-provider".to_string(),
14471 launch_nonce: "self-nonce".to_string(),
14472 }),
14473 None,
14474 ),
14475 )
14476 .await
14477 .expect("self-route refusal returns a typed frame");
14478 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14479 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14480 assert!(
14481 self_rx.try_recv().is_err(),
14482 "self denial must not relay route.bind"
14483 );
14484 }
14485
14486 #[test]
14487 fn unsupported_channel_zero_frame_returns_error() {
14488 let handler = ControlHandler::default();
14489 let request = Frame::build(
14490 FrameType::Request,
14491 control_flags(),
14492 0,
14493 0,
14494 21,
14495 b"opaque".to_vec(),
14496 )
14497 .unwrap();
14498
14499 let response = handler
14500 .handle_control(ConnectionId::new(1), request)
14501 .unwrap();
14502
14503 assert_eq!(response[0].header.ty, FrameType::Error);
14504 assert_eq!(
14505 parse_error(&response[0])["code"],
14506 "unsupported_control_frame"
14507 );
14508 }
14509
14510 mod swap {
14515 use super::*;
14516
14517 const INCUMBENT: ConnectionId = ConnectionId::new(30);
14518 const CANDIDATE: ConnectionId = ConnectionId::new(40);
14519
14520 struct Swap {
14521 registry: Arc<Registry>,
14522 forwarding: Arc<ForwardingTable>,
14523 handler: ControlHandler,
14524 incumbent_ctx: RouteCtx,
14525 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14526 candidate_ctx: RouteCtx,
14527 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14528 }
14529
14530 async fn swap_with_incumbent() -> Swap {
14531 let registry = Arc::new(Registry::default());
14532 let forwarding = Arc::new(ForwardingTable::default());
14533 let handler =
14534 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14535 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
14536 hello_via_sink(
14537 &handler,
14538 &incumbent_ctx,
14539 &mut incumbent_rx,
14540 hello_frame("aft", PROTOCOL_VERSION, 7),
14541 )
14542 .await;
14543 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
14544 Swap {
14545 registry,
14546 forwarding,
14547 handler,
14548 incumbent_ctx,
14549 incumbent_rx,
14550 candidate_ctx,
14551 candidate_rx,
14552 }
14553 }
14554
14555 fn register_candidate(swap: &Swap, ready: Option<bool>) {
14556 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
14557 candidate_manifest.ready = ready;
14558 let registration = swap
14559 .registry
14560 .register_candidate_with_control_ops(
14561 candidate_manifest,
14562 PROTOCOL_VERSION,
14563 CANDIDATE,
14564 module_baseline_control_ops(),
14565 )
14566 .unwrap();
14567 swap.forwarding
14568 .register_candidate_module_connection(
14569 CANDIDATE,
14570 "aft".to_string(),
14571 PROTOCOL_VERSION,
14572 manifest_concurrency(®istration.manifest),
14573 swap.candidate_ctx.egress.clone(),
14574 )
14575 .unwrap();
14576 }
14577
14578 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
14579 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
14580 swap.registry.promote_candidate("aft").unwrap().unwrap();
14581 cutover.incumbent.unwrap()
14582 }
14583
14584 #[tokio::test]
14593 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
14594 let mut swap = swap_with_incumbent().await;
14595 let hold = Arc::new(std::sync::Barrier::new(2));
14596 let handler = swap
14597 .handler
14598 .clone()
14599 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14600 handler
14601 .supervisor
14602 .open_swap("aft", "candidate-nonce".to_string());
14603 let hello = {
14604 let handler = handler.clone();
14605 let candidate_ctx = swap.candidate_ctx.clone();
14606 std::thread::spawn(move || {
14607 handler
14608 .handle_hello(
14609 candidate_ctx.connection_id,
14610 Some(candidate_ctx.egress.clone()),
14611 hello_frame_with_nonce(
14612 "aft",
14613 PROTOCOL_VERSION,
14614 8,
14615 Some("candidate-nonce"),
14616 ),
14617 )
14618 .unwrap()
14619 })
14620 };
14621
14622 hold.wait();
14623 assert!(swap
14626 .registry
14627 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
14628 .unwrap()
14629 .is_some());
14630 assert!(swap
14631 .registry
14632 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14633 .unwrap()
14634 .is_none());
14635 assert!(swap
14636 .forwarding
14637 .module_endpoint_for_connection(CANDIDATE)
14638 .unwrap()
14639 .is_none());
14640
14641 hold.wait();
14642 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
14643 assert_eq!(
14644 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
14645 FrameType::HelloAck
14646 );
14647 let published = swap
14648 .registry
14649 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14650 .unwrap()
14651 .expect("candidate is published once its HELLO completes");
14652 assert_eq!(published.connection_id, CANDIDATE);
14653 assert!(swap
14654 .forwarding
14655 .module_endpoint_for_connection(CANDIDATE)
14656 .unwrap()
14657 .is_some());
14658 }
14659
14660 fn keyed_total(counters: &Value, key: &str) -> u64 {
14661 counters[key]
14662 .as_object()
14663 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
14664 .unwrap_or(0)
14665 }
14666
14667 #[tokio::test]
14668 async fn replacement_logs_old_end_and_new_admission_once() {
14669 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14670 let mut swap = swap_with_incumbent().await;
14671 swap.handler
14672 .supervisor
14673 .open_swap("aft", "candidate-nonce".to_string());
14674 hello_via_sink(
14675 &swap.handler,
14676 &swap.candidate_ctx,
14677 &mut swap.candidate_rx,
14678 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
14679 )
14680 .await;
14681 cutover(&swap);
14682 swap.handler.cleanup_connection(INCUMBENT).unwrap();
14683
14684 let captured = crate::router::test_log::captured_logs(&logs);
14685 println!("captured replacement registry logs:\n{captured}");
14686 let new_admissions: Vec<_> = captured
14687 .lines()
14688 .filter(|line| {
14689 line.contains("connection_id=40")
14690 && line.contains("swap candidate registered; not routable until cutover")
14691 })
14692 .collect();
14693 assert_eq!(
14694 new_admissions.len(),
14695 1,
14696 "unexpected admission logs: {captured}"
14697 );
14698 assert!(new_admissions[0].contains("module_id=aft"));
14699 assert!(new_admissions[0].contains("connection_id=40"));
14700 assert!(
14701 new_admissions[0].contains("swap candidate registered; not routable until cutover")
14702 );
14703 assert!(new_admissions[0].contains("ready=true"));
14704
14705 let old_admissions: Vec<_> = captured
14706 .lines()
14707 .filter(|line| {
14708 line.contains("module registered module_id=aft ")
14709 && line.contains("connection_id=30")
14710 })
14711 .collect();
14712 assert_eq!(
14713 old_admissions.len(),
14714 1,
14715 "unexpected incumbent admission: {captured}"
14716 );
14717
14718 let old_ends: Vec<_> = captured
14719 .lines()
14720 .filter(|line| {
14721 line.contains("connection_id=30") && line.contains("module registration ended")
14722 })
14723 .collect();
14724 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
14725 assert!(old_ends[0].contains("module_id=aft"));
14726 assert!(old_ends[0].contains("connection_id=30"));
14727 assert!(old_ends[0].contains("reason=replaced"));
14728 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
14729 assert_eq!(
14730 captured
14731 .lines()
14732 .filter(|line| line.contains("module registration promoted"))
14733 .count(),
14734 1,
14735 "unexpected promotion logs: {captured}"
14736 );
14737 }
14738
14739 #[tokio::test]
14745 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
14746 let mut swap = swap_with_incumbent().await;
14747 let handler = swap.handler.clone();
14748
14749 let cotenant = ConnectionId::new(31);
14751 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
14752 let (cotenant_task, cotenant_bind) = relay_route_open(
14753 &handler,
14754 cotenant,
14755 &cotenant_ctx.egress,
14756 &mut swap.incumbent_rx,
14757 100,
14758 "aft",
14759 "swap-cotenant",
14760 )
14761 .await;
14762 handler
14763 .handle_control_frame(
14764 &swap.incumbent_ctx,
14765 route_bind_ack(cotenant_bind.header.corr),
14766 )
14767 .await
14768 .unwrap();
14769 assert!(cotenant_task.await.unwrap().is_empty());
14770 let (cotenant_channel, cotenant_epoch) =
14771 published_route(&cotenant_rx.recv().await.unwrap());
14772
14773 let caller = ConnectionId::new(32);
14775 let (caller_ctx, mut caller_rx) = route_ctx(caller);
14776 let (caller_task, caller_bind) = relay_route_open(
14777 &handler,
14778 caller,
14779 &caller_ctx.egress,
14780 &mut swap.incumbent_rx,
14781 101,
14782 "aft",
14783 "swap-caller",
14784 )
14785 .await;
14786 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
14787
14788 register_candidate(&swap, None);
14789 cutover(&swap);
14790
14791 let ack = handler
14793 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
14794 .await;
14795 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
14796 if module_loop_error.is_some() {
14797 handler.cleanup_connection(INCUMBENT).unwrap();
14800 }
14801
14802 assert!(
14804 cotenant_rx.try_recv().is_err(),
14805 "the co-tenant route on the incumbent was torn down by one late ack: \
14806 {module_loop_error:?}"
14807 );
14808 assert!(matches!(
14809 swap.forwarding
14810 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14811 .unwrap(),
14812 DataRoute::Client(DataRouteState::Bound(_))
14813 ));
14814 assert_eq!(module_loop_error, None);
14815 assert!(swap
14816 .registry
14817 .get_module_by_connection(INCUMBENT)
14818 .unwrap()
14819 .is_some());
14820
14821 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
14823 .await
14824 .expect("the incumbent is told to drop the abandoned binding")
14825 .unwrap()
14826 .frame;
14827 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14828 assert_eq!(goodbye.header.channel, abandoned_channel);
14829 assert_eq!(goodbye.header.epoch, abandoned_epoch);
14830 assert!(swap.incumbent_rx.try_recv().is_err());
14831
14832 let response = caller_task.await.unwrap();
14834 assert_eq!(response.len(), 1);
14835 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14836 assert!(caller_rx.try_recv().is_err());
14837
14838 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
14842 let counters = handler.counters().snapshot();
14843 assert_eq!(
14844 keyed_total(&counters, "route_open_accepted_by_principal"),
14845 1
14846 );
14847 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
14848 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
14849 }
14850
14851 #[tokio::test]
14855 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
14856 let mut swap = swap_with_incumbent().await;
14857 register_candidate(&swap, None);
14858 let incumbent = cutover(&swap);
14859 swap.forwarding
14860 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
14861 .unwrap()
14862 .expect("the incumbent is still registered");
14863
14864 let client = ConnectionId::new(33);
14865 let (client_ctx, mut client_rx) = route_ctx(client);
14866 let route_handler = swap.handler.clone();
14867 let open_ctx = RouteCtx {
14868 connection_id: client,
14869 egress: client_ctx.egress.clone(),
14870 };
14871 let mut route_task = tokio::spawn(async move {
14872 route_handler
14873 .handle_control_frame(
14874 &open_ctx,
14875 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
14876 )
14877 .await
14878 .unwrap()
14879 });
14880 let bind = tokio::select! {
14881 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
14882 response = &mut route_task => {
14883 let response = response.unwrap();
14884 panic!(
14885 "post-cutover route.open was refused instead of relayed to the candidate: {}",
14886 parse_error(&response[0])["code"]
14887 );
14888 }
14889 };
14890 swap.handler
14891 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
14892 .await
14893 .unwrap();
14894 assert!(route_task.await.unwrap().is_empty());
14895 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
14896 match swap
14897 .forwarding
14898 .lookup_data_route(client, channel, epoch)
14899 .unwrap()
14900 {
14901 DataRoute::Client(DataRouteState::Bound(route)) => {
14902 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
14903 }
14904 other => panic!("expected a bound route on the candidate, got {other:?}"),
14905 }
14906 assert!(swap.incumbent_rx.try_recv().is_err());
14907 }
14908
14909 #[tokio::test]
14914 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
14915 let swap = swap_with_incumbent().await;
14916 register_candidate(&swap, Some(false));
14917 let update = Frame::build(
14918 FrameType::Request,
14919 control_flags(),
14920 0,
14921 0,
14922 55,
14923 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14924 provides: manifest("aft", PROTOCOL_VERSION).provides,
14925 capabilities: None,
14926 ready: Some(true),
14927 })
14928 .unwrap(),
14929 )
14930 .unwrap();
14931
14932 let replies = swap
14933 .handler
14934 .handle_control_frame(&swap.candidate_ctx, update)
14935 .await
14936 .unwrap();
14937
14938 assert_eq!(replies.len(), 1);
14939 assert_eq!(
14940 replies[0].header.ty,
14941 FrameType::Response,
14942 "candidate catalog.update was refused: {:?}",
14943 serde_json::from_slice::<Value>(&replies[0].body).ok()
14944 );
14945 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
14946 assert_eq!(
14947 swap.registry
14948 .get_module("aft")
14949 .unwrap()
14950 .unwrap()
14951 .connection_id,
14952 INCUMBENT
14953 );
14954 }
14955 }
14956
14957 mod swap_admission {
14961 use super::*;
14962
14963 const INCUMBENT_NONCE: &str = "incumbent-nonce";
14964 const CANDIDATE_NONCE: &str = "candidate-nonce";
14965
14966 fn handler_with_incumbent(
14967 module_id: &str,
14968 reserved: bool,
14969 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
14970 let registry = Arc::new(Registry::default());
14971 let supervisor = SupervisorHandle::new();
14972 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
14973 if reserved {
14974 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
14975 }
14976 let handler =
14977 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
14978 let incumbent = handler
14979 .handle_control(
14980 ConnectionId::new(1),
14981 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
14982 )
14983 .unwrap();
14984 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
14985 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
14986 (registry, supervisor, handler)
14987 }
14988
14989 #[test]
14995 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
14996 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
14997
14998 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
14999 let replies = handler
15000 .handle_control(
15001 ConnectionId::new(connection),
15002 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
15003 )
15004 .unwrap();
15005 assert_eq!(replies[0].header.ty, FrameType::Error);
15006 assert_eq!(
15007 parse_error(&replies[0])["code"],
15008 "swap_token_invalid",
15009 "nonce {nonce:?}"
15010 );
15011 }
15012 assert!(registry.get_candidate("aft").unwrap().is_none());
15013 assert_eq!(
15014 registry.get_module("aft").unwrap().unwrap().connection_id,
15015 ConnectionId::new(1)
15016 );
15017
15018 let admitted = handler
15021 .handle_control(
15022 ConnectionId::new(4),
15023 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
15024 )
15025 .unwrap();
15026 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
15027 assert_eq!(
15028 registry
15029 .get_candidate("aft")
15030 .unwrap()
15031 .unwrap()
15032 .connection_id,
15033 ConnectionId::new(4)
15034 );
15035 assert_eq!(
15036 registry.get_module("aft").unwrap().unwrap().connection_id,
15037 ConnectionId::new(1),
15038 "the candidate must not take the active slot"
15039 );
15040 let replayed = handler
15041 .handle_control(
15042 ConnectionId::new(5),
15043 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
15044 )
15045 .unwrap();
15046 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
15047
15048 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
15052 let squatter = handler
15053 .handle_control(
15054 ConnectionId::new(6),
15055 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
15056 )
15057 .unwrap();
15058 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
15059 assert!(
15060 registry.get_module("aft").unwrap().is_none(),
15061 "a squatter took the active slot of an id being swapped"
15062 );
15063 }
15064
15065 #[test]
15070 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
15071 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
15072
15073 let replies = handler
15074 .handle_control(
15075 ConnectionId::new(2),
15076 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15077 )
15078 .unwrap();
15079
15080 assert_eq!(
15081 replies[0].header.ty,
15082 FrameType::HelloAck,
15083 "reserved candidate refused: {:?}",
15084 serde_json::from_slice::<Value>(&replies[0].body).ok()
15085 );
15086 assert_eq!(
15087 registry
15088 .get_candidate("vault")
15089 .unwrap()
15090 .unwrap()
15091 .connection_id,
15092 ConnectionId::new(2)
15093 );
15094 }
15095
15096 #[test]
15099 fn without_an_open_swap_the_ordinary_gates_decide() {
15100 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
15101 supervisor.close_swap("vault");
15102
15103 let candidate = handler
15104 .handle_control(
15105 ConnectionId::new(2),
15106 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
15107 )
15108 .unwrap();
15109 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
15110 let duplicate = handler
15111 .handle_control(
15112 ConnectionId::new(3),
15113 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
15114 )
15115 .unwrap();
15116 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15117 assert!(registry.get_candidate("vault").unwrap().is_none());
15118 }
15119 }
15120
15121 mod scopes {
15125 mod idle_tests {
15126 use super::*;
15127 include!("scope_idle_tests.rs");
15128 }
15129 mod route_open_timings {
15130 use super::*;
15131 include!("route_open_timing_tests.rs");
15132 }
15133 mod activity {
15134 include!("activity_scope_tests.rs");
15135 }
15136 mod delta {
15137 include!("scope_handler_delta_tests.rs");
15138 }
15139 mod call_modes {
15140 include!("scope_call_mode_tests.rs");
15141 }
15142 #[tokio::test]
15143 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
15144 let rig = rig().await;
15145 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
15146 "s".into(),
15147 1,
15148 "tool".into(),
15149 ))
15150 .unwrap();
15151 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
15152 let (direct, _rx) = wide_ctx(999);
15153 for (ctx, expected) in [
15154 (&rig.owner, FrameType::Response),
15155 (&direct, FrameType::Error),
15156 ] {
15157 let frame =
15158 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
15159 .unwrap();
15160 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
15161 assert_eq!(replies[0].header.ty, expected);
15162 if expected == FrameType::Response {
15163 let reply: subc_control::ScopeServedByReply =
15164 serde_json::from_slice(&replies[0].body).unwrap();
15165 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
15166 } else {
15167 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
15168 }
15169 }
15170 let reply = call(
15171 &rig.handler,
15172 &rig.owner,
15173 &ModuleControlRequestFromModule::ScopeDescribe {
15174 owner: Principal::Reserved {
15175 module_id: OWNER.into(),
15176 },
15177 scope_ref: "s".into(),
15178 },
15179 )
15180 .await;
15181 assert!(matches!(
15182 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
15183 ModuleControlResponseToModule::ScopeDescribe { .. }
15184 ));
15185 }
15186
15187 use subc_protocol::scope::{
15188 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
15189 ScopeStatus,
15190 };
15191
15192 use super::*;
15193
15194 const OWNER: &str = "prefrontal-core";
15195
15196 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
15197 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
15198 }
15199
15200 async fn call(
15201 handler: &ControlHandler,
15202 ctx: &RouteCtx,
15203 request: &ModuleControlRequestFromModule,
15204 ) -> Frame {
15205 let body = serde_json::to_vec(request).unwrap();
15206 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
15207 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
15208 assert_eq!(replies.len(), 1, "{replies:?}");
15209 replies.pop().unwrap()
15210 }
15211
15212 async fn sync(
15213 handler: &ControlHandler,
15214 ctx: &RouteCtx,
15215 generation: u64,
15216 scopes: Vec<ScopeRecord>,
15217 ) -> Result<ModuleControlResponseToModule, String> {
15218 let reply = call(
15219 handler,
15220 ctx,
15221 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
15222 )
15223 .await;
15224 match reply.header.ty {
15225 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
15226 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
15227 }
15228 }
15229
15230 async fn describe(
15231 handler: &ControlHandler,
15232 ctx: &RouteCtx,
15233 owner: &str,
15234 scope_ref: &str,
15235 ) -> ModuleControlResponseToModule {
15236 let reply = call(
15237 handler,
15238 ctx,
15239 &ModuleControlRequestFromModule::ScopeDescribe {
15240 owner: Principal::Reserved {
15241 module_id: owner.to_string(),
15242 },
15243 scope_ref: scope_ref.to_string(),
15244 },
15245 )
15246 .await;
15247 assert_eq!(
15248 reply.header.ty,
15249 FrameType::Response,
15250 "{:?}",
15251 parse_error(&reply)
15252 );
15253 serde_json::from_slice(&reply.body).unwrap()
15254 }
15255
15256 #[tokio::test]
15257 async fn scope_describe_never_echoes_bind_proof_or_hash() {
15258 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15259 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15260 let mut rig = rig().await;
15261 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15262 .await;
15263 let _bound = rig
15264 .bound(
15265 Some(AFT),
15266 PLEXUS,
15267 Some(
15268 rig.selector("s", Some(1))
15269 .with_bind_proof(Some(PROOF.into())),
15270 ),
15271 )
15272 .await;
15273 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
15274 let output = serde_json::to_string(&reply).unwrap();
15275 assert!(output.contains("\"status\":\"live\""));
15276 assert!(!output.contains(PROOF));
15277 assert!(!output.contains(HASH));
15278 }
15279
15280 async fn module(
15282 handler: &ControlHandler,
15283 connection: u64,
15284 module_id: &str,
15285 nonce: Option<&str>,
15286 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15287 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
15288 hello_via_sink(
15289 handler,
15290 &ctx,
15291 &mut rx,
15292 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
15293 )
15294 .await;
15295 (ctx, rx)
15296 }
15297
15298 #[tokio::test]
15301 async fn a_client_connection_cannot_sync_or_describe() {
15302 let handler = ControlHandler::new(Arc::new(Registry::default()));
15303 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
15304 for request in [
15305 ModuleControlRequestFromModule::ScopeSync {
15306 generation: 1,
15307 scopes: vec![head("s", 1)],
15308 },
15309 ModuleControlRequestFromModule::ScopeDescribe {
15310 owner: Principal::Direct,
15311 scope_ref: "s".to_string(),
15312 },
15313 ] {
15314 let reply = call(&handler, &ctx, &request).await;
15315 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
15316 }
15317 assert!(
15318 !handler
15319 .scopes
15320 .read()
15321 .unwrap()
15322 .describe(
15323 &Principal::Reserved {
15324 module_id: OWNER.to_string()
15325 },
15326 "s"
15327 )
15328 .owner_synced
15329 );
15330 }
15331
15332 #[tokio::test]
15335 async fn a_module_without_a_supervised_launch_cannot_sync() {
15336 let handler = ControlHandler::new(Arc::new(Registry::default()));
15337 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
15338 assert_eq!(
15339 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
15340 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15341 );
15342 }
15343
15344 #[tokio::test]
15345 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
15346 let supervisor = SupervisorHandle::new();
15347 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15348 let handler = ControlHandler::new(Arc::new(Registry::default()))
15349 .with_supervisor(supervisor.clone());
15350 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15351 sync(&handler, &incumbent, 1, vec![head("s", 1)])
15352 .await
15353 .expect("the current launch syncs");
15354
15355 supervisor.open_swap(OWNER, "n2".to_string());
15358 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
15359 assert_eq!(
15360 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15361 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15362 );
15363 sync(&handler, &incumbent, 2, vec![head("s", 1)])
15364 .await
15365 .expect("the serving owner syncs during the swap");
15366
15367 supervisor.close_swap(OWNER);
15370 assert_eq!(
15371 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15372 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15373 );
15374 sync(&handler, &incumbent, 3, vec![head("s", 1)])
15375 .await
15376 .expect("the serving owner syncs after the rollback");
15377 handler.cleanup_connection(candidate.connection_id).unwrap();
15378
15379 supervisor.open_swap(OWNER, "n3".to_string());
15384 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
15385 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
15386 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
15387 .await
15388 .expect("the promoted launch takes authority");
15389 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15390 panic!("unexpected reply {reply:?}");
15391 };
15392 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
15393 assert_eq!(
15394 sync(&handler, &incumbent, 4, Vec::new()).await,
15395 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15396 );
15397 }
15398
15399 #[tokio::test]
15402 async fn closing_the_authority_connection_frees_sync_authority() {
15403 let supervisor = SupervisorHandle::new();
15404 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15405 let handler = ControlHandler::new(Arc::new(Registry::default()))
15406 .with_supervisor(supervisor.clone());
15407 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15408 sync(&handler, &first, 10, vec![head("s", 1)])
15409 .await
15410 .unwrap();
15411 handler.cleanup_connection(first.connection_id).unwrap();
15412
15413 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
15414 sync(&handler, &second, 1, vec![head("s", 1)])
15415 .await
15416 .expect("the next connection takes the released authority");
15417 }
15418
15419 #[tokio::test]
15420 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
15421 let supervisor = SupervisorHandle::new();
15422 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15423 let handler =
15424 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
15425 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
15426 sync(&handler, &first, 10, vec![head("s", 1)])
15427 .await
15428 .unwrap();
15429 handler
15430 .handle_control_frame(
15431 &first,
15432 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
15433 )
15434 .await
15435 .unwrap();
15436 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
15437 sync(&handler, &second, 1, vec![head("s", 1)])
15438 .await
15439 .expect("GOODBYE releases authority even if the old socket remains open");
15440 }
15441
15442 #[tokio::test]
15443 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
15444 let registry = Arc::new(Registry::default());
15445 let supervisor_handle = SupervisorHandle::new();
15446 let supervisor =
15447 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15448 .with_handle(supervisor_handle.clone())
15449 .with_daemon_incarnation("incarnation-7".to_string());
15450 supervisor
15453 .supervise_configured(
15454 ModuleSpec {
15455 module_id: OWNER.to_string(),
15456 program: PathBuf::from("/nonexistent/prefrontal-core"),
15457 args: Vec::new(),
15458 env: Vec::new(),
15459 reserved: false,
15460 reserved_prefixes: Vec::new(),
15461 protocol: ModuleProtocol::Subc,
15462 overlap: Default::default(),
15463 },
15464 false,
15465 )
15466 .unwrap();
15467 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
15468 let handler =
15469 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15470 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
15471
15472 let ModuleControlResponseToModule::ScopeDescribe {
15474 status,
15475 daemon_incarnation,
15476 owner_synced,
15477 owner_configured,
15478 scope,
15479 ..
15480 } = describe(&handler, &reader, OWNER, "s").await
15481 else {
15482 panic!("not a describe reply");
15483 };
15484 assert_eq!(status, ScopeStatus::NotLive);
15485 assert_eq!(daemon_incarnation, "incarnation-7");
15486 assert!(!owner_synced);
15487 assert!(owner_configured);
15488 assert!(scope.is_none());
15489
15490 let ModuleControlResponseToModule::ScopeDescribe {
15493 status,
15494 owner_configured,
15495 ..
15496 } = describe(&handler, &reader, "ghost", "s").await
15497 else {
15498 panic!("not a describe reply");
15499 };
15500 assert_eq!(status, ScopeStatus::NotLive);
15501 assert!(!owner_configured);
15502
15503 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
15505 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
15506 let ModuleControlResponseToModule::ScopeDescribe {
15507 status,
15508 scope_epoch,
15509 owner_synced,
15510 scope,
15511 ..
15512 } = describe(&handler, &reader, OWNER, "s").await
15513 else {
15514 panic!("not a describe reply");
15515 };
15516 assert_eq!(status, ScopeStatus::Live);
15517 assert_eq!(scope_epoch, Some(4));
15518 assert!(owner_synced);
15519 let stamp = scope.expect("a live scope carries its stamp");
15520 assert!(
15521 stamp.owner_authorized,
15522 "prefrontal-core is the default authority"
15523 );
15524 assert_eq!(stamp.kind, ScopeKind::Head);
15525 }
15526
15527 #[tokio::test]
15528 async fn scope_authority_owners_decides_owner_authorized() {
15529 let supervisor = SupervisorHandle::new();
15530 supervisor.set_spawn_nonce("broca", "b1".to_string());
15531 let handler = ControlHandler::new(Arc::new(Registry::default()))
15532 .with_supervisor(supervisor)
15533 .with_scope_authority_owners(vec!["broca".to_string()]);
15534 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
15535 let mut gated = head("s", 1);
15536 gated.attributes.agent_id = Some("agent".to_string());
15537 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
15538 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
15539 describe(&handler, &broca, "broca", "s").await
15540 else {
15541 panic!("not a describe reply");
15542 };
15543 assert!(scope.unwrap().owner_authorized);
15544 }
15545
15546 #[tokio::test]
15550 async fn scope_ops_and_the_scopes_capability_are_advertised() {
15551 let handler = ControlHandler::new(Arc::new(Registry::default()));
15552 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
15553 let ack = hello_via_sink(
15554 &handler,
15555 &ctx,
15556 &mut rx,
15557 hello_frame("m", PROTOCOL_VERSION, 1),
15558 )
15559 .await;
15560 let ack = parse_ack(&ack);
15561 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
15562 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
15563 }
15564 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
15565
15566 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
15567 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15568 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15569 let reply = handler
15570 .handle_control_frame(&client, frame)
15571 .await
15572 .unwrap()
15573 .pop()
15574 .unwrap();
15575 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15576 serde_json::from_slice(&reply.body).unwrap()
15577 else {
15578 panic!("not a server.describe reply");
15579 };
15580 assert!(
15581 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
15582 "{capabilities:?}"
15583 );
15584 }
15585
15586 const PLEXUS: &str = "plexus";
15589 const OTHER: &str = "other";
15590 const AFT: &str = "aft";
15591 const BROCA: &str = "broca";
15592 const MAGIC: &str = "magic-context";
15593
15594 fn nonce(module_id: &str) -> String {
15595 format!("nonce-{module_id}")
15596 }
15597
15598 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15599 let (tx, rx) = mpsc::channel(64);
15600 (
15601 RouteCtx {
15602 connection_id: ConnectionId::new(connection),
15603 egress: FrameSink::new(tx),
15604 },
15605 rx,
15606 )
15607 }
15608
15609 struct Rig {
15613 handler: ControlHandler,
15614 forwarding: Arc<ForwardingTable>,
15615 owner: RouteCtx,
15616 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15617 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
15618 generation: u64,
15619 next_connection: u64,
15620 _supervisor: Supervisor,
15621 }
15622
15623 async fn rig() -> Rig {
15624 rig_with_flow_support(true).await
15625 }
15626
15627 async fn rig_with_flow_support(flow_support: bool) -> Rig {
15628 rig_with_scope_capabilities(flow_support, false).await
15629 }
15630
15631 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
15632 let mut provides = Vec::new();
15633 if flow_support {
15634 provides.push("flow-scopes/v1");
15635 }
15636 if run_support {
15637 provides.push("agent-run-scopes/v1");
15638 }
15639 rig_with_target_capabilities(&provides).await
15640 }
15641
15642 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
15643 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
15644 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15645 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15648 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
15649 Frame::build(
15650 FrameType::Hello,
15651 control_flags(),
15652 0,
15653 0,
15654 connection,
15655 serde_json::to_vec(&body).unwrap(),
15656 )
15657 .unwrap()
15658 }
15659
15660 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
15661 let registry = Arc::new(Registry::default());
15662 let forwarding = Arc::new(ForwardingTable::default());
15663 let supervisor_handle = SupervisorHandle::new();
15664 let supervisor =
15665 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15666 .with_handle(supervisor_handle.clone());
15667 supervisor
15668 .supervise_configured(
15669 ModuleSpec {
15670 module_id: OWNER.to_string(),
15671 program: PathBuf::from("/nonexistent/prefrontal-core"),
15672 args: Vec::new(),
15673 env: Vec::new(),
15674 reserved: false,
15675 reserved_prefixes: Vec::new(),
15676 protocol: ModuleProtocol::Subc,
15677 overlap: Default::default(),
15678 },
15679 false,
15680 )
15681 .unwrap();
15682 for module_id in [OWNER, AFT, BROCA, MAGIC] {
15683 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
15684 }
15685 let handler =
15686 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15687 .with_supervisor(supervisor_handle);
15688 let (owner, mut owner_rx) = wide_ctx(1);
15689 hello_via_sink(
15690 &handler,
15691 &owner,
15692 &mut owner_rx,
15693 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
15694 )
15695 .await;
15696 let mut modules = BTreeMap::new();
15697 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
15698 let (ctx, mut rx) = wide_ctx(connection);
15699 let hello = hello_with_capabilities(module_id, connection, provides);
15700 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
15701 modules.insert(module_id.to_string(), (ctx, rx));
15702 }
15703 Rig {
15704 handler,
15705 forwarding,
15706 owner,
15707 _owner_rx: owner_rx,
15708 modules,
15709 generation: 0,
15710 next_connection: 100,
15711 _supervisor: supervisor,
15712 }
15713 }
15714
15715 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
15716 ScopeCarrier::new(Principal::Reserved {
15717 module_id: module_id.to_string(),
15718 })
15719 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
15720 }
15721
15722 fn session(scope_epoch: u64) -> ScopeRecord {
15725 let mut record = head("s", scope_epoch);
15726 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
15727 record.attributes.agent_id = Some("agent-1".to_string());
15728 record.attributes.delegates = true;
15729 record
15730 }
15731
15732 impl Rig {
15733 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
15734 self.generation += 1;
15735 sync(&self.handler, &self.owner, self.generation, scopes)
15736 .await
15737 .expect("the owner's sync is accepted");
15738 }
15739
15740 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15741 ScopeSelector::new(
15742 Principal::Reserved {
15743 module_id: OWNER.to_string(),
15744 },
15745 scope_ref.to_string(),
15746 scope_epoch,
15747 )
15748 }
15749
15750 fn open_frame(
15751 &mut self,
15752 opener: Option<&str>,
15753 target: &str,
15754 scope: Option<ScopeSelector>,
15755 ) -> (
15756 RouteCtx,
15757 mpsc::Receiver<crate::router::OutboundFrame>,
15758 Frame,
15759 ) {
15760 self.next_connection += 1;
15761 let (ctx, rx) = wide_ctx(self.next_connection);
15762 let root = unique_project_root("scoped-open");
15763 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15764 target: RouteTarget::ToolProvider {
15765 module_id: target.to_string(),
15766 },
15767 identity: BindIdentity::new(
15768 root.path().to_path_buf(),
15769 "unit".to_string(),
15770 "session".to_string(),
15771 ),
15772 consumer_identity: opener.map(|module_id| ConsumerIdentity {
15773 module_id: module_id.to_string(),
15774 launch_nonce: nonce(module_id),
15775 }),
15776 consumer_capabilities: None,
15777 role_versions: None,
15778 admission_facts: None,
15779 scope,
15780 })
15781 .unwrap();
15782 let frame = Frame::build(
15783 FrameType::Request,
15784 control_flags(),
15785 0,
15786 0,
15787 self.next_connection,
15788 body,
15789 )
15790 .unwrap();
15791 (ctx, rx, frame)
15792 }
15793
15794 async fn refused(
15796 &mut self,
15797 opener: Option<&str>,
15798 target: &str,
15799 scope: Option<ScopeSelector>,
15800 ) -> String {
15801 self.refusal_body(opener, target, scope).await["code"]
15802 .as_str()
15803 .unwrap()
15804 .to_string()
15805 }
15806
15807 async fn refusal_body(
15808 &mut self,
15809 opener: Option<&str>,
15810 target: &str,
15811 scope: Option<ScopeSelector>,
15812 ) -> Value {
15813 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
15814 let replies = tokio::time::timeout(
15815 Duration::from_secs(2),
15816 self.handler.handle_control_frame(&ctx, frame),
15817 )
15818 .await
15819 .expect("the open must be refused before waiting for a bind ack")
15820 .unwrap();
15821 assert_eq!(replies.len(), 1, "{replies:?}");
15822 assert_eq!(replies[0].header.ty, FrameType::Error);
15823 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15824 assert!(
15825 module_rx.try_recv().is_err(),
15826 "a refused open relays nothing"
15827 );
15828 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
15829 parse_error(&replies[0])
15830 }
15831
15832 async fn relayed(
15834 &mut self,
15835 opener: Option<&str>,
15836 target: &str,
15837 scope: Option<ScopeSelector>,
15838 ) -> Relayed {
15839 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
15840 let handler = self.handler.clone();
15841 let task_ctx = ctx.clone();
15842 let task = tokio::spawn(async move {
15843 handler
15844 .handle_control_frame(&task_ctx, frame)
15845 .await
15846 .unwrap()
15847 });
15848 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15849 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
15850 .await
15851 .expect("the target receives the relayed route.bind")
15852 .unwrap()
15853 .frame;
15854 Relayed {
15855 target: target.to_string(),
15856 client: ctx,
15857 client_rx: rx,
15858 task,
15859 bind,
15860 }
15861 }
15862
15863 async fn ack(&self, relayed: &Relayed) {
15864 let (module, _) = &self.modules[&relayed.target];
15865 self.handler
15866 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
15867 .await
15868 .unwrap();
15869 }
15870
15871 async fn bound(
15873 &mut self,
15874 opener: Option<&str>,
15875 target: &str,
15876 scope: Option<ScopeSelector>,
15877 ) -> Bound {
15878 let relayed = self.relayed(opener, target, scope).await;
15879 self.ack(&relayed).await;
15880 let Relayed {
15881 target,
15882 client,
15883 mut client_rx,
15884 task,
15885 bind,
15886 } = relayed;
15887 assert!(
15888 task.await.unwrap().is_empty(),
15889 "the open is answered by commit"
15890 );
15891 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
15892 Bound {
15893 target,
15894 client,
15895 client_rx,
15896 channel,
15897 epoch,
15898 bind,
15899 }
15900 }
15901
15902 fn live(&self, route: &Bound) -> bool {
15903 matches!(
15904 self.forwarding
15905 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15906 .unwrap(),
15907 DataRoute::Client(DataRouteState::Bound(_))
15908 )
15909 }
15910 }
15911
15912 struct Relayed {
15913 target: String,
15914 client: RouteCtx,
15915 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15916 task: tokio::task::JoinHandle<Vec<Frame>>,
15917 bind: Frame,
15918 }
15919
15920 struct Bound {
15921 target: String,
15922 client: RouteCtx,
15923 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15924 channel: u16,
15925 epoch: u32,
15926 bind: Frame,
15927 }
15928
15929 impl Bound {
15930 fn closed_reason(&mut self) -> RouteCloseReason {
15933 let mut reason = None;
15934 let mut goodbye = false;
15935 while let Ok(outbound) = self.client_rx.try_recv() {
15936 let frame = outbound.frame;
15937 match frame.header.ty {
15938 FrameType::Goodbye => {
15939 assert_eq!(
15940 (frame.header.channel, frame.header.epoch),
15941 (self.channel, self.epoch)
15942 );
15943 goodbye = true;
15944 }
15945 FrameType::Push => {
15946 let ClientControlPush::RouteClosed {
15947 reason: r,
15948 module_id,
15949 ..
15950 } = serde_json::from_slice(&frame.body).unwrap()
15951 else {
15952 panic!("unexpected push");
15953 };
15954 assert_eq!(module_id, self.target);
15955 reason = Some(r);
15956 }
15957 other => panic!("unexpected frame {other:?}"),
15958 }
15959 }
15960 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
15961 reason.expect("the client is told why the route closed")
15962 }
15963
15964 fn untouched(&mut self) -> bool {
15965 self.client_rx.try_recv().is_err()
15966 }
15967
15968 fn stamp(&self) -> Option<ScopeStamp> {
15969 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
15970 ModuleControlRequest::RouteBind { scope, .. } => scope,
15971 other => panic!("expected a route.bind, got {other:?}"),
15972 }
15973 }
15974 }
15975
15976 #[tokio::test]
15977 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
15978 ) {
15979 let mut rig = rig().await;
15980 let mut record = session(1);
15981 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
15982 record.child_owners = vec![Principal::Reserved {
15983 module_id: MAGIC.to_string(),
15984 }];
15985 rig.sync(vec![record]).await;
15986 let scope = || Some(rig_selector("s", Some(1)));
15987
15988 rig.bound(Some(OWNER), PLEXUS, scope()).await;
15991 rig.bound(Some(AFT), OTHER, scope()).await;
15992 rig.bound(Some(BROCA), PLEXUS, scope()).await;
15993
15994 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
15998 assert_eq!(
15999 rig.refused(opener, target, scope()).await,
16000 error_codes::SCOPE_NOT_CARRIER,
16001 "{opener:?} -> {target}"
16002 );
16003 }
16004 }
16005
16006 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
16007 ScopeSelector::new(
16008 Principal::Reserved {
16009 module_id: OWNER.to_string(),
16010 },
16011 scope_ref.to_string(),
16012 scope_epoch,
16013 )
16014 }
16015
16016 #[tokio::test]
16017 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
16018 let mut rig = rig().await;
16019 rig.sync(vec![session(1)]).await;
16020 for opener in [OWNER, AFT] {
16021 assert_eq!(
16022 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
16023 .await,
16024 error_codes::SCOPE_EPOCH_REQUIRED,
16025 "{opener}"
16026 );
16027 }
16028 }
16029
16030 #[tokio::test]
16031 async fn admission_separates_not_synced_not_live_and_ended() {
16032 let mut rig = rig().await;
16033 let code = rig
16035 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16036 .await;
16037 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
16038 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
16039
16040 let ghost = ScopeSelector::new(
16042 Principal::Reserved {
16043 module_id: "ghost".to_string(),
16044 },
16045 "s".to_string(),
16046 Some(1),
16047 );
16048 assert_eq!(
16049 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
16050 error_codes::SCOPE_NOT_LIVE
16051 );
16052
16053 rig.sync(vec![session(2)]).await;
16054 assert_eq!(
16055 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
16056 .await,
16057 error_codes::SCOPE_NOT_LIVE
16058 );
16059 for epoch in [1, 3] {
16060 assert_eq!(
16061 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
16062 .await,
16063 error_codes::SCOPE_ENDED,
16064 "epoch {epoch}"
16065 );
16066 }
16067 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
16069 .await;
16070 }
16071
16072 #[tokio::test]
16073 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
16074 let mut rig = rig().await;
16075 rig.sync(vec![session(1)]).await;
16076 let route = rig
16077 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16078 .await;
16079 let stamp = route.stamp().expect("a scoped bind carries the stamp");
16080 assert_eq!(stamp.scope_ref, "s");
16081 assert_eq!(stamp.scope_epoch, 1);
16082 assert_eq!(stamp.kind, ScopeKind::Head);
16083 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
16084 assert!(stamp.attributes.delegates);
16085 assert!(stamp.owner_authorized);
16086 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16087 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
16088
16089 let (broca, mut broca_rx) = wide_ctx(50);
16092 hello_via_sink(
16093 &rig.handler,
16094 &broca,
16095 &mut broca_rx,
16096 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
16097 )
16098 .await;
16099 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
16100 .await
16101 .unwrap();
16102 let own = ScopeSelector::new(
16103 Principal::Reserved {
16104 module_id: BROCA.to_string(),
16105 },
16106 "b".to_string(),
16107 Some(1),
16108 );
16109 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
16110 assert!(!route.stamp().unwrap().owner_authorized);
16111 }
16112
16113 #[tokio::test]
16114 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
16115 let mut rig = rig().await;
16116 let mut record = head("s", 1);
16117 record.carriers = vec![carrier(AFT, None)];
16118 let flow_id = "Flow:run-7/step_2!~";
16119 record.attributes.flow_id = Some(flow_id.to_string());
16120 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
16121 .await
16122 .unwrap();
16123 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16124 panic!("not a sync reply");
16125 };
16126 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
16127 let route = rig
16128 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16129 .await;
16130 assert!(rig.live(&route), "the stamped bind committed");
16131 let stamp = route.stamp().expect("a flow scope carries a stamp");
16132 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
16133 assert_eq!(stamp.attributes.agent_id, None);
16134 assert!(!stamp.attributes.delegates);
16135 assert!(stamp.owner_authorized);
16136 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16137 assert_eq!(unscoped.stamp(), None);
16138 }
16139
16140 #[tokio::test]
16141 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
16142 let mut rig = rig_with_flow_support(false).await;
16143 let mut record = session(1);
16144 record.attributes.flow_id = Some("flow:7".to_string());
16145 rig.sync(vec![record]).await;
16146 for opener in [OWNER, AFT] {
16147 let body = rig
16148 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16149 .await;
16150 assert_eq!(body["code"], "target_flow_unsupported");
16151 let message = body["message"].as_str().unwrap();
16152 for required in [PLEXUS, "flow-scopes/v1"] {
16153 assert!(message.contains(required), "{message}");
16154 }
16155 }
16156 }
16157
16158 #[tokio::test]
16159 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
16160 let mut rig = rig_with_flow_support(true).await;
16161 let mut record = session(1);
16162 record.attributes.flow_id = Some("flow:7".to_string());
16163 rig.sync(vec![record]).await;
16164 for opener in [OWNER, AFT] {
16165 let route = rig
16166 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16167 .await;
16168 assert!(rig.live(&route));
16169 assert_eq!(
16170 route.stamp().unwrap().attributes.flow_id.as_deref(),
16171 Some("flow:7")
16172 );
16173 }
16174 }
16175
16176 #[tokio::test]
16177 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
16178 use std::future::Future;
16179
16180 let mut rig = rig_with_flow_support(true).await;
16181 let mut record = session(1);
16182 record.attributes.flow_id = Some("flow:7".to_string());
16183 rig.sync(vec![record]).await;
16184 let (client, mut client_rx, frame) =
16185 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
16186 for _ in 0..64 {
16189 client.egress.try_send(route_bind_ack(1)).unwrap();
16190 }
16191 let handler = rig.handler.clone();
16192 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16193 std::future::poll_fn(|cx| {
16194 assert!(open.as_mut().poll(cx).is_pending());
16195 std::task::Poll::Ready(())
16196 })
16197 .await;
16198 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16199
16200 let old_connection = rig.modules[PLEXUS].0.connection_id;
16201 rig.handler.cleanup_connection(old_connection).unwrap();
16202 let (replacement, mut replacement_rx) = wide_ctx(200);
16203 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
16204 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16205 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16206 let hello = Frame::build(
16207 FrameType::Hello,
16208 control_flags(),
16209 0,
16210 0,
16211 200,
16212 serde_json::to_vec(&body).unwrap(),
16213 )
16214 .unwrap();
16215 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
16216
16217 client_rx.try_recv().unwrap();
16218 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16219 .await
16220 .expect("the replacement is refused without waiting for a bind ack")
16221 .unwrap();
16222 assert_eq!(replies.len(), 1);
16223 let body = parse_error(&replies[0]);
16224 assert_eq!(body["code"], "target_flow_unsupported");
16225 for required in [PLEXUS, "flow-scopes/v1"] {
16226 assert!(body["message"].as_str().unwrap().contains(required));
16227 }
16228 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
16229 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16230 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16231 assert!(rig
16232 .handler
16233 .registry
16234 .get_module_by_connection(replacement.connection_id)
16235 .unwrap()
16236 .is_some());
16237 }
16238
16239 #[tokio::test]
16240 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
16241 use std::future::Future;
16242 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16243 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16244 const NEXT_HASH: &str =
16245 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16246 let mut rig = rig().await;
16247 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
16248 record.carriers.push(ScopeCarrier::new(Principal::Direct));
16249 rig.sync(vec![record.clone()]).await;
16250 let (client, mut rx, frame) = rig.open_frame(
16251 None,
16252 PLEXUS,
16253 Some(
16254 rig.selector("s", Some(1))
16255 .with_bind_proof(Some(PROOF.into())),
16256 ),
16257 );
16258 for _ in 0..64 {
16261 client.egress.try_send(route_bind_ack(1)).unwrap();
16262 }
16263 let handler = rig.handler.clone();
16264 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16265 std::future::poll_fn(|cx| {
16266 assert!(open.as_mut().poll(cx).is_pending());
16267 std::task::Poll::Ready(())
16268 })
16269 .await;
16270 record.bind_proof_sha256 = Some(NEXT_HASH.into());
16271 rig.sync(vec![record]).await;
16272 rx.try_recv().unwrap();
16273 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16274 .await
16275 .unwrap()
16276 .unwrap();
16277 assert_eq!(
16278 parse_error(&replies[0])["code"],
16279 error_codes::SCOPE_BIND_PROOF_MISMATCH
16280 );
16281 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
16282 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16283 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16284 }
16285
16286 #[tokio::test]
16287 async fn scope_bind_proof_reserved_openers_ignore_proof() {
16288 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16289 let mut rig = rig().await;
16290 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16291 .await;
16292 for opener in [OWNER, AFT] {
16293 for proof in [None, Some("malformed+proof=")] {
16294 let route = rig
16295 .bound(
16296 Some(opener),
16297 PLEXUS,
16298 Some(
16299 rig.selector("s", Some(1))
16300 .with_bind_proof(proof.map(str::to_string)),
16301 ),
16302 )
16303 .await;
16304 assert!(rig.live(&route));
16305 }
16306 }
16307 }
16308
16309 #[tokio::test]
16310 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
16311 use std::future::Future;
16312 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16313 const NEXT_HASH: &str =
16314 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16315 let mut rig = rig().await;
16316 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16317 .await;
16318 let (client, mut rx, frame) =
16319 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
16320 for _ in 0..64 {
16321 client.egress.try_send(route_bind_ack(1)).unwrap();
16322 }
16323 let handler = rig.handler.clone();
16324 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16325 std::future::poll_fn(|cx| {
16326 assert!(open.as_mut().poll(cx).is_pending());
16327 std::task::Poll::Ready(())
16328 })
16329 .await;
16330 rig.sync(vec![
16331 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
16332 ])
16333 .await;
16334 rx.try_recv().unwrap();
16335 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16336 let bind = std::future::poll_fn(|cx| {
16337 assert!(open.as_mut().poll(cx).is_pending());
16338 module_rx.poll_recv(cx)
16339 })
16340 .await
16341 .unwrap()
16342 .frame;
16343 handler
16344 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
16345 .await
16346 .unwrap();
16347 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16348 .await
16349 .unwrap()
16350 .unwrap();
16351 assert_eq!(replies.len(), 1);
16352 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
16355 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16356 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
16357 let route = rig
16358 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
16359 .await;
16360 assert!(rig.live(&route));
16361 }
16362
16363 #[tokio::test]
16364 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
16365 {
16366 let mut rig = rig_with_flow_support(false).await;
16367 rig.sync(vec![session(1)]).await;
16368 let route = rig
16369 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16370 .await;
16371 assert!(rig.live(&route));
16372 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
16373 let mut record = session(1);
16374 record.attributes.flow_id = Some("flow:7".to_string());
16375 rig.sync(vec![record]).await;
16376 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
16377 assert!(rig.live(&unscoped));
16378 assert_eq!(unscoped.stamp(), None);
16379 }
16380
16381 #[tokio::test]
16382 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
16383 let mut rig = rig().await;
16384 let mut record = session(1);
16385 record.attributes.flow_id = Some("flow:7".to_string());
16386 rig.sync(vec![record.clone()]).await;
16387 let before = rig.forwarding.published_scope_tag(OWNER, "s");
16388 let mut owner_route = rig
16389 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16390 .await;
16391 let mut carrier_route = rig
16392 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
16393 .await;
16394 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16395 rig.sync(vec![record.clone()]).await;
16396 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16397 assert!(rig.live(&owner_route) && owner_route.untouched());
16398 assert!(rig.live(&carrier_route) && carrier_route.untouched());
16399
16400 record.attributes.flow_id = Some("flow:8".to_string());
16401 rig.sync(vec![record]).await;
16402 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
16403 let before = before.unwrap();
16404 assert_eq!(after.scope_epoch, before.scope_epoch);
16405 assert!(after.version > before.version);
16406 for route in [&mut owner_route, &mut carrier_route] {
16407 assert!(!rig.live(route));
16408 assert_eq!(
16409 route.closed_reason(),
16410 RouteCloseReason::ScopeDelegationChanged
16411 );
16412 }
16413 assert!(rig.live(&unscoped) && unscoped.untouched());
16414 for target in [PLEXUS, OTHER] {
16417 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
16418 let goodbye = module_rx
16419 .try_recv()
16420 .expect("the provider sees the drain")
16421 .frame;
16422 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16423 assert!(module_rx.try_recv().is_err());
16424 }
16425 let rebound = rig
16426 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16427 .await;
16428 assert_eq!(
16429 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
16430 Some("flow:8")
16431 );
16432 }
16433
16434 #[tokio::test]
16439 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
16440 let mut rig = rig().await;
16441 rig.sync(vec![session(1)]).await;
16442 let mut cotenant = rig
16443 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16444 .await;
16445
16446 let mut changed = session(1);
16447 changed.child_owners.push(Principal::Reserved {
16448 module_id: MAGIC.to_string(),
16449 });
16450 let mut ended = None;
16451 for (code, next) in [
16452 (error_codes::SCOPE_CHANGED, vec![changed]),
16453 (error_codes::SCOPE_ENDED, Vec::new()),
16454 ] {
16455 let relayed = rig
16456 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16457 .await;
16458 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
16459 ended = Some(next.is_empty());
16460 rig.sync(next).await;
16461 rig.ack(&relayed).await;
16462 let replies = relayed.task.await.unwrap();
16463 assert_eq!(replies.len(), 1, "{replies:?}");
16464 assert_eq!(parse_error(&replies[0])["code"], code);
16465 assert_eq!(
16466 subc_protocol::error_codes::is_retryable_route_open(code),
16467 code == error_codes::SCOPE_CHANGED
16468 );
16469 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16470 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16472 let mut goodbyes = Vec::new();
16475 while let Ok(outbound) = plexus_rx.try_recv() {
16476 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
16477 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
16478 }
16479 assert!(
16480 goodbyes.contains(&(bind_channel, bind_epoch)),
16481 "{goodbyes:?}"
16482 );
16483 assert!(rig
16484 .handler
16485 .registry
16486 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
16487 .unwrap()
16488 .is_some());
16489 }
16490 assert_eq!(ended, Some(true));
16491 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
16494 }
16495
16496 #[tokio::test]
16499 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
16500 struct Case {
16501 name: &'static str,
16502 change: fn(&mut ScopeRecord),
16503 closed: [Option<RouteCloseReason>; 4],
16506 }
16507 use RouteCloseReason::*;
16508 let cases = [
16509 Case {
16510 name: "a carrier entry removed",
16511 change: |r| {
16512 r.carriers.retain(|c| {
16513 c.principal
16514 != Principal::Reserved {
16515 module_id: AFT.to_string(),
16516 }
16517 })
16518 },
16519 closed: [None, Some(ScopeCarrierRemoved), None, None],
16520 },
16521 Case {
16522 name: "a target removed from a carrier",
16523 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
16524 closed: [None, None, None, Some(ScopeCarrierRemoved)],
16525 },
16526 Case {
16527 name: "a bare carrier narrowed to targets",
16528 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
16529 closed: [None, Some(ScopeCarrierRemoved), None, None],
16530 },
16531 Case {
16532 name: "delegates turned off",
16533 change: |r| r.attributes.delegates = false,
16534 closed: [Some(ScopeDelegationChanged); 4],
16535 },
16536 Case {
16537 name: "agent_id changed",
16538 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
16539 closed: [Some(ScopeDelegationChanged); 4],
16540 },
16541 Case {
16542 name: "a carrier added, child owners changed, the record re-sent",
16543 change: |r| {
16544 r.carriers.push(carrier(MAGIC, None));
16545 r.child_owners.push(Principal::Reserved {
16546 module_id: MAGIC.to_string(),
16547 });
16548 },
16549 closed: [None; 4],
16550 },
16551 Case {
16552 name: "a target added",
16553 change: |r| {
16554 r.carriers[1]
16555 .targets
16556 .as_mut()
16557 .unwrap()
16558 .push("third".to_string())
16559 },
16560 closed: [None; 4],
16561 },
16562 Case {
16563 name: "delegates turned on",
16564 change: |r| r.attributes.delegates = true,
16565 closed: [None; 4],
16566 },
16567 ];
16568 for case in cases {
16569 let mut rig = rig().await;
16570 rig.sync(vec![session(1)]).await;
16571 let scope = || Some(rig_selector("s", Some(1)));
16572 let mut routes = [
16573 rig.bound(Some(OWNER), PLEXUS, scope()).await,
16574 rig.bound(Some(AFT), PLEXUS, scope()).await,
16575 rig.bound(Some(BROCA), PLEXUS, scope()).await,
16576 rig.bound(Some(BROCA), OTHER, scope()).await,
16577 ];
16578 let mut record = session(1);
16579 (case.change)(&mut record);
16580 rig.sync(vec![record]).await;
16581 for (index, expected) in case.closed.iter().enumerate() {
16582 let route = &mut routes[index];
16583 match expected {
16584 Some(reason) => {
16585 assert!(!rig.live(route), "{}: route {index} still live", case.name);
16586 assert_eq!(
16587 route.closed_reason(),
16588 *reason,
16589 "{}: route {index}",
16590 case.name
16591 );
16592 }
16593 None => {
16594 assert!(rig.live(route), "{}: route {index} closed", case.name);
16595 assert!(
16596 route.untouched(),
16597 "{}: route {index} was told something",
16598 case.name
16599 );
16600 }
16601 }
16602 }
16603 }
16604 }
16605
16606 #[tokio::test]
16607 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
16608 for next in [Vec::new(), vec![session(2)]] {
16610 let mut rig = rig().await;
16611 rig.sync(vec![session(1)]).await;
16612 let mut route = rig
16613 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16614 .await;
16615 rig.sync(next).await;
16616 assert!(!rig.live(&route));
16617 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
16618 }
16619
16620 let mut rig = rig().await;
16623 let mut child = session(1);
16624 child.scope_ref = "child".to_string();
16625 child.kind = ScopeKind::Worker;
16626 child.parent = Some(ScopeParent::new(
16627 Principal::Reserved {
16628 module_id: OWNER.to_string(),
16629 },
16630 "s".to_string(),
16631 1,
16632 ));
16633 rig.sync(vec![session(1), child.clone()]).await;
16634 let mut child_route = rig
16635 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
16636 .await;
16637 assert_eq!(
16638 child_route.stamp().unwrap().parent_state,
16639 Some(ParentState::Linked)
16640 );
16641 rig.sync(vec![child]).await;
16642 assert!(!rig.live(&child_route));
16643 assert_eq!(
16644 child_route.closed_reason(),
16645 RouteCloseReason::ScopeParentEnded
16646 );
16647 }
16648
16649 #[tokio::test]
16650 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
16651 ) {
16652 let mut rig = rig().await;
16653 rig.sync(vec![session(1)]).await;
16654 let mut route = rig
16655 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16656 .await;
16657 let before = rig.forwarding.published_scope_tag(OWNER, "s");
16658 rig.sync(vec![session(1)]).await;
16659 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16660 assert!(rig.live(&route) && route.untouched());
16661
16662 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
16666 .forwarding
16667 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16668 .unwrap()
16669 else {
16670 panic!("the route is bound");
16671 };
16672 binding.flow.acquire_tagged(9, false).await.unwrap();
16673 let mut widened = session(1);
16674 widened.carriers.push(carrier(MAGIC, None));
16675 rig.sync(vec![widened]).await;
16676 assert!(rig.live(&route) && route.untouched());
16677 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16678 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
16679 assert_eq!(binding.flow.in_flight(), 1);
16682 binding
16683 .flow
16684 .acquire_tagged(10, false)
16685 .await
16686 .expect("the flow is still open");
16687 }
16688
16689 #[tokio::test]
16692 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
16693 let mut rig = rig().await;
16694 rig.sync(vec![session(1)]).await;
16695 let mut on_incumbent = rig
16696 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16697 .await;
16698
16699 let (candidate, _candidate_rx) = wide_ctx(9);
16702 let registration = rig
16703 .handler
16704 .registry
16705 .register_candidate_with_control_ops(
16706 manifest(PLEXUS, PROTOCOL_VERSION),
16707 PROTOCOL_VERSION,
16708 candidate.connection_id,
16709 module_baseline_control_ops(),
16710 )
16711 .unwrap();
16712 rig.forwarding
16713 .register_candidate_module_connection(
16714 candidate.connection_id,
16715 PLEXUS.to_string(),
16716 PROTOCOL_VERSION,
16717 manifest_concurrency(®istration.manifest),
16718 candidate.egress.clone(),
16719 )
16720 .unwrap();
16721 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
16722 rig.handler
16723 .registry
16724 .promote_candidate(PLEXUS)
16725 .unwrap()
16726 .unwrap();
16727 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
16728
16729 rig.sync(Vec::new()).await;
16730 assert!(!rig.live(&on_incumbent));
16731 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
16732 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16733 let goodbye = incumbent_rx
16734 .try_recv()
16735 .expect("the superseded endpoint is told")
16736 .frame;
16737 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16738 }
16739 }
16740}
16741
16742#[cfg(test)]
16743mod concurrency_default_exposure_tests {
16744 use super::*;
16745
16746 fn hello_body(role_json: &str) -> Vec<u8> {
16747 format!(
16748 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
16749 )
16750 .into_bytes()
16751 }
16752
16753 fn manifest_from(body: &[u8]) -> ModuleManifest {
16754 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
16755 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
16756 .expect("manifest parses")
16757 }
16758
16759 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
16760
16761 #[test]
16762 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
16763 let body = hello_body(&format!(
16764 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
16765 ));
16766 let manifest = manifest_from(&body);
16767 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16770 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
16771 }
16772
16773 #[test]
16774 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
16775 let body = hello_body(&format!(
16776 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
16777 ));
16778 let manifest = manifest_from(&body);
16779 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16780 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16781 }
16782
16783 #[test]
16784 fn non_management_roles_are_never_reported() {
16785 let body = hello_body(
16786 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
16787 );
16788 let manifest = manifest_from(&body);
16789 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16790 }
16791}