1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_flow_operations, validate_hello_rich_ui_descriptor,
26 validate_hello_role_declarations, validate_hello_self_signal_declarations,
27 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
28 ProviderRole,
29 },
30 scope::{
31 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
32 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
33 SCOPE_SYNC_OP,
34 },
35 session::{
36 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
37 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
38 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
39 ROLE_VERSIONS_FIELD,
40 },
41 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
42 Priority, RouteTarget, PROTOCOL_VERSION,
43};
44use tokio::time::{timeout_at, Instant};
45use tracing::{debug, info, warn};
46
47use crate::{
48 caller_identity::CallerIdentity,
49 capability_requirements::{
50 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
51 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
52 RequirementStatus, RuntimeModule,
53 },
54 child_roster::DaemonShutdownFlag,
55 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
56 forwarding::{
57 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
58 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
59 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
60 },
61 observability::{
62 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
63 },
64 privileged_prompt::{prompt_text, PrivilegedOperation},
65 provenance::{
66 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
67 },
68 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
69 router::{daemon_error_flags, RouteCtx, RouterError},
70 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
71 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
72 stderr_tail::{CaptureState, TailEntry},
73 supervise::{
74 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
75 SupervisorHandle, SwapHelloAdmission,
76 },
77 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
78};
79
80pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
86
87const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
88const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
89const CAP_PING_PONG: &str = "ping_pong_v1";
90const CAP_SESSION_ATTACH: &str = "session_attach_v1";
91const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
92
93const SUBC_CONTROL_OPS: &[&str] = &[
94 ops::ACTIVITY_RECORD,
95 ops::ACTIVITY_QUERY,
96 ops::ACTIVITY_SUBSCRIBE,
97 ops::SERVER_DESCRIBE,
98 ops::CATALOG_LIST,
99 ops::ROUTE_OPEN,
100 ops::ROUTE_POLL,
101 ops::ROUTE_CLOSING,
102 ops::ROUTE_CLOSED,
103 ops::SUPERVISOR_LIST,
104 ops::SUPERVISOR_RESTART,
105 ops::SUPERVISOR_SWAP,
106 ops::SUPERVISOR_RELOAD,
107 ops::SUPERVISOR_RESCAN,
108 ops::SUPERVISOR_RELEASE_RESERVED,
109 ops::SUPERVISOR_SET_ENABLED,
110 ops::SUPERVISOR_HEALTH_PROBE,
111 ops::SUPERVISOR_HEALTH,
112 ops::SUPERVISOR_STDERR_TAIL,
113 ops::SUPERVISOR_TERMINALS,
114 ops::SUPERVISOR_ROUTES,
115 ops::SUPERVISOR_PROVENANCE,
116 ops::SUPERVISOR_SPAWN_SNAPSHOT,
117 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
118];
119
120const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
127
128mod rescan_checked;
129use rescan_checked::{RescanMode, RescanPlan};
130
131const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
132 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
133 "supervisor.live_roots",
134 SCOPE_SYNC_OP,
135 SCOPE_APPLY_OP,
136 SCOPE_DESCRIBE_OP,
137 "operator.confirm",
138];
139
140const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
144
145const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
146
147pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
155
156pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
171
172pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
190
191const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
192const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
193
194fn reload_verdict(
195 configured: &Path,
196 spawned_from: Option<&Path>,
197 image: subc_control::RunningImageAgreement,
198) -> PendingReloadVerdict {
199 let path = match spawned_from {
200 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
201 ReloadPathAgreement::Match
202 }
203 Some(spawned_from) => ReloadPathAgreement::Mismatch {
204 configured: configured.to_path_buf(),
205 spawned_from: spawned_from.to_path_buf(),
206 },
207 None => ReloadPathAgreement::Unavailable {
208 reason: if matches!(
209 image,
210 subc_control::RunningImageAgreement::Unavailable {
211 reason: subc_control::RunningImageUnavailableReason::NotRunning
212 }
213 ) {
214 ReloadPathUnavailableReason::NotRunning
215 } else {
216 ReloadPathUnavailableReason::SpawnedPathUnavailable
217 },
218 },
219 };
220 PendingReloadVerdict {
221 path,
222 image,
223 spec: None,
224 }
225}
226
227fn launch_spec_agreement(
228 configured: &crate::supervise::ModuleSpec,
229 running: bool,
230 spawned: Option<&crate::supervise::LaunchSpecDigest>,
231) -> subc_control::LaunchSpecAgreement {
232 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
233 if !running {
234 return Agreement::Unavailable {
235 reason: Reason::NotRunning,
236 };
237 }
238 match spawned {
239 None => Agreement::Unavailable {
240 reason: Reason::SpawnedSpecUnavailable,
241 },
242 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
243 Agreement::Match
244 }
245 Some(_) => Agreement::Mismatch,
246 }
247}
248
249fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
250 if configured == spawned_from {
251 return true;
252 }
253 #[cfg(windows)]
254 {
255 match (
261 std::fs::canonicalize(configured),
262 std::fs::canonicalize(spawned_from),
263 ) {
264 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
265 _ => false,
266 }
267 }
268 #[cfg(not(windows))]
269 {
270 false
271 }
272}
273
274#[derive(Clone)]
275struct DaemonProvenanceFacts {
276 build: DaemonBuildProvenance,
277 pid: Option<u32>,
278 started_at_ms: Option<u64>,
279 start_clock: Option<crate::clock::StartClock>,
280 executable_path: Option<PathBuf>,
281 executable_identity: Option<SpawnedFileIdentity>,
282 process_start_time: Option<u64>,
283 probe: ExecutableIdentityProbe,
284}
285
286impl Default for DaemonProvenanceFacts {
287 fn default() -> Self {
288 Self {
289 build: DaemonBuildProvenance {
290 build_git_sha: None,
291 build_lock_digest: None,
292 },
293 pid: None,
294 started_at_ms: None,
295 start_clock: None,
296 executable_path: None,
297 executable_identity: None,
298 process_start_time: None,
299 probe: ExecutableIdentityProbe::default(),
300 }
301 }
302}
303
304#[derive(Debug, Clone)]
305struct SupervisorRescanContext {
306 supervisor: Supervisor,
307 config_path: PathBuf,
308 configured_port: Option<u16>,
309 storage_config: Option<crate::daemon_config::StorageConfig>,
310 admission_facts_carrier_module_id: Option<String>,
311 admission_facts_targets: Option<Vec<String>>,
312 scope_authority_owners: Vec<String>,
313 privileged_operator_prompt: PrivilegedOperatorPrompt,
314 generation: Arc<std::sync::atomic::AtomicU64>,
320}
321
322const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
343 "reloading",
344 "supervisor_not_live",
345 "registration_not_active",
346 "no_forwarding_connection",
347 "relay_send_failed",
348];
349
350#[derive(Clone)]
352pub struct ControlHandler {
353 registry: Arc<Registry>,
354 activity: Option<crate::activity::ActivityLedger>,
355 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
356 forwarding: Arc<ForwardingTable>,
357 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
358 supervisor: SupervisorHandle,
359 subc_capabilities: Arc<[String]>,
360 route_bind_relay_timeout: Duration,
364 route_bind_relay_timeouts: BTreeMap<String, Duration>,
368 route_bind_breakers: RouteBindBreakers,
371 route_bind_concurrency: RouteBindConcurrency,
374 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
379 route_bind_breaker_threshold: u32,
381 route_bind_breaker_cooldown: Duration,
383 health_probe_timeout: Duration,
384 storage_config: Option<crate::daemon_config::StorageConfig>,
387 machine_id: Option<crate::machine_id::MachineId>,
391 admission_facts_carrier_module_id: Option<String>,
392 admission_facts_targets: Option<Vec<String>>,
393 scopes: Arc<RwLock<ScopeTable>>,
397 scope_deadline_added: Arc<tokio::sync::Notify>,
401 scope_authority_owners: Vec<String>,
404 privileged_operator_prompt: PrivilegedOperatorPrompt,
406 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
409 #[cfg(test)]
410 caller_lookup: Option<TestCallerLookup>,
411 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
414 rescan: Option<SupervisorRescanContext>,
415 connected_clients: ConnectedClients,
416 counters: DaemonCounters,
417 local_endpoint_unavailable: Option<String>,
420 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
421 daemon_shutdown: DaemonShutdownFlag,
426 daemon_provenance: DaemonProvenanceFacts,
427 #[cfg(test)]
428 control_dispatch_delay: Option<Duration>,
429 #[cfg(test)]
430 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
431 #[cfg(test)]
435 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
436 #[cfg(test)]
442 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
443}
444
445impl fmt::Debug for ControlHandler {
446 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
447 f.debug_struct("ControlHandler")
448 .field("registry", &self.registry)
449 .field("forwarding", &self.forwarding)
450 .field("process_liveness", &self.process_liveness.is_some())
451 .field("supervisor", &self.supervisor)
452 .field("subc_capabilities", &self.subc_capabilities)
453 .finish()
454 }
455}
456
457struct RouteOpenRequest {
458 target: RouteTarget,
459 identity: BindIdentity,
460 consumer_identity: Option<ConsumerIdentity>,
461 consumer_capabilities: Option<Vec<String>>,
462 role_versions: Option<BTreeMap<String, String>>,
463 admission_facts: Option<serde_json::Value>,
464 scope: Option<ScopeSelector>,
465}
466
467#[derive(Clone, Copy)]
468enum RouteOpenPhase {
469 Admission,
470 ProjectRoot,
471 EgressReservation,
472 BindSend,
473 BindAckWait,
474}
475
476struct RouteOpenTimings {
480 module_id: String,
481 principal: String,
482 admission: Duration,
483 project_root: Duration,
484 egress_reservation: Duration,
485 bind_send: Duration,
486 bind_ack_wait: Duration,
487 phase: Option<RouteOpenPhase>,
488 phase_started_at: Instant,
489}
490
491impl RouteOpenTimings {
492 fn new() -> Self {
493 Self {
494 module_id: String::new(),
495 principal: "unresolved".to_string(),
497 admission: Duration::ZERO,
498 project_root: Duration::ZERO,
499 egress_reservation: Duration::ZERO,
500 bind_send: Duration::ZERO,
501 bind_ack_wait: Duration::ZERO,
502 phase: Some(RouteOpenPhase::Admission),
503 phase_started_at: Instant::now(),
504 }
505 }
506
507 fn finish(&mut self) {
508 let elapsed = self.phase_started_at.elapsed();
509 match self.phase.take() {
510 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
511 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
512 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
513 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
514 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
515 None => {}
516 }
517 }
518
519 fn enter(&mut self, phase: RouteOpenPhase) {
520 self.finish();
521 self.phase = Some(phase);
522 self.phase_started_at = Instant::now();
523 }
524}
525
526struct RouteBindReservationGuard {
527 forwarding: Arc<ForwardingTable>,
528 endpoint: ModuleEndpointId,
529 relay_corr: u64,
530 armed: bool,
531}
532
533struct ModuleControlRpcGuard {
534 forwarding: Arc<ForwardingTable>,
535 endpoint: ModuleEndpointId,
536 corr: u64,
537 armed: bool,
538}
539
540impl ModuleControlRpcGuard {
541 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
542 Self {
543 forwarding,
544 endpoint,
545 corr,
546 armed: true,
547 }
548 }
549
550 fn disarm(&mut self) {
551 self.armed = false;
552 }
553}
554
555impl Drop for ModuleControlRpcGuard {
556 fn drop(&mut self) {
557 if self.armed {
558 let _ = self
559 .forwarding
560 .cancel_module_control_rpc(self.endpoint, self.corr);
561 }
562 }
563}
564
565impl RouteBindReservationGuard {
566 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
567 Self {
568 forwarding,
569 endpoint,
570 relay_corr,
571 armed: true,
572 }
573 }
574
575 fn release_and_disarm(&mut self) {
576 if !self.armed {
577 return;
578 }
579 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
580 self.endpoint,
581 self.relay_corr,
582 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
583 ) {
584 send_goodbye_target_best_effort(
585 &self.forwarding.counters(),
586 &target,
587 "canceled route.bind",
588 );
589 }
590 self.armed = false;
591 }
592
593 fn disarm(&mut self) {
594 self.armed = false;
595 }
596}
597
598impl Drop for RouteBindReservationGuard {
599 fn drop(&mut self) {
600 self.release_and_disarm();
601 }
602}
603
604#[derive(Debug, Clone, Default)]
634pub(crate) struct RouteBindBreakers {
635 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
636}
637
638#[derive(Debug, Clone, Default)]
639pub(crate) struct RouteBindConcurrency {
640 modules: Arc<Mutex<HashMap<String, usize>>>,
641}
642
643struct RouteBindConcurrencyGuard {
644 concurrency: RouteBindConcurrency,
645 module_id: String,
646}
647
648impl RouteBindConcurrency {
649 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
652 let mut modules = self
653 .modules
654 .lock()
655 .expect("route.bind concurrency mutex poisoned");
656 let in_flight = modules.entry(module_id.to_string()).or_default();
657 if *in_flight >= limit {
658 return Err(*in_flight);
659 }
660 *in_flight += 1;
661 Ok(RouteBindConcurrencyGuard {
662 concurrency: self.clone(),
663 module_id: module_id.to_string(),
664 })
665 }
666}
667
668impl Drop for RouteBindConcurrencyGuard {
669 fn drop(&mut self) {
670 let mut modules = self
671 .concurrency
672 .modules
673 .lock()
674 .expect("route.bind concurrency mutex poisoned");
675 let remove = {
676 let in_flight = modules
677 .get_mut(&self.module_id)
678 .expect("admitted route.bind has a concurrency entry");
679 *in_flight -= 1;
680 *in_flight == 0
681 };
682 if remove {
683 modules.remove(&self.module_id);
684 }
685 }
686}
687
688#[derive(Debug, Default)]
689struct ModuleBreakerState {
690 consecutive_timeouts: u32,
692 cooldown_until: Option<Instant>,
695 probe_in_flight: Option<Arc<()>>,
700}
701
702enum RouteBindAdmission<'a> {
704 Admitted {
705 guard: RouteBindBreakerGuard<'a>,
706 probe: bool,
709 },
710 Refused {
711 consecutive_timeouts: u32,
712 retry_in: Duration,
716 probe_in_flight: bool,
717 },
718}
719
720struct RouteBindBreakerGuard<'a> {
727 breakers: RouteBindBreakers,
728 module_id: &'a str,
729 probe_token: Option<Arc<()>>,
730 settled: bool,
731}
732
733impl RouteBindBreakerGuard<'_> {
734 fn record_accepted(&mut self) -> bool {
738 self.settled = true;
739 self.breakers.record_accepted(self.module_id)
740 }
741
742 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
745 self.settled = true;
746 self.breakers.record_timeout(
747 self.module_id,
748 self.probe_token.as_ref(),
749 threshold,
750 cooldown,
751 )
752 }
753
754 fn record_inconclusive(&mut self) {
763 self.settled = true;
764 self.breakers
765 .record_inconclusive(self.module_id, self.probe_token.as_ref());
766 }
767}
768
769impl Drop for RouteBindBreakerGuard<'_> {
770 fn drop(&mut self) {
771 if !self.settled {
772 self.breakers
773 .record_inconclusive(self.module_id, self.probe_token.as_ref());
774 }
775 }
776}
777
778struct BreakerOpened {
782 consecutive_timeouts: u32,
783 reopened_after_probe: bool,
786}
787
788impl RouteBindBreakers {
789 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
790 self.modules
791 .lock()
792 .expect("route.bind breaker mutex poisoned")
793 }
794
795 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
798 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
799 probe: probe_token.is_some(),
800 guard: RouteBindBreakerGuard {
801 breakers: self.clone(),
802 module_id,
803 probe_token,
804 settled: false,
805 },
806 };
807
808 let mut modules = self.lock();
809 let Some(state) = modules.get_mut(module_id) else {
810 return admitted(None);
811 };
812 let Some(cooldown_until) = state.cooldown_until else {
813 return admitted(None);
814 };
815 if state.probe_in_flight.is_some() {
816 return RouteBindAdmission::Refused {
817 consecutive_timeouts: state.consecutive_timeouts,
818 retry_in: Duration::ZERO,
819 probe_in_flight: true,
820 };
821 }
822 let now = Instant::now();
823 if now < cooldown_until {
824 return RouteBindAdmission::Refused {
825 consecutive_timeouts: state.consecutive_timeouts,
826 retry_in: cooldown_until - now,
827 probe_in_flight: false,
828 };
829 }
830 let token = Arc::new(());
831 state.probe_in_flight = Some(Arc::clone(&token));
832 admitted(Some(token))
833 }
834
835 fn record_accepted(&self, module_id: &str) -> bool {
836 self.lock()
837 .remove(module_id)
838 .is_some_and(|state| state.cooldown_until.is_some())
839 }
840
841 fn record_timeout(
842 &self,
843 module_id: &str,
844 probe_token: Option<&Arc<()>>,
845 threshold: u32,
846 cooldown: Duration,
847 ) -> Option<BreakerOpened> {
848 let mut modules = self.lock();
849 let state = modules.entry(module_id.to_string()).or_default();
850 let was_open = state.cooldown_until.is_some();
851 let was_probe = Self::owns_probe(state, probe_token);
852 if was_probe {
853 state.probe_in_flight = None;
854 }
855 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
856 if state.consecutive_timeouts < threshold {
857 return None;
858 }
859 state.cooldown_until = Some(Instant::now() + cooldown);
860 Some(BreakerOpened {
861 consecutive_timeouts: state.consecutive_timeouts,
862 reopened_after_probe: was_open && was_probe,
863 })
864 }
865
866 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
867 state
871 .probe_in_flight
872 .as_ref()
873 .zip(token)
874 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
875 }
876
877 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
878 if let Some(state) = self.lock().get_mut(module_id) {
879 if Self::owns_probe(state, probe_token) {
880 state.probe_in_flight = None;
881 }
882 }
883 }
884
885 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
900 self.lock()
901 .remove(module_id)
902 .map(|state| state.consecutive_timeouts)
903 .filter(|discarded| *discarded > 0)
904 }
905
906 fn open_snapshot(&self) -> Option<serde_json::Value> {
913 let now = Instant::now();
914 let modules = self.lock();
915 let open = modules
916 .iter()
917 .filter_map(|(module_id, state)| {
918 let cooldown_until = state.cooldown_until?;
919 Some((
920 module_id.clone(),
921 serde_json::json!({
922 "consecutive_timeouts": state.consecutive_timeouts,
923 "cooldown_remaining_ms":
924 cooldown_until.saturating_duration_since(now).as_millis() as u64,
925 "probe_in_flight": state.probe_in_flight.is_some(),
926 }),
927 ))
928 })
929 .collect::<serde_json::Map<String, serde_json::Value>>();
930 (!open.is_empty()).then_some(serde_json::Value::Object(open))
931 }
932}
933
934impl ControlHandler {
935 pub fn new(registry: Arc<Registry>) -> Self {
936 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
937 }
938
939 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
940 let counters = forwarding.counters();
941 let route_bind_breakers = forwarding.route_bind_breakers();
945 let route_bind_concurrency = forwarding.route_bind_concurrency();
946 let route_outages = forwarding.route_outages();
947 Self {
948 registry,
949 activity: None,
950 activity_streams: Arc::new(Mutex::new(HashMap::new())),
951 forwarding,
952 process_liveness: None,
953 supervisor: SupervisorHandle::new(),
954 subc_capabilities: Arc::from([
955 CAP_MANIFEST_REGISTRATION.to_string(),
956 CAP_CHANNEL_LIFECYCLE.to_string(),
957 CAP_PING_PONG.to_string(),
958 CAP_SESSION_ATTACH.to_string(),
959 CAP_ADMISSION_FACTS_RELAY.to_string(),
960 CAP_SCOPES_V1.to_string(),
961 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
962 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
963 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
964 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
965 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
966 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
967 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
968 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
969 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
970 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
971 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
972 ]),
973 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
974 route_bind_relay_timeouts: BTreeMap::new(),
975 route_bind_breakers,
976 route_bind_concurrency,
977 route_outages,
978 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
979 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
980 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
981 storage_config: None,
982 machine_id: None,
983 admission_facts_carrier_module_id: None,
984 admission_facts_targets: None,
985 scopes: Arc::new(RwLock::new(ScopeTable::new(
986 crate::daemon_config::default_scope_authority_owners(),
987 ))),
988 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
989 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
990 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
991 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
992 #[cfg(test)]
993 caller_lookup: None,
994 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
995 rescan: None,
996 connected_clients: ConnectedClients::new(),
997 local_endpoint_unavailable: None,
998 counters,
999 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1000 daemon_shutdown: DaemonShutdownFlag::default(),
1001 daemon_provenance: DaemonProvenanceFacts::default(),
1002 #[cfg(test)]
1003 control_dispatch_delay: None,
1004 #[cfg(test)]
1005 provenance_probe_override: None,
1006 #[cfg(test)]
1007 route_bind_relay_sent_hold: None,
1008 #[cfg(test)]
1009 hello_registry_to_forwarding_hold: None,
1010 }
1011 }
1012
1013 pub fn with_storage_config(
1016 mut self,
1017 storage_config: Option<crate::daemon_config::StorageConfig>,
1018 ) -> Self {
1019 self.storage_config = storage_config;
1020 self
1021 }
1022
1023 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1026 self.activity = Some(crate::activity::ActivityLedger::open(
1027 run_dir.join("activity.jsonl"),
1028 incarnation,
1029 ));
1030 self.supervisor.set_activity(self.activity.clone());
1031 self
1032 }
1033
1034 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1037 self.machine_id = machine_id;
1038 self
1039 }
1040
1041 pub fn with_admission_facts_config(
1045 mut self,
1046 carrier_module_id: Option<String>,
1047 targets: Option<Vec<String>>,
1048 ) -> Self {
1049 self.admission_facts_carrier_module_id = carrier_module_id;
1050 self.admission_facts_targets = targets;
1051 self
1052 }
1053
1054 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1058 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1059 self.scope_authority_owners = owners;
1060 self
1061 }
1062
1063 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1065 self.privileged_operator_prompt = mode;
1066 self
1067 }
1068
1069 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1072 self.route_bind_relay_timeout = timeout;
1073 self
1074 }
1075
1076 pub fn with_route_bind_relay_timeouts(
1082 mut self,
1083 timeouts: impl IntoIterator<Item = (String, Duration)>,
1084 ) -> Self {
1085 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1086 self
1087 }
1088
1089 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1095 self.route_bind_relay_timeouts
1096 .get(module_id)
1097 .copied()
1098 .unwrap_or(self.route_bind_relay_timeout)
1099 }
1100
1101 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1109 self.route_bind_breaker_threshold = threshold.max(1);
1110 self.route_bind_breaker_cooldown = cooldown;
1111 self
1112 }
1113
1114 #[cfg(test)]
1115 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1116 self.health_probe_timeout = timeout;
1117 self
1118 }
1119
1120 #[cfg(test)]
1121 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1122 self.control_dispatch_delay = Some(delay);
1123 self
1124 }
1125
1126 #[cfg(test)]
1130 pub(crate) fn with_route_bind_relay_sent_hold(
1131 mut self,
1132 hold: Arc<tokio::sync::Barrier>,
1133 ) -> Self {
1134 self.route_bind_relay_sent_hold = Some(hold);
1135 self
1136 }
1137
1138 #[cfg(test)]
1143 pub(crate) fn with_hello_registry_to_forwarding_hold(
1144 mut self,
1145 hold: Arc<std::sync::Barrier>,
1146 ) -> Self {
1147 self.hello_registry_to_forwarding_hold = Some(hold);
1148 self
1149 }
1150
1151 #[cfg(test)]
1152 fn hold_between_registry_and_forwarding(&self) {
1153 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1154 hold.wait();
1155 hold.wait();
1156 }
1157 }
1158
1159 pub fn with_process_liveness(
1160 mut self,
1161 process_liveness: Arc<dyn ModuleProcessLiveness>,
1162 ) -> Self {
1163 self.process_liveness = Some(process_liveness);
1164 self
1165 }
1166
1167 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1168 self.supervisor = supervisor;
1169 self.supervisor.set_activity(self.activity.clone());
1170 self
1171 }
1172
1173 pub fn with_daemon_provenance(
1174 mut self,
1175 pid: u32,
1176 started_at_ms: u64,
1177 executable_path: Option<PathBuf>,
1178 build_git_sha: Option<String>,
1179 build_lock_digest: Option<String>,
1180 ) -> Self {
1181 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1182 let process_start_time = process_start_time(pid);
1183 self.daemon_provenance = DaemonProvenanceFacts {
1184 build: DaemonBuildProvenance {
1185 build_git_sha,
1186 build_lock_digest,
1187 },
1188 pid: Some(pid),
1189 started_at_ms: Some(started_at_ms),
1190 start_clock: None,
1191 executable_path,
1192 executable_identity,
1193 process_start_time,
1194 probe: ExecutableIdentityProbe::default(),
1195 };
1196 self
1197 }
1198
1199 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1200 self.daemon_provenance.start_clock = Some(clock);
1201 self
1202 }
1203
1204 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1206 self.scopes
1207 .write()
1208 .unwrap_or_else(|poisoned| poisoned.into_inner())
1209 .set_wall_clock(Arc::new(clock));
1210 self
1211 }
1212
1213 #[cfg(test)]
1214 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1215 self.provenance_probe_override = Some(result);
1216 self
1217 }
1218
1219 pub fn with_capability_config(
1223 self,
1224 modules: impl IntoIterator<Item = (String, bool)>,
1225 reserved_capabilities: BTreeMap<String, String>,
1226 ) -> Self {
1227 self.capability_evaluator
1228 .configure(modules, reserved_capabilities);
1229 self
1230 }
1231
1232 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1236 self.daemon_shutdown = flag;
1237 self
1238 }
1239
1240 pub fn with_supervisor_rescan(
1241 mut self,
1242 supervisor: Supervisor,
1243 config_path: impl Into<PathBuf>,
1244 configured_port: Option<u16>,
1245 ) -> Self {
1246 self.rescan = Some(SupervisorRescanContext {
1247 supervisor,
1248 config_path: config_path.into(),
1249 configured_port,
1250 storage_config: self.storage_config.clone(),
1251 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1252 admission_facts_targets: self.admission_facts_targets.clone(),
1253 scope_authority_owners: self.scope_authority_owners.clone(),
1254 privileged_operator_prompt: self.privileged_operator_prompt,
1255 generation: Arc::default(),
1256 });
1257 self
1258 }
1259
1260 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1261 self.connected_clients = connected_clients;
1262 self
1263 }
1264
1265 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1268 self.local_endpoint_unavailable = reason;
1269 self
1270 }
1271
1272 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1273 Arc::clone(&self.forwarding)
1274 }
1275
1276 pub(crate) fn counters(&self) -> DaemonCounters {
1277 self.counters.clone()
1278 }
1279
1280 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1283 tokio::spawn(async move {
1284 loop {
1285 self.capability_evaluator
1286 .wait_for_change_or_deadline()
1287 .await;
1288 self.refresh_capability_requirements();
1289 }
1290 });
1291 }
1292
1293 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1299 tokio::spawn(async move {
1300 loop {
1301 let has_deadlines = self
1302 .scopes
1303 .read()
1304 .unwrap_or_else(|poisoned| poisoned.into_inner())
1305 .has_deadlines();
1306 if !has_deadlines {
1307 self.scope_deadline_added.notified().await;
1308 continue;
1309 }
1310 tokio::time::sleep(Duration::from_secs(1)).await;
1311 if let Err(error) = self.sweep_expired_scopes() {
1312 warn!(%error, "scope expiry sweep failed");
1313 }
1314 }
1315 });
1316 }
1317
1318 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1319 let mut table = self
1320 .scopes
1321 .write()
1322 .unwrap_or_else(|poisoned| poisoned.into_inner());
1323 let swept = table.sweep_expired();
1324 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1325 drop(table);
1326 self.close_scope_drained_routes(drained);
1327 Ok(())
1328 }
1329
1330 fn publish_scope_changes(
1331 &self,
1332 changes: &[crate::scopes::ScopeTagChange],
1333 expired: &[crate::scopes::ScopeExpired],
1334 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1335 if changes.is_empty() {
1336 return Ok(Vec::new());
1337 }
1338 let drained = self
1339 .forwarding
1340 .publish_scope_changes(changes)
1341 .map_err(RouterError::Forwarding)?;
1342 let mut counts = HashMap::new();
1343 for route in &drained {
1344 *counts
1345 .entry((
1346 route.scope.owner.as_str(),
1347 route.scope.scope_ref.as_str(),
1348 route.scope.tag.scope_epoch,
1349 ))
1350 .or_insert(0usize) += 1;
1351 }
1352 for scope in expired {
1353 let routes_closed = counts
1354 .get(&(
1355 scope.owner.as_str(),
1356 scope.scope_ref.as_str(),
1357 scope.scope_epoch,
1358 ))
1359 .copied()
1360 .unwrap_or(0);
1361 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1362 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1363 routes_closed, "scope expired");
1364 }
1365 Ok(drained)
1366 }
1367
1368 fn runtime_capability_snapshot(
1369 &self,
1370 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1371 let runtime = self
1372 .supervisor
1373 .list()
1374 .into_iter()
1375 .map(|module| {
1376 let status = module.status().map_err(|err| {
1377 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1378 })?;
1379 Ok(RuntimeModule {
1380 module_id: status.module_id,
1381 state: status.state,
1382 enabled: status.enabled,
1383 })
1384 })
1385 .collect::<Result<Vec<_>, RouterError>>()?;
1386 let registrations = self.registry.list_registered_modules().map_err(|err| {
1392 RouterError::backend(
1393 0,
1394 0,
1395 format!("failed to list capability registrations: {err}"),
1396 )
1397 })?;
1398 let registrations = registrations
1399 .into_iter()
1400 .map(|registration| RegisteredModule {
1401 module_id: registration.manifest.module_id,
1402 module_version: registration.manifest.module_version,
1403 capabilities: registration.manifest.capabilities,
1404 })
1405 .collect();
1406 Ok((runtime, registrations))
1407 }
1408
1409 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1416 let cached_registration = RegisteredModule {
1417 module_id: registration.manifest.module_id.clone(),
1418 module_version: registration.manifest.module_version.clone(),
1419 capabilities: registration.manifest.capabilities.clone(),
1420 };
1421 let change = self.capability_evaluator.record_hello(&cached_registration);
1422 log_manifest_change(&cached_registration.module_id, &change);
1423 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1424 self.enforce_capability_denies();
1425 }
1426 self.refresh_capability_requirements();
1427 }
1428
1429 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1433 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1434 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1435 self.supervisor.set_swap_promotion_observer(observer);
1436 }
1437
1438 pub fn refresh_capability_requirements(&self) {
1439 match self.runtime_capability_snapshot() {
1440 Ok((runtime, registrations)) => {
1441 log_requirement_events(
1442 self.capability_evaluator
1443 .evaluate_now(&runtime, ®istrations),
1444 &self.daemon_shutdown,
1445 );
1446 }
1447 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1448 }
1449 }
1450
1451 fn enforce_capability_denies(&self) {
1455 let registrations = match self.registry.list_registered_modules() {
1456 Ok(snapshot) => snapshot,
1457 Err(err) => {
1458 warn!(error = %err, "failed to read registrations for capability deny census");
1459 return;
1460 }
1461 };
1462 let manifests = registrations
1463 .into_iter()
1464 .map(|registration| {
1465 (
1466 registration.manifest.module_id.clone(),
1467 registration.manifest,
1468 )
1469 })
1470 .collect::<BTreeMap<_, _>>();
1471 let census = match self.forwarding.route_census(None) {
1472 Ok(census) => census,
1473 Err(err) => {
1474 warn!(error = %err, "failed to read route census for capability deny enforcement");
1475 return;
1476 }
1477 };
1478
1479 for (target_module_id, routes) in census {
1480 let Some(target_manifest) = manifests.get(&target_module_id) else {
1481 continue;
1482 };
1483 let mut closed_routes = Vec::new();
1484 let mut goodbyes = Vec::new();
1485 for route in routes {
1486 let Principal::Reserved {
1487 module_id: opening_module_id,
1488 } = &route.principal
1489 else {
1490 continue;
1491 };
1492 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1493 continue;
1494 };
1495 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1496 continue;
1497 };
1498
1499 match self.forwarding.release_client_route(
1500 route.goodbye_target.connection_id,
1501 route.goodbye_target.channel,
1502 route.goodbye_target.epoch,
1503 ) {
1504 Ok(RouteRelease::Removed(module_goodbye)) => {
1505 warn!(
1506 opening_module_id,
1507 target_module_id,
1508 capability,
1509 "force-closing route because an attested capability deny edge now matches"
1510 );
1511 goodbyes.push(module_goodbye);
1512 goodbyes.push(route.goodbye_target.clone());
1513 closed_routes.push(route);
1514 }
1515 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1516 Err(err) => warn!(
1517 opening_module_id,
1518 target_module_id,
1519 capability,
1520 error = %err,
1521 "failed to force-close capability-denied route"
1522 ),
1523 }
1524 }
1525
1526 if closed_routes.is_empty() {
1527 continue;
1528 }
1529 send_route_control_pushes(
1530 &self.forwarding,
1531 closed_routes,
1532 ClientControlPush::RouteClosed {
1533 module_id: target_module_id,
1534 channels: Vec::new(),
1535 reason: RouteCloseReason::CapabilityDenied,
1536 drained: false,
1537 abandoned: 0,
1538 excluded_subscriptions: 0,
1539 terminal: Some(false),
1540 },
1541 );
1542 self.emit_route_goodbyes(goodbyes);
1545 }
1546 }
1547
1548 fn not_ready_reason(
1554 &self,
1555 registration: &crate::registry::ModuleRegistration,
1556 ) -> Option<NotReadyReason> {
1557 if !registration.ready {
1558 return Some(NotReadyReason {
1559 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1560 capability: None,
1561 });
1562 }
1563 self.first_unprovided_required_capability(registration)
1564 .map(|capability| NotReadyReason {
1565 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1566 capability: Some(capability),
1567 })
1568 }
1569
1570 fn first_unprovided_required_capability(
1588 &self,
1589 registration: &crate::registry::ModuleRegistration,
1590 ) -> Option<String> {
1591 let required = registration
1592 .manifest
1593 .capabilities
1594 .iter()
1595 .flat_map(|declarations| declarations.requires.iter())
1596 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1597 .map(|requirement| requirement.capability.as_str())
1598 .collect::<BTreeSet<_>>();
1599 if required.is_empty() {
1600 return None;
1601 }
1602 let module_id = registration.manifest.module_id.as_str();
1603 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1604 if required
1605 .iter()
1606 .any(|capability| verdict(capability).is_none())
1607 {
1608 self.refresh_capability_requirements();
1609 }
1610 required
1611 .into_iter()
1612 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1613 .map(str::to_string)
1614 }
1615
1616 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1617 self.capability_evaluator
1618 .statuses()
1619 .into_iter()
1620 .map(capability_requirement_status)
1621 .collect()
1622 }
1623
1624 fn deregister_connection(
1631 &self,
1632 connection_id: ConnectionId,
1633 reason: RegistrationEndReason,
1634 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1635 self.registry
1636 .deregister_connection_with_reason(connection_id, reason)
1637 }
1638
1639 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1640 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1641 return None;
1642 }
1643 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1644 parse_client_control_request(&frame.body)
1645 else {
1646 return None;
1647 };
1648 Some(target_module_id(&target).to_string())
1649 }
1650
1651 pub(crate) fn route_open_capacity_refusal(
1652 &self,
1653 ctx: &RouteCtx,
1654 frame: &Frame,
1655 target_module_id: &str,
1656 in_flight: usize,
1657 limit: usize,
1658 ) -> Result<Frame, RouterError> {
1659 let caller = self.route_open_caller_from_frame(frame);
1662 self.route_open_admission_refusal_frame(
1663 ctx,
1664 &caller,
1665 frame,
1666 target_module_id,
1667 "open_admission_full",
1668 (in_flight, limit),
1669 format!(
1670 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1671 ),
1672 )
1673 }
1674
1675 fn route_open_target_capacity_refusal(
1676 &self,
1677 ctx: &RouteCtx,
1678 caller: &RouteOpenCaller,
1679 frame: &Frame,
1680 target_module_id: &str,
1681 in_flight: usize,
1682 ) -> Result<Frame, RouterError> {
1683 self.route_open_admission_refusal_frame(
1684 ctx,
1685 caller,
1686 frame,
1687 target_module_id,
1688 "target_binds_full",
1689 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1690 format!(
1691 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1692 ),
1693 )
1694 }
1695
1696 #[allow(clippy::too_many_arguments)]
1704 fn route_open_admission_refusal_frame(
1705 &self,
1706 ctx: &RouteCtx,
1707 caller: &RouteOpenCaller,
1708 frame: &Frame,
1709 target_module_id: &str,
1710 reason: &'static str,
1711 (in_flight, limit): (usize, usize),
1712 message: impl Into<String>,
1713 ) -> Result<Frame, RouterError> {
1714 let code = error_codes::TARGET_UNAVAILABLE;
1715 self.counters.increment_route_open_refused(code);
1716 info!(
1717 target: "control",
1718 code,
1719 reason,
1720 module_id = ?target_module_id,
1721 connection_id = ctx.connection_id.get(),
1722 principal = %caller.principal,
1723 scope = %caller.scope,
1724 harness = %caller.harness,
1725 in_flight,
1726 limit,
1727 "route.open refused"
1728 );
1729 control_error_frame(frame, code, message.into())
1730 }
1731
1732 #[cfg(test)]
1738 pub fn handle_control(
1739 &self,
1740 connection_id: ConnectionId,
1741 frame: Frame,
1742 ) -> Result<Vec<Frame>, RouterError> {
1743 match frame.header.ty {
1744 FrameType::Ping => Ok(vec![pong(&frame)?]),
1745 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1746 FrameType::Goodbye => self.handle_goodbye(connection_id),
1747 ty => Ok(vec![control_error_frame(
1748 &frame,
1749 "unsupported_control_frame",
1750 format!("unsupported channel-0 frame {ty:?}"),
1751 )?]),
1752 }
1753 }
1754
1755 pub async fn handle_control_frame(
1756 &self,
1757 ctx: &RouteCtx,
1758 frame: Frame,
1759 ) -> Result<Vec<Frame>, RouterError> {
1760 self.handle_control_frame_timed(ctx, frame, None).await
1761 }
1762
1763 pub(crate) fn record_caller_endpoints(
1764 &self,
1765 connection: ConnectionId,
1766 endpoints: Option<(SocketAddr, SocketAddr)>,
1767 ) {
1768 let mut callers = self
1769 .caller_endpoints
1770 .lock()
1771 .unwrap_or_else(|p| p.into_inner());
1772 callers.remove(&connection);
1773 if let Some(endpoints) = endpoints {
1774 callers.insert(connection, endpoints);
1775 }
1776 }
1777
1778 #[cfg(test)]
1779 pub(crate) fn with_caller_lookup_for_test(
1780 mut self,
1781 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1782 ) -> Self {
1783 self.caller_lookup = Some(Arc::new(lookup));
1784 self
1785 }
1786
1787 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1788 let endpoints = self
1789 .caller_endpoints
1790 .lock()
1791 .unwrap_or_else(|p| p.into_inner())
1792 .get(&connection)
1793 .copied();
1794 #[cfg(test)]
1795 let lookup = self.caller_lookup.clone();
1796 tokio::task::spawn_blocking(move || {
1797 #[cfg(test)]
1798 if let Some(lookup) = lookup {
1799 return lookup(endpoints);
1800 }
1801 endpoints
1802 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1803 .unwrap_or_default()
1804 })
1805 .await
1806 .unwrap_or_default()
1807 }
1808
1809 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1810 frame.header.channel == 0
1811 && frame.header.ty == FrameType::Request
1812 && self
1813 .forwarding
1814 .module_endpoint_for_connection(ctx.connection_id)
1815 .is_ok_and(|endpoint| endpoint.is_none())
1816 && parse_client_control_request(&frame.body)
1817 .ok()
1818 .is_some_and(|request| privileged_operation(&request).is_some())
1819 }
1820
1821 pub(crate) async fn handle_control_frame_timed(
1822 &self,
1823 ctx: &RouteCtx,
1824 frame: Frame,
1825 dispatch_started_at: Option<StdInstant>,
1826 ) -> Result<Vec<Frame>, RouterError> {
1827 match frame.header.ty {
1828 FrameType::Ping => Ok(vec![pong(&frame)?]),
1829 FrameType::Hello => {
1830 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1831 }
1832 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1833 FrameType::Cancel => {
1834 if self
1837 .forwarding
1838 .operator_confirms()
1839 .cancel(ctx.connection_id, frame.header.corr)
1840 || self
1841 .supervisor
1842 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1843 || self.cancel_activity_subscription(ctx, &frame)?
1844 {
1845 Ok(Vec::new())
1846 } else {
1847 Ok(vec![control_error_frame(
1848 &frame,
1849 "unknown_subscription",
1850 "no subscription has this correlation id",
1851 )?])
1852 }
1853 }
1854 FrameType::Request => {
1855 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1858 if op
1859 .as_ref()
1860 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1861 {
1862 return self.handle_activity_record(ctx, frame).await;
1863 }
1864 if op.as_ref().is_some_and(|probe| {
1865 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1866 }) {
1867 return self.handle_activity_read(ctx, frame).await;
1868 }
1869 if op
1870 .as_ref()
1871 .is_some_and(|probe| probe.op == "operator.confirm")
1872 {
1873 return self.handle_operator_confirm(ctx, frame);
1874 }
1875 if op
1876 .as_ref()
1877 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1878 {
1879 return self.handle_scope_served_by(ctx.connection_id, frame);
1880 }
1881 if self
1882 .forwarding
1883 .module_endpoint_for_connection(ctx.connection_id)
1884 .map_err(RouterError::Forwarding)?
1885 .is_some()
1886 {
1887 if !is_known_module_request_op(&frame.body) {
1888 return Ok(vec![control_error_frame(
1889 &frame,
1890 "unsupported_control_frame",
1891 "module-originated channel-0 REQUEST is not supported",
1892 )?]);
1893 }
1894 let request = match parse_module_control_request_from_module(&frame.body) {
1895 Ok(request) => request,
1896 Err((err, ControlRequestBodyError::UnknownOp)) => {
1897 return Ok(vec![control_error_frame(
1898 &frame,
1899 "unsupported_control_frame",
1900 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1901 )?])
1902 }
1903 Err((err, ControlRequestBodyError::InvalidBody)) => {
1904 return Ok(vec![control_error_frame(
1905 &frame,
1906 "invalid_control_body",
1907 format!("malformed module control body: {err}"),
1908 )?])
1909 }
1910 };
1911 let op = module_control_request_op(&request);
1912 let corr = frame.header.corr;
1913 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1914 let result =
1915 self.handle_module_control_request(ctx.connection_id, frame, request);
1916 log_slow_control_dispatch(
1917 dispatch_started_at,
1918 op,
1919 ctx.connection_id,
1920 corr,
1921 None,
1922 );
1923 return result;
1924 }
1925
1926 if is_known_module_request_op(&frame.body) {
1927 return Ok(vec![control_error_frame(
1928 &frame,
1929 "not_registered",
1930 "catalog.update requires an active module registration owned by this connection",
1931 )?]);
1932 }
1933
1934 let request = match parse_client_control_request(&frame.body) {
1935 Ok(request) => request,
1936 Err((err, ControlRequestBodyError::UnknownOp)) => {
1937 return Ok(vec![control_error_frame(
1938 &frame,
1939 "unknown_control_op",
1940 format!("unknown client control op: {err}"),
1941 )?])
1942 }
1943 Err((err, ControlRequestBodyError::InvalidBody)) => {
1944 return Ok(vec![control_error_frame(
1945 &frame,
1946 "invalid_control_body",
1947 format!("malformed client control body: {err}"),
1948 )?])
1949 }
1950 };
1951 let op = client_control_request_op(&request);
1952 let corr = frame.header.corr;
1953 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1954 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1955 #[cfg(test)]
1956 if let Some(delay) = self.control_dispatch_delay {
1957 tokio::time::sleep(delay).await;
1958 }
1959 let result = self
1960 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1961 .await;
1962 if let Some(timings) = route_open_timings.as_mut() {
1963 timings.finish();
1964 }
1965 log_slow_control_dispatch(
1966 dispatch_started_at,
1967 op,
1968 ctx.connection_id,
1969 corr,
1970 route_open_timings.as_ref(),
1971 );
1972 result
1973 }
1974 FrameType::Push => {
1975 let Some(endpoint) = self
1976 .forwarding
1977 .module_endpoint_for_connection(ctx.connection_id)
1978 .map_err(RouterError::Forwarding)?
1979 else {
1980 return Ok(vec![control_error_frame(
1981 &frame,
1982 "unsupported_control_frame",
1983 "client-originated channel-0 PUSH is not supported",
1984 )?]);
1985 };
1986 self.handle_status_update(endpoint, frame)
1987 }
1988 FrameType::Response | FrameType::Error
1989 if self
1990 .forwarding
1991 .module_endpoint_for_connection(ctx.connection_id)
1992 .map_err(RouterError::Forwarding)?
1993 .is_some() =>
1994 {
1995 self.handle_module_relay_response(ctx.connection_id, frame)
1996 }
1997 ty => Ok(vec![control_error_frame(
1998 &frame,
1999 "unsupported_control_frame",
2000 format!("unsupported channel-0 frame {ty:?}"),
2001 )?]),
2002 }
2003 }
2004
2005 pub fn cleanup_connection(
2006 &self,
2007 connection_id: ConnectionId,
2008 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2009 self.caller_endpoints
2010 .lock()
2011 .unwrap_or_else(|p| p.into_inner())
2012 .remove(&connection_id);
2013 self.forwarding
2014 .operator_confirms()
2015 .connection_closed(connection_id);
2016 self.cleanup_connection_with_end_reason(
2017 connection_id,
2018 RegistrationEndReason::ConnectionClosed,
2019 )
2020 }
2021
2022 fn cleanup_connection_with_end_reason(
2023 &self,
2024 connection_id: ConnectionId,
2025 requested_reason: RegistrationEndReason,
2026 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2027 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2028 self.registry
2029 .get_module_by_connection(connection_id)?
2030 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2031 .and_then(|module| module.registration_end_reason().ok().flatten())
2032 .unwrap_or(requested_reason)
2033 } else {
2034 requested_reason
2035 };
2036 let crash_closed = self
2037 .registry
2038 .get_module_by_connection(connection_id)?
2039 .and_then(|registration| {
2040 self.forwarding
2041 .module_endpoint_for_connection(connection_id)
2042 .ok()
2043 .flatten()
2044 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2045 .map(|routes| (registration.manifest.module_id, routes))
2046 });
2047 let crash_closed = crash_closed.map(|(module_id, routes)| {
2048 let terminal = match self.supervisor.get(&module_id) {
2049 None => false,
2050 Some(module) => match module.will_recover_after_connection_loss() {
2051 Ok(will_recover) => !will_recover,
2052 Err(err) => {
2053 warn!(
2054 %module_id,
2055 error = %err,
2056 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2057 );
2058 false
2059 }
2060 },
2061 };
2062 let (reason, drained) = match self
2066 .forwarding
2067 .daemon_drain_result_for_connection(connection_id)
2068 {
2069 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2070 Ok(None) => (RouteCloseReason::Crash, false),
2071 Err(err) => {
2072 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2073 (RouteCloseReason::Crash, false)
2074 }
2075 };
2076 (module_id, routes, reason, drained, terminal)
2077 });
2078 let registrations = self.deregister_connection(connection_id, end_reason);
2079 let cleanup = if crash_closed.is_some() {
2080 self.forwarding.cleanup_connection_counted(connection_id)
2081 } else {
2082 self.forwarding
2088 .cleanup_connection(connection_id)
2089 .map(|released| crate::forwarding::ConnectionCleanup {
2090 released,
2091 abandoned_relays: 0,
2092 })
2093 };
2094 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2099 let abandoned = cleanup
2100 .as_ref()
2101 .map_or(0, |cleanup| cleanup.abandoned_relays);
2102 send_route_control_pushes(
2103 &self.forwarding,
2104 routes,
2105 ClientControlPush::RouteClosed {
2106 module_id,
2107 channels: Vec::new(),
2108 reason,
2109 drained,
2110 abandoned,
2111 excluded_subscriptions: 0,
2112 terminal: Some(terminal),
2113 },
2114 );
2115 }
2116 if let Ok(cleanup) = cleanup {
2117 self.emit_route_goodbyes(cleanup.released);
2118 }
2119 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2123 crate::supervise::notify_registration_release();
2124 self.capability_evaluator.wake_deadline_loop();
2125 self.refresh_capability_requirements();
2126 }
2127 self.supervisor.remove_spawn_subscribers(connection_id);
2128 self.activity_streams
2129 .lock()
2130 .unwrap_or_else(|p| p.into_inner())
2131 .retain(|(id, _), task| {
2132 if *id == connection_id {
2133 task.abort();
2134 false
2135 } else {
2136 true
2137 }
2138 });
2139 self.hello_launch_nonces
2142 .lock()
2143 .unwrap_or_else(|poisoned| poisoned.into_inner())
2144 .forget(connection_id);
2145 self.scopes
2146 .write()
2147 .unwrap_or_else(|poisoned| poisoned.into_inner())
2148 .release_connection(connection_id);
2149 registrations
2150 }
2151
2152 pub(crate) fn handle_route_goodbye(
2153 &self,
2154 connection_id: ConnectionId,
2155 route_channel: u16,
2156 route_epoch: u32,
2157 ) -> Result<bool, RouterError> {
2158 debug!(
2159 connection_id = connection_id.get(),
2160 route_channel, route_epoch, "handling route GOODBYE"
2161 );
2162 let RouteRelease::Removed(released_route) = self
2163 .forwarding
2164 .release_client_route(connection_id, route_channel, route_epoch)
2165 .map_err(RouterError::Forwarding)?
2166 else {
2167 return Ok(false);
2168 };
2169 self.emit_route_goodbyes(vec![released_route]);
2170 Ok(true)
2171 }
2172
2173 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2174 for released in released_routes {
2175 let frame = match Frame::build_with_version(
2176 released.negotiated_ver,
2177 FrameType::Goodbye,
2178 control_flags(),
2179 released.channel,
2180 released.epoch,
2181 0,
2182 Vec::new(),
2183 ) {
2184 Ok(frame) => frame,
2185 Err(err) => {
2186 warn!(
2187 route_channel = released.channel,
2188 error = %err,
2189 "failed to build route GOODBYE frame"
2190 );
2191 continue;
2192 }
2193 };
2194 if !released.close_on_delivery_failure() {
2195 crate::forwarding::send_module_route_goodbye(
2196 &self.counters,
2197 &released.sink,
2198 frame,
2199 released.module_id.as_deref(),
2200 "client route released",
2201 );
2202 continue;
2203 }
2204 if let Err(err) = released.sink.try_send(frame) {
2205 warn!(
2206 target_connection_id = released.connection_id.get(),
2207 route_channel = released.channel,
2208 error = %err,
2209 "route GOODBYE was not delivered to client; closing target connection"
2210 );
2211 if self
2212 .forwarding
2213 .escalate_client_delivery_failure(
2214 released.connection_id,
2215 released.channel,
2216 released.epoch,
2217 CloseReason::new(
2218 "route_goodbye_delivery_failed",
2219 format!(
2220 "failed to enqueue route GOODBYE for channel {}: {err}",
2221 released.channel
2222 ),
2223 ),
2224 crate::forwarding::UndeliveredFrame {
2225 module_id: released.module_id.as_deref(),
2226 sink: &released.sink,
2227 },
2228 )
2229 .unwrap_or(false)
2230 {
2231 self.counters.increment_goodbye_relay_client_failed();
2232 }
2233 }
2234 }
2235 }
2236
2237 fn send_abandoned_route_bind_goodbye(
2249 &self,
2250 module_sink: &crate::FrameSink,
2251 negotiated_ver: u8,
2252 module_channel: u16,
2253 module_epoch: u32,
2254 ) {
2255 let frame = match Frame::build_with_version(
2256 negotiated_ver,
2257 FrameType::Goodbye,
2258 control_flags(),
2259 module_channel,
2260 module_epoch,
2261 0,
2262 Vec::new(),
2263 ) {
2264 Ok(frame) => frame,
2265 Err(err) => {
2266 warn!(
2267 route_channel = module_channel,
2268 error = %err,
2269 "failed to build GOODBYE for abandoned route.bind"
2270 );
2271 return;
2272 }
2273 };
2274 crate::forwarding::send_module_route_goodbye(
2275 &self.counters,
2276 module_sink,
2277 frame,
2278 None,
2279 "abandoned route.bind",
2280 );
2281 }
2282
2283 fn handle_hello(
2284 &self,
2285 connection_id: ConnectionId,
2286 sink: Option<crate::FrameSink>,
2287 frame: Frame,
2288 ) -> Result<Vec<Frame>, RouterError> {
2289 debug!(
2290 connection_id = connection_id.get(),
2291 corr = frame.header.corr,
2292 "handling HELLO"
2293 );
2294 if self
2298 .registry
2299 .get_module_by_connection(connection_id)
2300 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2301 .is_some()
2302 {
2303 return Ok(vec![control_error_frame(
2304 &frame,
2305 "invalid_hello",
2306 "connection is already registered as a module",
2307 )?]);
2308 }
2309 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2310 Ok(value) => value,
2311 Err(err) => {
2312 return Ok(vec![control_error_frame(
2313 &frame,
2314 "invalid_hello",
2315 format!("malformed HELLO body: {err}"),
2316 )?])
2317 }
2318 };
2319 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2320 return Ok(vec![control_error_frame(
2321 &frame,
2322 "invalid_capability_grammar",
2323 err.to_string(),
2324 )?]);
2325 }
2326 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2327 return Ok(vec![control_error_frame(
2328 &frame,
2329 "invalid_manifest",
2330 err.to_string(),
2331 )?]);
2332 }
2333 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2334 return Ok(vec![control_error_frame(
2335 &frame,
2336 "invalid_event_declaration",
2337 err.to_string(),
2338 )?]);
2339 }
2340 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2341 return Ok(vec![control_error_frame(
2342 &frame,
2343 "invalid_manifest",
2344 err.to_string(),
2345 )?]);
2346 }
2347 if let Some(provenance) = hello_value
2348 .get("manifest")
2349 .and_then(|manifest| manifest.get("provenance"))
2350 {
2351 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2352 return Ok(vec![control_error_frame(
2353 &frame,
2354 "invalid_manifest",
2355 format!("malformed manifest provenance: {err}"),
2356 )?]);
2357 }
2358 }
2359 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2360 let module_id = hello_value["manifest"]["module_id"]
2361 .as_str()
2362 .unwrap_or("<unknown>");
2363 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2364 }
2365 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2366 let module_id = hello_value["manifest"]["module_id"]
2367 .as_str()
2368 .unwrap_or("<unknown>");
2369 warn!(module_id, %reason, "dropping flow_operations from HELLO");
2370 }
2371 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2372 Ok(hello) => hello,
2373 Err(err) => {
2374 return Ok(vec![control_error_frame(
2375 &frame,
2376 "invalid_hello",
2377 format!("malformed HELLO body: {err}"),
2378 )?])
2379 }
2380 };
2381
2382 if hello.protocol_ver != hello.manifest.protocol_ver {
2383 return Ok(vec![control_error_frame(
2384 &frame,
2385 "invalid_manifest",
2386 format!(
2387 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2388 hello.protocol_ver, hello.manifest.protocol_ver
2389 ),
2390 )?]);
2391 }
2392
2393 if hello.manifest.module_id.trim().is_empty() {
2394 return Ok(vec![control_error_frame(
2395 &frame,
2396 "invalid_manifest",
2397 "manifest module_id must not be empty",
2398 )?]);
2399 }
2400
2401 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2402 Ok(negotiated_ver) => negotiated_ver,
2403 Err(message) => {
2404 return Ok(vec![control_error_frame(
2405 &frame,
2406 "version_unsupported",
2407 message,
2408 )?])
2409 }
2410 };
2411
2412 let swap_admission = self
2420 .supervisor
2421 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2422 if swap_admission == SwapHelloAdmission::Refused {
2423 warn!(
2424 module_id = %hello.manifest.module_id,
2425 connection_id = connection_id.get(),
2426 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2427 );
2428 return Ok(vec![control_error_frame(
2429 &frame,
2430 "swap_token_invalid",
2431 format!(
2432 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2433 hello.manifest.module_id
2434 ),
2435 )?]);
2436 }
2437 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2438
2439 let nonce_verified = if swap_candidate {
2444 true
2445 } else {
2446 match self.supervisor.reserved_hello_verification(
2447 &hello.manifest.module_id,
2448 hello.launch_nonce.as_deref(),
2449 ) {
2450 Ok(verified) => verified,
2451 Err(rejection) => {
2452 let message = match rejection {
2453 ReservedHelloRejection::Exact { module_id } => format!(
2454 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2455 ),
2456 ReservedHelloRejection::Prefix {
2457 prefix,
2458 owner_module_id,
2459 } => format!(
2460 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2461 hello.manifest.module_id
2462 ),
2463 };
2464 return Ok(vec![control_error_frame(
2465 &frame,
2466 "reserved_module",
2467 message,
2468 )?]);
2469 }
2470 }
2471 };
2472
2473 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2474 &hello.manifest.module_id,
2475 hello.manifest.capabilities.as_ref(),
2476 );
2477 if let Some(refusal) = reserved_capability_refusals.first() {
2478 let capability = refusal.capability.clone();
2479 let bound_module = refusal.claimants[0].clone();
2480 log_duplicate_claim_events(reserved_capability_refusals);
2481 return Ok(vec![control_error_frame(
2482 &frame,
2483 "reserved_capability",
2484 format!(
2485 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2486 capability, bound_module, hello.manifest.module_id
2487 ),
2488 )?]);
2489 }
2490
2491 if self
2494 .forwarding
2495 .connection_has_client_routes(connection_id)
2496 .map_err(RouterError::Forwarding)?
2497 {
2498 return Ok(vec![control_error_frame(
2499 &frame,
2500 "invalid_hello",
2501 "connection has open client routes and cannot also register as a module",
2502 )?]);
2503 }
2504
2505 self.hello_launch_nonces
2510 .lock()
2511 .unwrap_or_else(|poisoned| poisoned.into_inner())
2512 .record(connection_id, hello.launch_nonce.as_deref());
2513 let control_ops = effective_module_control_ops(hello.control_ops);
2514 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2517 if swap_candidate {
2518 return self.register_swap_candidate(
2519 connection_id,
2520 sink,
2521 &frame,
2522 hello.manifest,
2523 negotiated_ver,
2524 control_ops,
2525 hello_ack,
2526 );
2527 }
2528 let registration = match self
2535 .registry
2536 .register_unpublished_verified_with_control_ops(
2537 hello.manifest,
2538 negotiated_ver,
2539 connection_id,
2540 control_ops,
2541 nonce_verified,
2542 ) {
2543 Ok(registration) => registration,
2544 Err(RegistryError::DuplicateModuleId { module_id }) => {
2545 return Ok(vec![control_error_frame(
2546 &frame,
2547 "duplicate_module_id",
2548 format!(
2549 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2550 ),
2551 )?])
2552 }
2553 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2554 return Ok(vec![control_error_frame(
2555 &frame,
2556 "invalid_module_id",
2557 err.to_string(),
2558 )?])
2559 }
2560 Err(err) => {
2561 return Ok(vec![control_error_frame(
2562 &frame,
2563 "registry_error",
2564 err.to_string(),
2565 )?])
2566 }
2567 };
2568 #[cfg(test)]
2569 self.hold_between_registry_and_forwarding();
2570
2571 let reply = if let Some(sink) = sink {
2572 let concurrency = manifest_concurrency(®istration.manifest);
2588 if let Err(err) = self.forwarding.register_module_connection_acked(
2589 connection_id,
2590 registration.manifest.module_id.clone(),
2591 negotiated_ver,
2592 concurrency,
2593 sink,
2594 hello_ack,
2595 ) {
2596 if matches!(
2601 self.deregister_connection(
2602 connection_id,
2603 RegistrationEndReason::RegistrationFailed,
2604 ),
2605 Ok(r) if !r.is_empty()
2606 ) {
2607 crate::supervise::notify_registration_release();
2608 }
2609 return Ok(vec![control_error_frame(
2610 &frame,
2611 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2612 "invalid_hello"
2613 } else {
2614 forwarding_error_code(&err)
2615 },
2616 err.to_string(),
2617 )?]);
2618 }
2619 Vec::new()
2620 } else {
2621 vec![hello_ack]
2624 };
2625
2626 if !self
2631 .registry
2632 .publish_registration(connection_id)
2633 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2634 {
2635 debug!(
2636 module_id = %registration.manifest.module_id,
2637 connection_id = connection_id.get(),
2638 "registration was removed before it could be published"
2639 );
2640 }
2641
2642 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2651 info!(
2652 module_id = %registration.manifest.module_id,
2653 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2654 );
2655 }
2656
2657 self.apply_registration_capabilities(®istration);
2658
2659 self.supervisor
2660 .record_activity_registration(®istration.manifest.module_id);
2661
2662 info!(
2663 module_id = %registration.manifest.module_id,
2664 module_version = %registration.manifest.module_version,
2665 negotiated_ver,
2666 routable_provider = manifest_provides_routable_role(®istration.manifest),
2667 connection_id = connection_id.get(),
2668 "module registered"
2669 );
2670
2671 Ok(reply)
2672 }
2673
2674 #[allow(clippy::too_many_arguments)]
2691 fn register_swap_candidate(
2692 &self,
2693 connection_id: ConnectionId,
2694 sink: Option<crate::FrameSink>,
2695 frame: &Frame,
2696 manifest: ModuleManifest,
2697 negotiated_ver: u8,
2698 control_ops: Vec<String>,
2699 hello_ack: Frame,
2700 ) -> Result<Vec<Frame>, RouterError> {
2701 let module_id = manifest.module_id.clone();
2702 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2703 manifest,
2704 negotiated_ver,
2705 connection_id,
2706 control_ops,
2707 ) {
2708 Ok(registration) => registration,
2709 Err(RegistryError::DuplicateModuleId { module_id }) => {
2710 return Ok(vec![control_error_frame(
2711 frame,
2712 "duplicate_module_id",
2713 format!(
2714 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2715 ),
2716 )?])
2717 }
2718 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2719 return Ok(vec![control_error_frame(
2720 frame,
2721 "invalid_module_id",
2722 err.to_string(),
2723 )?])
2724 }
2725 Err(err) => {
2726 return Ok(vec![control_error_frame(
2727 frame,
2728 "registry_error",
2729 err.to_string(),
2730 )?])
2731 }
2732 };
2733 #[cfg(test)]
2734 self.hold_between_registry_and_forwarding();
2735 let reply = if let Some(sink) = sink {
2736 let concurrency = manifest_concurrency(®istration.manifest);
2740 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2741 connection_id,
2742 module_id.clone(),
2743 negotiated_ver,
2744 concurrency,
2745 sink,
2746 hello_ack,
2747 ) {
2748 if matches!(
2749 self.deregister_connection(
2750 connection_id,
2751 RegistrationEndReason::RegistrationFailed,
2752 ),
2753 Ok(r) if !r.is_empty()
2754 ) {
2755 crate::supervise::notify_registration_release();
2756 }
2757 return Ok(vec![control_error_frame(
2758 frame,
2759 forwarding_error_code(&err),
2760 err.to_string(),
2761 )?]);
2762 }
2763 Vec::new()
2764 } else {
2765 vec![hello_ack]
2766 };
2767 if !self
2768 .registry
2769 .publish_registration(connection_id)
2770 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2771 {
2772 debug!(
2773 module_id = %module_id,
2774 connection_id = connection_id.get(),
2775 "swap candidate registration was removed before it could be published"
2776 );
2777 }
2778 self.supervisor.mark_swap_candidate_admitted(&module_id);
2779 info!(
2780 module_id = %module_id,
2781 module_version = %registration.manifest.module_version,
2782 negotiated_ver,
2783 ready = registration.ready,
2784 connection_id = connection_id.get(),
2785 "swap candidate registered; not routable until cutover"
2786 );
2787 Ok(reply)
2788 }
2789
2790 fn build_hello_ack(
2791 &self,
2792 frame: &Frame,
2793 negotiated_ver: u8,
2794 module_id: &str,
2795 ) -> Result<Frame, RouterError> {
2796 let ack = ModuleHelloAckBody {
2797 negotiated_ver,
2798 subc_ops: module_subc_ops(),
2799 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2800 storage: self
2801 .storage_config
2802 .as_ref()
2803 .map(|cfg| cfg.descriptor_for(module_id)),
2804 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2805 };
2806 let body = serde_json::to_vec(&ack).map_err(|err| {
2807 RouterError::backend(
2808 0,
2809 frame.header.corr,
2810 format!("failed to encode HELLO_ACK: {err}"),
2811 )
2812 })?;
2813
2814 Frame::build_with_version(
2815 negotiated_ver,
2816 FrameType::HelloAck,
2817 control_flags(),
2818 0,
2819 0,
2820 frame.header.corr,
2821 body,
2822 )
2823 .map_err(RouterError::FrameBuild)
2824 }
2825
2826 async fn handle_client_control_request(
2827 &self,
2828 ctx: &RouteCtx,
2829 frame: Frame,
2830 request: ClientControlRequest,
2831 route_open_timings: &mut Option<RouteOpenTimings>,
2832 ) -> Result<Vec<Frame>, RouterError> {
2833 if let Some(operation) = privileged_operation(&request) {
2834 let caller = self.resolve_caller(ctx.connection_id).await;
2835 let op = client_control_request_op(&request);
2836 let text = prompt_text(operation, &caller);
2837 match self.privileged_operator_prompt {
2838 PrivilegedOperatorPrompt::Log => {
2839 info!(target: "subc_daemon::privileged_prompt", operation = op,
2840 target = ?operation, caller = %caller.description(), summary = %text,
2841 "privileged request would prompt the operator");
2842 }
2843 PrivilegedOperatorPrompt::Enforce => {
2844 let key =
2845 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2846 let outcome = self
2847 .forwarding
2848 .operator_confirms()
2849 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2850 .await;
2851 if outcome.code != "confirmed" {
2852 return Ok(vec![outcome.frame(&frame)]);
2853 }
2854 }
2855 }
2856 }
2857 match request {
2858 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2859 ClientControlRequest::CatalogList { module_id } => {
2860 self.handle_catalog_list(frame, module_id)
2861 }
2862 ClientControlRequest::RouteOpen {
2863 target,
2864 identity,
2865 consumer_identity,
2866 consumer_capabilities,
2867 role_versions,
2868 admission_facts,
2869 scope,
2870 } => {
2871 self.handle_route_open(
2872 ctx,
2873 frame,
2874 RouteOpenRequest {
2875 target,
2876 identity,
2877 consumer_identity,
2878 consumer_capabilities,
2879 role_versions,
2880 admission_facts,
2881 scope,
2882 },
2883 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2884 )
2885 .await
2886 }
2887 ClientControlRequest::RoutePoll {
2888 route_channel,
2889 route_epoch,
2890 kind,
2891 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2892 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2893 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2894 self.handle_supervisor_spawn_snapshot(frame)
2895 }
2896 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2897 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2898 }
2899 ClientControlRequest::SupervisorRestart {
2900 module_id,
2901 drain_timeout_ms,
2902 } => {
2903 self.log_supervisor_request_received(
2904 ctx,
2905 frame.header.corr,
2906 ops::SUPERVISOR_RESTART,
2907 Some(&module_id),
2908 None,
2909 )?;
2910 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2911 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2912 .await
2913 }
2914 ClientControlRequest::SupervisorSwap {
2915 module_id,
2916 ready_timeout_ms,
2917 } => {
2918 self.log_supervisor_request_received(
2919 ctx,
2920 frame.header.corr,
2921 ops::SUPERVISOR_SWAP,
2922 Some(&module_id),
2923 None,
2924 )?;
2925 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2926 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2927 .await
2928 }
2929 ClientControlRequest::SupervisorReload { module_id } => {
2930 self.log_supervisor_request_received(
2931 ctx,
2932 frame.header.corr,
2933 ops::SUPERVISOR_RELOAD,
2934 Some(&module_id),
2935 None,
2936 )?;
2937 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2938 self.handle_supervisor_reload(frame, module_id, requester)
2939 .await
2940 }
2941 ClientControlRequest::SupervisorRescan { preview } => {
2942 if !preview {
2943 self.log_supervisor_request_received(
2944 ctx,
2945 frame.header.corr,
2946 ops::SUPERVISOR_RESCAN,
2947 None,
2948 None,
2949 )?;
2950 }
2951 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2952 self.handle_supervisor_rescan(
2953 frame,
2954 if preview {
2955 RescanMode::Preview
2956 } else {
2957 RescanMode::Execute
2958 },
2959 requester,
2960 )
2961 .await
2962 }
2963 ClientControlRequest::SupervisorRescanChecked {
2964 preview,
2965 candidate,
2966 expect,
2967 } => {
2968 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
2969 .await
2970 }
2971 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2972 self.log_supervisor_request_received(
2973 ctx,
2974 frame.header.corr,
2975 ops::SUPERVISOR_RELEASE_RESERVED,
2976 Some(&module_id),
2977 None,
2978 )?;
2979 self.handle_supervisor_release_reserved(frame, module_id)
2980 .await
2981 }
2982 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2983 self.log_supervisor_request_received(
2984 ctx,
2985 frame.header.corr,
2986 ops::SUPERVISOR_SET_ENABLED,
2987 Some(&module_id),
2988 Some(enabled),
2989 )?;
2990 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2991 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2992 .await
2993 }
2994 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2995 self.handle_supervisor_health_probe(frame, module_id).await
2996 }
2997 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2998 ClientControlRequest::SupervisorRoutes { module_id } => {
2999 self.handle_supervisor_routes(frame, module_id)
3000 }
3001 ClientControlRequest::SupervisorProvenance { module_id } => {
3002 self.handle_supervisor_provenance(frame, module_id).await
3003 }
3004 ClientControlRequest::SupervisorStderrTail {
3005 module_id,
3006 max_lines,
3007 max_bytes,
3008 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3009 ClientControlRequest::SupervisorTerminals { module_id } => {
3010 self.handle_supervisor_terminals(frame, module_id).await
3011 }
3012 }
3013 }
3014
3015 fn log_supervisor_request_received(
3016 &self,
3017 ctx: &RouteCtx,
3018 corr: u64,
3019 op: &'static str,
3020 module_id: Option<&str>,
3021 enabled: Option<bool>,
3022 ) -> Result<(), RouterError> {
3023 let caller = self.supervisor_requester(ctx, corr)?;
3024 let caller = principal_label(&caller);
3025 info!(
3028 target: "control",
3029 op,
3030 module_id,
3031 enabled,
3032 connection_id = ctx.connection_id.get(),
3033 caller = %caller,
3034 "supervisor request received"
3035 );
3036 Ok(())
3037 }
3038
3039 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3040 Ok(self
3041 .registry
3042 .get_module_by_connection(ctx.connection_id)
3043 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3044 .map(|registration| Principal::Reserved {
3045 module_id: registration.manifest.module_id,
3046 })
3047 .unwrap_or(Principal::Direct))
3048 }
3049
3050 fn handle_module_control_request(
3051 &self,
3052 connection_id: ConnectionId,
3053 frame: Frame,
3054 request: ModuleControlRequestFromModule,
3055 ) -> Result<Vec<Frame>, RouterError> {
3056 match request {
3057 ModuleControlRequestFromModule::CatalogUpdate {
3058 provides,
3059 capabilities,
3060 ready,
3061 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3062 ModuleControlRequestFromModule::LiveRoots {} => {
3063 let registered = self
3064 .registry
3065 .get_module_by_connection(connection_id)
3066 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3067 let Some(registration) = registered else {
3068 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3069 };
3070 let response = self
3071 .forwarding
3072 .live_roots(®istration.manifest.module_id)
3073 .map_err(RouterError::Forwarding)?;
3074 Ok(vec![control_response_body_frame(
3075 &frame,
3076 &response,
3077 "ModuleControlResponseToModule::LiveRoots",
3078 )?])
3079 }
3080 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3081 self.handle_scope_sync(connection_id, frame, generation, scopes)
3082 }
3083 ModuleControlRequestFromModule::ScopeApply {
3084 generation,
3085 upsert,
3086 end,
3087 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3088 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3089 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3090 }
3091 }
3092 }
3093
3094 fn handle_scope_served_by(
3095 &self,
3096 connection_id: ConnectionId,
3097 frame: Frame,
3098 ) -> Result<Vec<Frame>, RouterError> {
3099 let registered = self
3100 .registry
3101 .get_module_by_connection(connection_id)
3102 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3103 let Some(registered) = registered else {
3104 return Ok(vec![control_error_frame(
3105 &frame,
3106 "not_registered",
3107 "scope.served_by requires an active module registration owned by this connection",
3108 )?]);
3109 };
3110 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3111 {
3112 Ok(request) => request,
3113 Err(err) => {
3114 return Ok(vec![control_error_frame(
3115 &frame,
3116 "invalid_control_body",
3117 format!("malformed scope.served_by body: {err}"),
3118 )?])
3119 }
3120 };
3121 let answer = self
3122 .scopes
3123 .read()
3124 .unwrap_or_else(|poisoned| poisoned.into_inner())
3125 .served_by(
3126 ®istered.manifest.module_id,
3127 &request.scope_ref,
3128 request.scope_epoch,
3129 &request.module_id,
3130 );
3131 let reply = subc_control::ScopeServedByReply::new(
3132 answer,
3133 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3134 );
3135 Ok(vec![control_response_body_frame(
3136 &frame,
3137 &reply,
3138 "ScopeServedByReply",
3139 )?])
3140 }
3141
3142 fn handle_operator_confirm(
3143 &self,
3144 ctx: &RouteCtx,
3145 frame: Frame,
3146 ) -> Result<Vec<Frame>, RouterError> {
3147 use crate::operator_confirm::{audit, Outcome};
3148 let registration = self
3149 .registry
3150 .get_module_by_connection(ctx.connection_id)
3151 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3152 let Some(registration) = registration else {
3153 let outcome = Outcome::refusal("not_registered");
3154 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3155 return Ok(vec![outcome.frame(&frame)]);
3156 };
3157 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3158 Ok(request) => request,
3159 Err(_) => {
3160 let outcome = Outcome::refusal("invalid_control_body");
3161 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3162 return Ok(vec![outcome.frame(&frame)]);
3163 }
3164 };
3165 let module_id = registration.manifest.module_id;
3166 let nonce = self
3170 .hello_launch_nonces
3171 .lock()
3172 .unwrap_or_else(|p| p.into_inner())
3173 .nonce(ctx.connection_id)
3174 .map(str::to_owned);
3175 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3176 self.supervisor
3177 .spawned_consumer_authorized(&module_id, nonce)
3178 });
3179 let confirms = self.forwarding.operator_confirms();
3180 self.forwarding
3181 .with_operator_route(
3182 ctx.connection_id,
3183 request.route_channel,
3184 request.route_epoch,
3185 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3186 )
3187 .map_err(RouterError::Forwarding)
3188 }
3189
3190 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3193 let mut scopes = ScopeRecords::from(scopes);
3194 for (index, record) in scopes.records.iter_mut().enumerate() {
3195 if let Some(refusal) = Self::canonicalize_workdir(record) {
3196 scopes.refusals.insert(index, refusal);
3197 }
3198 }
3199 scopes
3200 }
3201
3202 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3203 let workdir = record.workdir.as_mut()?;
3204 let path = std::path::Path::new(workdir);
3205 if !path.is_absolute() {
3206 return Some((
3207 error_codes::INVALID_WORKDIR,
3208 "workdir must be absolute".to_string(),
3209 ));
3210 }
3211 match ProjectRootId::from_path_allowing_missing(path) {
3212 Ok(root) => match root.as_path().to_str() {
3213 Some(canonical) => {
3214 *workdir = canonical.to_string();
3215 None
3216 }
3217 None => Some((
3218 error_codes::INVALID_WORKDIR,
3219 "canonical workdir is not UTF-8".to_string(),
3220 )),
3221 },
3222 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3223 }
3224 }
3225
3226 fn handle_scope_sync(
3230 &self,
3231 connection_id: ConnectionId,
3232 frame: Frame,
3233 generation: u64,
3234 scopes: Vec<ScopeRecord>,
3235 ) -> Result<Vec<Frame>, RouterError> {
3236 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3237 }
3238
3239 fn handle_scope_change(
3240 &self,
3241 connection_id: ConnectionId,
3242 frame: Frame,
3243 generation: u64,
3244 scopes: Vec<ScopeRecord>,
3245 end: Option<Vec<ScopeEnd>>,
3246 ) -> Result<Vec<Frame>, RouterError> {
3247 let is_apply = end.is_some();
3248 let Some(registration) = self
3249 .registry
3250 .get_module_by_connection(connection_id)
3251 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3252 else {
3253 return Ok(vec![control_error_frame(
3254 &frame,
3255 "not_registered",
3256 "scope.sync and scope.apply require an active module registration owned by this connection",
3257 )?]);
3258 };
3259 let owner = registration.manifest.module_id;
3260 let scopes = Self::prepare_scope_records(scopes);
3261 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3262 let is_current_launch = |connection: ConnectionId| {
3263 self.hello_launch_nonces
3264 .lock()
3265 .unwrap_or_else(|poisoned| poisoned.into_inner())
3266 .presented(connection, current_nonce.as_deref())
3267 };
3268 let mut table = self
3273 .scopes
3274 .write()
3275 .unwrap_or_else(|poisoned| poisoned.into_inner());
3276 let outcome = match end {
3277 Some(end) => table.apply(
3278 &owner,
3279 connection_id,
3280 is_current_launch,
3281 generation,
3282 scopes,
3283 end,
3284 ),
3285 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3286 };
3287 let drained = match &outcome {
3288 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3289 Err(_) => Vec::new(),
3290 };
3291 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3292 drop(table);
3293 if has_deadlines {
3294 self.scope_deadline_added.notify_one();
3295 }
3296 match outcome {
3297 Ok(applied) => {
3298 let counts = ScopeOutcomeCounts::of(&applied.results);
3299 info!(
3300 owner = %owner,
3301 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3302 generation,
3303 records = applied.results.len(),
3304 created = counts.created,
3305 replaced = counts.replaced,
3306 updated = counts.updated,
3307 unchanged = counts.unchanged,
3308 refused = counts.refused,
3309 ended = applied.ended.len(),
3310 tag_changes = applied.tag_changes.len(),
3311 routes_closed = drained.len(),
3312 "scope change accepted"
3313 );
3314 for refused in applied
3320 .results
3321 .iter()
3322 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3323 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3324 {
3325 warn!(
3326 owner = %owner,
3327 generation,
3328 scope_ref = %refused.scope_ref,
3329 scope_epoch = refused.scope_epoch,
3330 code = refused.code.as_deref().unwrap_or(""),
3331 "scope record refused"
3332 );
3333 }
3334 self.close_scope_drained_routes(drained);
3335 let response = if is_apply {
3336 ModuleControlResponseToModule::ScopeApply {
3337 generation,
3338 results: applied.results,
3339 end_results: applied.end_results,
3340 ended: applied.ended,
3341 }
3342 } else {
3343 ModuleControlResponseToModule::ScopeSync {
3344 generation,
3345 results: applied.results,
3346 ended: applied.ended,
3347 }
3348 };
3349 Ok(vec![control_response_body_frame(
3350 &frame,
3351 &response,
3352 "scope change response",
3353 )?])
3354 }
3355 Err(refusal) => {
3356 info!(
3357 owner = %owner,
3358 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3359 generation,
3360 code = refusal.code,
3361 "scope change refused"
3362 );
3363 Ok(vec![control_error_frame(
3364 &frame,
3365 refusal.code,
3366 refusal.message,
3367 )?])
3368 }
3369 }
3370 }
3371
3372 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3378 if drained.is_empty() {
3379 return;
3380 }
3381 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3382 BTreeMap::new();
3383 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3384 for route in drained {
3385 warn!(
3386 module_id = %route.module_id,
3387 reason = ?route.reason,
3388 client_connection_id = route.client.connection_id.get(),
3389 route_channel = route.client.channel,
3390 "closing route because its scope changed"
3391 );
3392 pushes
3393 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3394 .or_insert_with(|| (route.reason, Vec::new()))
3395 .1
3396 .push(EndpointRoute {
3397 goodbye_target: route.client.clone(),
3398 principal: Principal::Unverified,
3399 bound_at: Instant::now(),
3400 draining: false,
3401 drain_reason: None,
3402 });
3403 goodbyes.push(route.module);
3404 goodbyes.push(route.client);
3405 }
3406 for ((module_id, _), (reason, routes)) in pushes {
3407 send_route_control_pushes(
3408 &self.forwarding,
3409 routes,
3410 ClientControlPush::RouteClosed {
3411 module_id,
3412 channels: Vec::new(),
3413 reason,
3414 drained: false,
3415 abandoned: 0,
3416 excluded_subscriptions: 0,
3417 terminal: Some(false),
3418 },
3419 );
3420 }
3421 self.emit_route_goodbyes(goodbyes);
3422 }
3423
3424 fn handle_scope_describe(
3427 &self,
3428 connection_id: ConnectionId,
3429 frame: Frame,
3430 owner: Principal,
3431 scope_ref: String,
3432 ) -> Result<Vec<Frame>, RouterError> {
3433 let registered = self
3434 .registry
3435 .get_module_by_connection(connection_id)
3436 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3437 let Some(registered) = registered else {
3438 return Ok(vec![control_error_frame(
3439 &frame,
3440 "not_registered",
3441 "scope.describe requires an active module registration owned by this connection",
3442 )?]);
3443 };
3444 let mut description = self
3445 .scopes
3446 .read()
3447 .unwrap_or_else(|poisoned| poisoned.into_inner())
3448 .describe(&owner, &scope_ref);
3449 if !registered
3452 .manifest
3453 .capabilities
3454 .as_ref()
3455 .is_some_and(|caps| {
3456 caps.provides
3457 .iter()
3458 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3459 })
3460 {
3461 if let Some(stamp) = description.stamp.as_mut() {
3462 stamp.attributes.on_behalf_of_agent = None;
3463 }
3464 }
3465 if !registered
3466 .manifest
3467 .capabilities
3468 .as_ref()
3469 .is_some_and(|caps| {
3470 caps.provides
3471 .iter()
3472 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3473 })
3474 {
3475 if let Some(stamp) = description.stamp.as_mut() {
3476 stamp.attributes.flow_version = None;
3477 stamp.attributes.flow_code_hash = None;
3478 stamp.attributes.flow_install_id = None;
3479 }
3480 }
3481 let owner_configured = match &owner {
3482 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3487 _ => false,
3488 };
3489 let response = ModuleControlResponseToModule::ScopeDescribe {
3490 status: description.status,
3491 scope_epoch: description.scope_epoch,
3492 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3493 owner_synced: description.owner_synced,
3494 owner_configured,
3495 scope: description.stamp,
3496 };
3497 Ok(vec![control_response_body_frame(
3498 &frame,
3499 &response,
3500 "ModuleControlResponseToModule::ScopeDescribe",
3501 )?])
3502 }
3503
3504 fn handle_catalog_update(
3505 &self,
3506 connection_id: ConnectionId,
3507 frame: Frame,
3508 provides: Vec<ProviderRole>,
3509 capabilities: Option<CapabilityDeclarations>,
3510 ready: Option<bool>,
3511 ) -> Result<Vec<Frame>, RouterError> {
3512 self.refresh_capability_requirements();
3513 let Some(registration) = self
3514 .registry
3515 .get_module_by_connection(connection_id)
3516 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3517 else {
3518 return Ok(vec![control_error_frame(
3519 &frame,
3520 "not_registered",
3521 "catalog.update requires an active module registration owned by this connection",
3522 )?]);
3523 };
3524
3525 if let Some(message) =
3526 catalog_update_frozen_field_message(®istration.manifest, &provides)
3527 {
3528 return Ok(vec![control_error_frame(
3529 &frame,
3530 "catalog_update_frozen_field",
3531 message,
3532 )?]);
3533 }
3534
3535 let mut candidate = registration.manifest.clone();
3536 candidate.provides = provides.clone();
3537 candidate.capabilities = capabilities
3538 .clone()
3539 .or_else(|| registration.manifest.capabilities.clone());
3540 if let Some(rich_ui) = &candidate.rich_ui {
3541 if let Err(err) = rich_ui.validate() {
3542 return Ok(vec![control_error_frame(
3543 &frame,
3544 "invalid_manifest",
3545 err.to_string(),
3546 )?]);
3547 }
3548 }
3549 if let Err(err) = candidate.validate_capability_grammar() {
3550 return Ok(vec![control_error_frame(
3551 &frame,
3552 "invalid_capability_grammar",
3553 err.to_string(),
3554 )?]);
3555 }
3556
3557 let mut conflicts = self
3560 .capability_evaluator
3561 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3562 if let Some(conflict) = conflicts.first() {
3563 let message = format!(
3564 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3565 conflict.capability, conflict.claimants[0], candidate.module_id
3566 );
3567 for conflict in &mut conflicts {
3568 conflict.source = DuplicateClaimSource::CatalogUpdate;
3569 }
3570 log_duplicate_claim_events(conflicts);
3571 return Ok(vec![control_error_frame(
3572 &frame,
3573 "reserved_capability",
3574 message,
3575 )?]);
3576 }
3577
3578 let updated = self
3579 .registry
3580 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3581 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3582 if updated.is_none() {
3583 return Ok(vec![control_error_frame(
3584 &frame,
3585 "not_registered",
3586 "catalog.update requires an active module registration owned by this connection",
3587 )?]);
3588 }
3589 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3590 log_duplicate_claim_events(
3591 self.capability_evaluator
3592 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3593 );
3594 }
3595 if capability_census_trigger(
3596 registration.manifest.capabilities.as_ref(),
3597 updated
3598 .as_ref()
3599 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3600 ) {
3601 self.enforce_capability_denies();
3602 }
3603 self.refresh_capability_requirements();
3604
3605 let response = ModuleControlResponseToModule::CatalogUpdate {};
3606 control_response_body_frame(
3607 &frame,
3608 &response,
3609 "ModuleControlResponseToModule::CatalogUpdate",
3610 )
3611 .map(|frame| vec![frame])
3612 }
3613
3614 pub(crate) fn with_activity_ledger_config(
3615 mut self,
3616 run_dir: &Path,
3617 incarnation: String,
3618 config: crate::activity::ActivityConfig,
3619 ) -> Self {
3620 self.activity = Some(crate::activity::ActivityLedger::open_config(
3621 run_dir.join("activity.jsonl"),
3622 incarnation,
3623 config,
3624 ));
3625 self.supervisor.set_activity(self.activity.clone());
3626 self
3627 }
3628
3629 pub(crate) async fn configure_activity(
3630 &self,
3631 config: crate::activity::ActivityConfig,
3632 ) -> Result<(), &'static str> {
3633 match &self.activity {
3634 Some(ledger) => ledger.configure(config).await,
3635 None => Ok(()),
3636 }
3637 }
3638
3639 fn cancel_activity_subscription(
3640 &self,
3641 ctx: &RouteCtx,
3642 frame: &Frame,
3643 ) -> Result<bool, RouterError> {
3644 let task = self
3645 .activity_streams
3646 .lock()
3647 .unwrap_or_else(|p| p.into_inner())
3648 .remove(&(ctx.connection_id, frame.header.corr));
3649 let Some(task) = task else {
3650 return Ok(false);
3651 };
3652 task.abort();
3653 let end = Frame::build_with_version(
3654 response_version(frame),
3655 FrameType::StreamEnd,
3656 control_flags(),
3657 0,
3658 0,
3659 frame.header.corr,
3660 Vec::new(),
3661 )
3662 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3663 let sink = ctx.egress.clone();
3664 tokio::spawn(async move {
3665 let _ = sink.send(end).await;
3666 });
3667 Ok(true)
3668 }
3669
3670 async fn handle_activity_read(
3671 &self,
3672 ctx: &RouteCtx,
3673 frame: Frame,
3674 ) -> Result<Vec<Frame>, RouterError> {
3675 use subc_control::{
3676 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3677 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3678 };
3679 #[derive(Deserialize)]
3680 struct Identity {
3681 #[serde(default)]
3682 consumer_identity: Option<ConsumerIdentity>,
3683 }
3684 let principal = match self
3688 .forwarding
3689 .module_id_for_connection(ctx.connection_id)
3690 .map_err(RouterError::Forwarding)?
3691 {
3692 Some(module_id) => Ok(Principal::Reserved { module_id }),
3693 None => {
3694 let identity: Identity = match serde_json::from_slice(&frame.body) {
3695 Ok(identity) => identity,
3696 Err(_) => {
3697 return Ok(vec![control_error_frame(
3698 &frame,
3699 ACTIVITY_MALFORMED,
3700 "malformed activity read",
3701 )?])
3702 }
3703 };
3704 self.route_open_principal(&frame, identity.consumer_identity)?
3705 }
3706 };
3707 let admitted = matches!(principal, Ok(Principal::Direct))
3708 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3709 if !admitted {
3710 return Ok(vec![control_error_frame(
3711 &frame,
3712 ACTIVITY_READ_FORBIDDEN,
3713 "activity reads require an operator or trusted mirror principal",
3714 )?]);
3715 }
3716 let Some(ledger) = &self.activity else {
3717 return Ok(vec![control_error_frame(
3718 &frame,
3719 ACTIVITY_UNAVAILABLE,
3720 "activity ledger unavailable",
3721 )?]);
3722 };
3723 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3724 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3725 {
3726 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3727 Ok(request) => ledger.query(request).await,
3728 Err(_) => Err(ACTIVITY_MALFORMED),
3729 };
3730 return match result {
3731 Ok(reply) => Ok(vec![control_response_body_frame(
3732 &frame,
3733 &reply,
3734 ops::ACTIVITY_QUERY,
3735 )?]),
3736 Err(code) => Ok(vec![control_error_frame(
3737 &frame,
3738 code,
3739 "activity query refused",
3740 )?]),
3741 };
3742 }
3743 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3744 Ok(request) => request,
3745 Err(_) => {
3746 return Ok(vec![control_error_frame(
3747 &frame,
3748 ACTIVITY_MALFORMED,
3749 "malformed activity subscription",
3750 )?])
3751 }
3752 };
3753 let mut subscription = match ledger.subscribe(request.since).await {
3754 Ok(subscription) => subscription,
3755 Err(code) => {
3756 return Ok(vec![control_error_frame(
3757 &frame,
3758 code,
3759 "activity subscription refused",
3760 )?])
3761 }
3762 };
3763 let key = (ctx.connection_id, frame.header.corr);
3764 let sink = ctx.egress.clone();
3765 let mut streams = self
3766 .activity_streams
3767 .lock()
3768 .unwrap_or_else(|p| p.into_inner());
3769 streams.retain(|_, task| !task.is_finished());
3770 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3771 return Ok(vec![control_error_frame(
3772 &frame,
3773 "duplicate_subscription",
3774 "activity subscription correlation id already in use",
3775 )?]);
3776 }
3777 let task = tokio::spawn(async move {
3778 loop {
3779 let entry = if let Some(entry) = subscription.replay.pop_front() {
3780 Some(entry)
3781 } else {
3782 subscription.receiver.recv().await
3783 };
3784 let Some(entry) = entry else {
3785 break;
3786 };
3787 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3788 return;
3789 };
3790 let Ok(next) = Frame::build_with_version(
3791 response_version(&frame),
3792 FrameType::StreamData,
3793 control_flags(),
3794 0,
3795 0,
3796 frame.header.corr,
3797 body,
3798 ) else {
3799 return;
3800 };
3801 if sink.send(next).await.is_err() {
3802 return;
3803 }
3804 }
3805 if let Some(code) = subscription.terminal_code() {
3808 if let Ok(end) =
3809 control_error_frame(&frame, code, "activity subscriber fell behind")
3810 {
3811 let _ = sink.send(end).await;
3812 }
3813 }
3814 });
3815 streams.insert(key, task.abort_handle());
3816 Ok(Vec::new())
3817 }
3818
3819 async fn handle_activity_record(
3820 &self,
3821 ctx: &RouteCtx,
3822 frame: Frame,
3823 ) -> Result<Vec<Frame>, RouterError> {
3824 use subc_control::{
3825 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3826 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3827 };
3828 let result = async {
3829 let module = self
3830 .forwarding
3831 .module_id_for_connection(ctx.connection_id)
3832 .ok()
3833 .flatten()
3834 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3835 let request: ActivityRecordRequest =
3836 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3837 crate::activity::validate(&module, &request)?;
3838 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3839 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3840 let at_ms = std::time::SystemTime::now()
3841 .duration_since(std::time::UNIX_EPOCH)
3842 .unwrap_or_default()
3843 .as_millis() as u64;
3844 ledger.record(module, request, actor, at_ms).await
3845 }
3846 .await;
3847 match result {
3848 Ok(entry) => Ok(vec![control_response_body_frame(
3849 &frame,
3850 &ActivityRecordResponse { seq: entry.seq },
3851 "activity.record",
3852 )?]),
3853 Err(code) => Ok(vec![control_error_frame(
3854 &frame,
3855 code,
3856 "activity recording refused",
3857 )?]),
3858 }
3859 }
3860
3861 fn activity_actor(
3862 &self,
3863 connection: ConnectionId,
3864 module: &str,
3865 on: Option<&subc_control::ActivityOn>,
3866 ) -> Result<subc_control::ActivityActor, &'static str> {
3867 use crate::forwarding::{DataRoute, DataRouteState};
3868 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3869 let principal = |id: &str| Principal::Reserved {
3870 module_id: id.to_owned(),
3871 };
3872 let mut actor = ActivityActor {
3873 principal: principal(module),
3874 scope_owner: None,
3875 agent_id: None,
3876 run_id: None,
3877 flow_id: None,
3878 project_root: None,
3879 workdir: None,
3880 project_id: None,
3881 };
3882 let scope = match on {
3883 None => return Ok(actor),
3884 Some(ActivityOn::Scope(scope)) => {
3885 let selector = subc_protocol::scope::ScopeSelector::new(
3886 principal(module),
3887 scope.scope_ref.clone(),
3888 Some(scope.epoch),
3889 );
3890 self.scopes
3891 .read()
3892 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3893 .attribute(&principal(module), module, &selector, true)
3894 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3895 }
3896 Some(ActivityOn::Route(route)) => {
3897 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3898 .forwarding
3899 .lookup_data_route(connection, route.channel, route.epoch)
3900 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3901 else {
3902 return Err(ACTIVITY_ACTOR_UNKNOWN);
3903 };
3904 if binding.module_id != module {
3905 return Err(ACTIVITY_ACTOR_UNKNOWN);
3906 }
3907 actor.principal = binding.principal.clone();
3908 actor.project_root = binding
3909 .project_root
3910 .as_ref()
3911 .map(|root| root.as_path().to_path_buf());
3912 let Some(bound) = &binding.scope else {
3913 return Ok(actor);
3914 };
3915 let owner = principal(&bound.owner);
3916 let selector = subc_protocol::scope::ScopeSelector::new(
3917 owner.clone(),
3918 bound.scope_ref.clone(),
3919 Some(bound.tag.scope_epoch),
3920 );
3921 let admitted = self
3922 .scopes
3923 .read()
3924 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3925 .attribute(&owner, module, &selector, true)
3926 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3927 if admitted.tag != bound.tag {
3928 return Err(ACTIVITY_ACTOR_UNKNOWN);
3929 }
3930 admitted
3931 }
3932 };
3933 actor.scope_owner = Some(scope.stamp.owner);
3934 actor.agent_id = scope.stamp.attributes.agent_id;
3935 actor.run_id = scope.stamp.attributes.run_id;
3936 actor.flow_id = scope.stamp.attributes.flow_id;
3937 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
3938 actor.project_id = scope.stamp.project_id;
3939 Ok(actor)
3940 }
3941
3942 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3943 self.refresh_capability_requirements();
3944 let mut counters = self.counters.snapshot();
3950 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3951 obj.insert(
3952 "activity_unparseable_lines".into(),
3953 ledger.read_errors().into(),
3954 );
3955 obj.insert(
3956 "activity_lifecycle_append_failures".into(),
3957 ledger.lifecycle_failures().into(),
3958 );
3959 obj.insert(
3960 "activity_started_digest_skips".into(),
3961 ledger.lifecycle_digest_skips().into(),
3962 );
3963 }
3964 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3965 self.forwarding.client_route_concentration(),
3966 counters.as_object_mut(),
3967 ) {
3968 obj.insert(
3969 "client_connections_with_routes".into(),
3970 connections_with_routes.into(),
3971 );
3972 obj.insert("max_routes_on_one_connection".into(), max.into());
3973 }
3974 if let (Some(open_breakers), Some(obj)) = (
3980 self.route_bind_breakers.open_snapshot(),
3981 counters.as_object_mut(),
3982 ) {
3983 obj.insert("route_bind_breakers_open".into(), open_breakers);
3984 }
3985 if let (Some(reason), Some(obj)) = (
3990 self.local_endpoint_unavailable.clone(),
3991 counters.as_object_mut(),
3992 ) {
3993 obj.insert("local_endpoint_unavailable".into(), reason.into());
3994 }
3995 let response = ClientControlResponse::ServerDescribe {
3996 protocol_ver: PROTOCOL_VERSION,
3997 subc_ops: subc_ops(),
3998 capabilities: self.describe_capabilities(),
3999 connected_clients: self.connected_clients.count(),
4000 counters: Some(counters),
4001 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4002 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4003 capability_requirements: self.capability_requirement_statuses(),
4004 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4005 config_path: self
4006 .rescan
4007 .as_ref()
4008 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4009 };
4010 Ok(vec![control_response_body_frame(
4011 &frame,
4012 &response,
4013 "ClientControlResponse::ServerDescribe",
4014 )?])
4015 }
4016
4017 fn handle_catalog_list(
4018 &self,
4019 frame: Frame,
4020 module_id: Option<String>,
4021 ) -> Result<Vec<Frame>, RouterError> {
4022 let (generation, modules) = self
4023 .registry
4024 .list_modules_with_nonce_verification()
4025 .map_err(|err| {
4026 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4027 })?;
4028 let entries = modules
4029 .into_iter()
4030 .filter(|(registration, _)| {
4031 module_id
4032 .as_deref()
4033 .map(|wanted| registration.manifest.module_id == wanted)
4034 .unwrap_or(true)
4035 })
4036 .map(|(registration, nonce_verified)| {
4037 let not_ready = self.not_ready_reason(®istration);
4038 let roles = registration.manifest.provides;
4039 CatalogEntry::new(
4040 registration.manifest.module_id,
4041 roles,
4042 registration.control_ops,
4043 )
4044 .with_ready(not_ready.is_none())
4045 .with_not_ready(not_ready)
4046 .with_module_version(Some(registration.manifest.module_version))
4047 .with_capabilities(registration.manifest.capabilities)
4048 .with_self_signals(registration.manifest.self_signals)
4049 .with_events(registration.manifest.events)
4050 .with_operation_delivery(registration.manifest.operation_delivery)
4051 .with_operation_payload(registration.manifest.operation_payload)
4052 .with_flow_operations(registration.manifest.flow_operations)
4053 .with_rich_ui(registration.manifest.rich_ui)
4054 .with_role_declarations(registration.manifest.role_declarations)
4055 .with_nonce_verified(nonce_verified)
4056 })
4057 .collect();
4058 let response = ClientControlResponse::CatalogList {
4059 generation,
4060 modules: entries,
4061 subc_ops: subc_ops(),
4062 };
4063 Ok(vec![control_response_body_frame(
4064 &frame,
4065 &response,
4066 "ClientControlResponse::CatalogList",
4067 )?])
4068 }
4069
4070 fn attest_route_open_consumer(
4075 &self,
4076 consumer_identity: Option<&ConsumerIdentity>,
4077 ) -> Option<Principal> {
4078 let Some(consumer_identity) = consumer_identity else {
4079 return Some(Principal::Direct);
4080 };
4081 self.supervisor
4082 .spawned_consumer_authorized(
4083 &consumer_identity.module_id,
4084 &consumer_identity.launch_nonce,
4085 )
4086 .then(|| Principal::Reserved {
4087 module_id: consumer_identity.module_id.clone(),
4088 })
4089 }
4090
4091 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4095 let Ok(ClientControlRequest::RouteOpen {
4096 identity,
4097 consumer_identity,
4098 scope,
4099 ..
4100 }) = parse_client_control_request(&frame.body)
4101 else {
4102 return RouteOpenCaller::unknown();
4103 };
4104 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4105 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4106 }
4107
4108 fn route_open_principal(
4109 &self,
4110 frame: &Frame,
4111 consumer_identity: Option<ConsumerIdentity>,
4112 ) -> Result<Result<Principal, Frame>, RouterError> {
4113 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4114 self.route_open_principal_attested(frame, consumer_identity, attested)
4115 }
4116
4117 fn route_open_principal_attested(
4121 &self,
4122 frame: &Frame,
4123 consumer_identity: Option<ConsumerIdentity>,
4124 attested: Option<Principal>,
4125 ) -> Result<Result<Principal, Frame>, RouterError> {
4126 if let Some(principal) = attested {
4127 return Ok(Ok(principal));
4128 }
4129 let Some(consumer_identity) = consumer_identity else {
4130 return Ok(Ok(Principal::Direct));
4131 };
4132
4133 Ok(Err(control_error_frame(
4134 frame,
4135 "bad_consumer_identity",
4136 format!(
4137 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4138 consumer_identity.module_id
4139 ),
4140 )?))
4141 }
4142
4143 #[allow(clippy::too_many_arguments)]
4150 fn route_open_refusal_frame(
4151 &self,
4152 ctx: &RouteCtx,
4153 caller: &RouteOpenCaller,
4154 frame: &Frame,
4155 module_id: &str,
4156 reason: &'static str,
4157 code: &'static str,
4158 message: impl Into<String>,
4159 ) -> Result<Frame, RouterError> {
4160 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4161 control_error_frame(frame, code, message.into())
4162 }
4163
4164 #[allow(clippy::too_many_arguments)]
4173 fn route_bind_rejection(
4174 &self,
4175 ctx: &RouteCtx,
4176 caller: &RouteOpenCaller,
4177 frame: &Frame,
4178 target_module_id: &str,
4179 body: ErrorBody,
4180 daemon_authored: bool,
4181 ) -> Result<Vec<Frame>, RouterError> {
4182 if daemon_authored {
4188 let scope_code = match body.code.as_str() {
4189 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4190 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4191 _ => None,
4192 };
4193 if let Some(code) = scope_code {
4194 self.observe_route_open_refusal(
4195 ctx,
4196 caller,
4197 target_module_id,
4198 "scope_changed_before_commit",
4199 code,
4200 );
4201 return Ok(vec![control_error_body_frame(frame, body)?]);
4202 }
4203 }
4204 self.counters
4205 .increment_route_open_refused("module_rejected");
4206 info!(
4207 target: "control",
4208 code = "module_rejected",
4209 module_code = ?body.code,
4210 module_id = ?target_module_id,
4211 connection_id = ctx.connection_id.get(),
4212 principal = %caller.principal,
4213 scope = %caller.scope,
4214 harness = %caller.harness,
4215 "route.open refused"
4216 );
4217 let refusal = if daemon_authored {
4218 control_error_body_frame(frame, body)?
4219 } else {
4220 control_relayed_error_body_frame(frame, body)?
4221 };
4222 Ok(vec![refusal])
4223 }
4224
4225 #[allow(clippy::too_many_arguments)]
4243 fn route_open_breaker_refusal_frame(
4244 &self,
4245 ctx: &RouteCtx,
4246 caller: &RouteOpenCaller,
4247 frame: &Frame,
4248 module_id: &str,
4249 consecutive_timeouts: u32,
4250 retry_in: Duration,
4251 probe_in_flight: bool,
4252 ) -> Result<Frame, RouterError> {
4253 self.counters
4254 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4255 debug!(
4256 target: "control",
4257 code = "module_timeout",
4258 module_id = ?module_id,
4259 connection_id = ctx.connection_id.get(),
4260 principal = %caller.principal,
4261 scope = %caller.scope,
4262 harness = %caller.harness,
4263 consecutive_timeouts,
4264 retry_in_ms = retry_in.as_millis() as u64,
4265 probe_in_flight,
4266 "route.open refused by open bind-relay breaker"
4267 );
4268 let detail = if probe_in_flight {
4275 "one test route is already being tried; retry once it settles".to_string()
4276 } else {
4277 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4278 };
4279 control_error_frame(
4280 frame,
4281 "module_timeout",
4282 format!(
4283 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4284 timed out in a row); {detail}; its established routes are unaffected"
4285 ),
4286 )
4287 }
4288
4289 fn observe_route_open_refusal(
4302 &self,
4303 ctx: &RouteCtx,
4304 caller: &RouteOpenCaller,
4305 module_id: &str,
4306 reason: &'static str,
4307 code: &'static str,
4308 ) {
4309 self.counters.increment_route_open_refused(code);
4310 info!(
4311 target: "control",
4312 code,
4313 reason,
4314 module_id = ?module_id,
4315 connection_id = ctx.connection_id.get(),
4316 principal = %caller.principal,
4317 scope = %caller.scope,
4318 harness = %caller.harness,
4319 "route.open refused"
4320 );
4321 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4322 self.route_outages.record_not_serving(module_id, reason);
4323 }
4324 }
4325
4326 fn observe_route_open_accept(
4371 &self,
4372 ctx: &RouteCtx,
4373 module_id: &str,
4374 principal: &str,
4375 scope: &str,
4376 ) {
4377 self.route_outages.record_accepted(module_id);
4378 self.counters.increment_route_open_accepted(principal);
4379 info!(
4380 target: "control",
4381 principal,
4382 module_id,
4383 connection_id = ctx.connection_id.get(),
4384 scope = %scope,
4385 "route.open accepted"
4386 );
4387 }
4388
4389 fn supervised_absent_route_open_refusal_frame(
4390 &self,
4391 ctx: &RouteCtx,
4392 caller: &RouteOpenCaller,
4393 frame: &Frame,
4394 module_id: &str,
4395 code: &'static str,
4396 status: &crate::supervise::ModuleStatus,
4397 ) -> Result<Frame, RouterError> {
4398 self.counters.increment_route_open_refused(code);
4399 info!(
4400 target: "control",
4401 code,
4402 reason = "supervised_not_registered",
4403 module_id = ?module_id,
4404 connection_id = ctx.connection_id.get(),
4405 principal = %caller.principal,
4406 scope = %caller.scope,
4407 harness = %caller.harness,
4408 state = %status.state,
4409 enabled = status.enabled,
4410 live = status.live,
4411 "route.open refused"
4412 );
4413 self.route_outages
4417 .record_not_serving(module_id, "supervised_not_registered");
4418 let message = if let Some(pid) = status.waiting_for_previous_pid {
4419 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4420 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4421 })
4422 } else {
4423 format!(
4424 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4425 status.state, status.enabled, status.live
4426 )
4427 };
4428 control_error_frame(frame, code, message)
4429 }
4430
4431 async fn handle_route_open(
4432 &self,
4433 ctx: &RouteCtx,
4434 frame: Frame,
4435 request: RouteOpenRequest,
4436 timings: &mut RouteOpenTimings,
4437 ) -> Result<Vec<Frame>, RouterError> {
4438 let RouteOpenRequest {
4439 target,
4440 mut identity,
4441 consumer_identity,
4442 consumer_capabilities,
4443 role_versions,
4444 admission_facts,
4445 scope,
4446 } = request;
4447 let target_module_id = target_module_id(&target).to_string();
4448 timings.module_id.clone_from(&target_module_id);
4449 if self
4450 .registry
4451 .get_module_by_connection(ctx.connection_id)
4452 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4453 .is_some()
4454 {
4455 return Ok(vec![control_error_frame(
4456 &frame,
4457 "invalid_request",
4458 "module connections cannot open client routes",
4459 )?]);
4460 }
4461 debug!(
4462 connection_id = ctx.connection_id.get(),
4463 corr = frame.header.corr,
4464 module_id = %target_module_id,
4465 "handling route.open"
4466 );
4467
4468 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4473 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4474
4475 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4481 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4482 self.observe_route_open_refusal(
4483 ctx,
4484 &caller,
4485 &target_module_id,
4486 "invalid_role_versions",
4487 error_codes::INVALID_REQUEST,
4488 );
4489 return Ok(vec![control_error_body_frame(
4490 &frame,
4491 ErrorBody {
4492 code: error_codes::INVALID_REQUEST.to_string(),
4493 message: error.to_string(),
4494 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4495 },
4496 )?]);
4497 }
4498
4499 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4500 if status.waiting_for_previous_pid.is_some() {
4501 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4502 ctx,
4503 &caller,
4504 &frame,
4505 &target_module_id,
4506 "module_warming",
4507 &status,
4508 )?]);
4509 }
4510 }
4511
4512 let Some(registration) = self
4531 .registry
4532 .get_module(&target_module_id)
4533 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4534 else {
4535 if let Some((status, warming)) =
4536 self.supervisor_status(&target_module_id, frame.header.corr)?
4537 {
4538 if status.protocol == ModuleProtocol::None {
4545 return Ok(vec![self.route_open_refusal_frame(
4546 ctx,
4547 &caller,
4548 &frame,
4549 &target_module_id,
4550 "protocol_none",
4551 error_codes::MODULE_NO_PROTOCOL,
4552 format!(
4553 "module_id '{target_module_id}' is declared protocol: none; \
4554 it speaks no subc wire and serves no routes"
4555 ),
4556 )?]);
4557 }
4558 let code = if warming {
4559 "module_warming"
4560 } else {
4561 "target_unavailable"
4562 };
4563 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4564 ctx,
4565 &caller,
4566 &frame,
4567 &target_module_id,
4568 code,
4569 &status,
4570 )?]);
4571 }
4572 if let Some(removed_ago_ms) =
4573 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4574 {
4575 return Ok(vec![self.route_open_refusal_frame(
4576 ctx,
4577 &caller,
4578 &frame,
4579 &target_module_id,
4580 "removed",
4581 error_codes::MODULE_REMOVED,
4582 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4583 )?]);
4584 }
4585 return Ok(vec![self.route_open_refusal_frame(
4586 ctx,
4587 &caller,
4588 &frame,
4589 &target_module_id,
4590 "not_registered",
4591 error_codes::UNKNOWN_MODULE,
4592 format!("module_id '{target_module_id}' is not registered"),
4593 )?]);
4594 };
4595
4596 if !registration.ready {
4600 self.counters
4601 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4602 info!(
4603 target: "control",
4604 code = error_codes::MODULE_WARMING,
4605 module_id = ?target_module_id,
4606 connection_id = ctx.connection_id.get(),
4607 principal = %caller.principal,
4608 scope = %caller.scope,
4609 harness = %caller.harness,
4610 reason = "declared_not_ready",
4611 "route.open refused"
4612 );
4613 self.route_outages
4616 .record_not_serving(&target_module_id, "declared_not_ready");
4617 return Ok(vec![control_error_body_frame(
4618 &frame,
4619 ErrorBody {
4620 code: error_codes::MODULE_WARMING.to_string(),
4621 message: format!(
4622 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4623 ),
4624 detail: Some(serde_json::json!({
4625 "reason": "declared_not_ready"
4626 })),
4627 },
4628 )?]);
4629 }
4630
4631 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4650 self.counters
4651 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4652 info!(
4653 target: "control",
4654 code = error_codes::MODULE_WARMING,
4655 module_id = ?target_module_id,
4656 connection_id = ctx.connection_id.get(),
4657 principal = %caller.principal,
4658 scope = %caller.scope,
4659 harness = %caller.harness,
4660 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4661 capability = %capability,
4662 "route.open refused"
4663 );
4664 return Ok(vec![control_error_body_frame(
4665 &frame,
4666 ErrorBody {
4667 code: error_codes::MODULE_WARMING.to_string(),
4668 message: format!(
4669 "module_id '{target_module_id}' requires capability '{capability}', \
4670 which no registered module provides; retry"
4671 ),
4672 detail: Some(serde_json::json!({
4673 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4674 "capability": capability,
4675 })),
4676 },
4677 )?]);
4678 }
4679
4680 if !target_has_required_role(&target, ®istration.manifest.provides) {
4681 return Ok(vec![self.route_open_refusal_frame(
4682 ctx,
4683 &caller,
4684 &frame,
4685 &target_module_id,
4686 "role_not_provided",
4687 "target_unavailable",
4688 format!("module_id '{target_module_id}' does not provide the requested target"),
4689 )?]);
4690 }
4691
4692 if registration.state != ChannelState::Active {
4693 return Ok(vec![self.route_open_refusal_frame(
4694 ctx,
4695 &caller,
4696 &frame,
4697 &target_module_id,
4698 "registration_not_active",
4699 "target_unavailable",
4700 format!("module_id '{target_module_id}' is not active"),
4701 )?]);
4702 }
4703
4704 if self
4705 .forwarding
4706 .module_is_draining(&target_module_id)
4707 .map_err(RouterError::Forwarding)?
4708 {
4709 return Ok(vec![self.route_open_refusal_frame(
4710 ctx,
4711 &caller,
4712 &frame,
4713 &target_module_id,
4714 "reloading",
4715 "module_reloading",
4716 format!("module_id '{target_module_id}' is reloading"),
4717 )?]);
4718 }
4719
4720 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4721 process_liveness.process_live(&target_module_id) == Some(false)
4722 }) {
4723 if process_liveness.process_replacing(&target_module_id) {
4730 return Ok(vec![self.route_open_refusal_frame(
4731 ctx,
4732 &caller,
4733 &frame,
4734 &target_module_id,
4735 "reloading",
4736 "module_reloading",
4737 format!("module_id '{target_module_id}' is reloading"),
4738 )?]);
4739 }
4740 return Ok(vec![self.route_open_refusal_frame(
4741 ctx,
4742 &caller,
4743 &frame,
4744 &target_module_id,
4745 "supervisor_not_live",
4746 "target_unavailable",
4747 format!("module_id '{target_module_id}' is not live"),
4748 )?]);
4749 }
4750
4751 if !self
4752 .forwarding
4753 .has_live_module_connection(&target_module_id)
4754 .map_err(RouterError::Forwarding)?
4755 {
4756 return Ok(vec![self.route_open_refusal_frame(
4757 ctx,
4758 &caller,
4759 &frame,
4760 &target_module_id,
4761 "no_forwarding_connection",
4762 "target_unavailable",
4763 format!("module_id '{target_module_id}' has no live forwarding connection"),
4764 )?]);
4765 }
4766
4767 if let Some(error) =
4768 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4769 {
4770 self.observe_route_open_refusal(
4771 ctx,
4772 &caller,
4773 &target_module_id,
4774 "op_not_allowed",
4775 "op_not_allowed",
4776 );
4777 return Ok(vec![error]);
4778 }
4779
4780 let principal =
4781 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
4782 Ok(principal) => principal,
4783 Err(error) => {
4784 self.observe_route_open_refusal(
4785 ctx,
4786 &caller,
4787 &target_module_id,
4788 "bad_consumer_identity",
4789 "bad_consumer_identity",
4790 );
4791 return Ok(vec![error]);
4792 }
4793 };
4794 timings.principal = principal_label(&principal);
4795
4796 if let Principal::Reserved {
4800 module_id: opening_module_id,
4801 } = &principal
4802 {
4803 if let Some(opening_registration) = self
4804 .registry
4805 .get_module(opening_module_id)
4806 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4807 {
4808 if let Some(capability) =
4809 denied_capability(&opening_registration.manifest, ®istration.manifest)
4810 {
4811 warn!(
4812 opening_module_id,
4813 target_module_id,
4814 capability,
4815 "refusing route.open because an attested capability deny edge matches"
4816 );
4817 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4818 &frame,
4819 &target_module_id,
4820 "capability_deny_edge",
4821 "capability_forbidden",
4822 format!(
4823 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4824 ),
4825 )?]);
4826 }
4827 }
4828 }
4829
4830 if admission_facts.is_some() {
4831 let carrier_matches = matches!(
4832 &principal,
4833 Principal::Reserved { module_id }
4834 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4835 );
4836 if !carrier_matches {
4837 return Ok(vec![self.route_open_refusal_frame(
4838 ctx,
4839 &caller,
4840 &frame,
4841 &target_module_id,
4842 "admission_facts_carrier_not_permitted",
4843 "admission_facts_not_permitted",
4844 "admission facts may only be carried by the configured reserved module",
4845 )?]);
4846 }
4847
4848 let target_allowed = self
4849 .admission_facts_targets
4850 .as_ref()
4851 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4852 if !target_allowed {
4853 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4854 &frame,
4855 &target_module_id,
4856 "admission_facts_target_not_listed",
4857 "admission_facts_target_not_allowed",
4858 format!(
4859 "admission facts are not permitted for target module_id '{target_module_id}'"
4860 ),
4861 )?]);
4862 }
4863
4864 }
4868
4869 let (bound_scope, mut scope_stamp) = match &scope {
4875 None => (None, None),
4876 Some(selector) => {
4877 let owner_configured = match &selector.owner {
4878 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4884 _ => false,
4885 };
4886 let mut scopes = self
4887 .scopes
4888 .write()
4889 .unwrap_or_else(|poisoned| poisoned.into_inner());
4890 let admitted = scopes
4891 .admit(&principal, &target_module_id, selector, owner_configured)
4892 .and_then(|admission| {
4893 crate::scopes::check_target_flow_support(
4894 &admission.stamp,
4895 &target_module_id,
4896 registration.manifest.capabilities.as_ref(),
4897 )?;
4898 crate::scopes::check_target_agent_run_support(
4899 &admission.stamp,
4900 &target_module_id,
4901 registration.manifest.capabilities.as_ref(),
4902 )?;
4903 let bound = scopes.reserve_bind(
4904 &admission,
4905 &principal,
4906 ctx.connection_id,
4907 selector.hold_presence,
4908 );
4909 Ok((admission, bound))
4910 });
4911 match admitted {
4912 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4913 Err(refusal) => {
4914 return Ok(vec![self.route_open_refusal_frame(
4915 ctx,
4916 &caller,
4917 &frame,
4918 &target_module_id,
4919 refusal.code,
4920 refusal.code,
4921 refusal.message,
4922 )?]);
4923 }
4924 }
4925 }
4926 };
4927 if let Some(stamp) = &scope_stamp {
4930 caller.scope = route_open_scope_label(stamp);
4931 }
4932
4933 timings.enter(RouteOpenPhase::ProjectRoot);
4955 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4956 Ok(project_root) => project_root,
4957 Err(err) => {
4958 return Ok(vec![control_error_frame(
4959 &frame,
4960 "invalid_project_root",
4961 err.to_string(),
4962 )?])
4963 }
4964 };
4965 timings.enter(RouteOpenPhase::Admission);
4966 identity.project_root = project_root.as_path().to_path_buf();
4967
4968 let _concurrency_guard = match self
4979 .route_bind_concurrency
4980 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4981 {
4982 Ok(guard) => guard,
4983 Err(in_flight) => {
4984 return Ok(vec![self.route_open_target_capacity_refusal(
4985 ctx,
4986 &caller,
4987 &frame,
4988 &target_module_id,
4989 in_flight,
4990 )?]);
4991 }
4992 };
4993
4994 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4997 RouteBindAdmission::Admitted { guard, probe } => {
4998 if probe {
4999 info!(
5000 module_id = %target_module_id,
5001 connection_id = ctx.connection_id.get(),
5002 "route.bind breaker half-open: admitting one probe"
5003 );
5004 }
5005 guard
5006 }
5007 RouteBindAdmission::Refused {
5008 consecutive_timeouts,
5009 retry_in,
5010 probe_in_flight,
5011 } => {
5012 return Ok(vec![self.route_open_breaker_refusal_frame(
5013 ctx,
5014 &caller,
5015 &frame,
5016 &target_module_id,
5017 consecutive_timeouts,
5018 retry_in,
5019 probe_in_flight,
5020 )?]);
5021 }
5022 };
5023
5024 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5029 let relay_deadline = Instant::now() + route_bind_relay_timeout;
5030 timings.enter(RouteOpenPhase::EgressReservation);
5031 let pending = match self
5032 .forwarding
5033 .begin_route_bind_relay_for(
5034 ctx.connection_id,
5035 ctx.egress.clone(),
5036 response_version(&frame),
5037 frame.header.corr,
5038 &target_module_id,
5039 principal.clone(),
5040 bound_scope,
5041 Some(project_root),
5042 relay_deadline,
5043 )
5044 .await
5045 {
5046 Ok(pending) => pending,
5047 Err(err) => {
5048 return Ok(vec![self.route_open_refusal_frame(
5049 ctx,
5050 &caller,
5051 &frame,
5052 &target_module_id,
5053 "relay_reservation_failed",
5054 forwarding_error_code(&err),
5055 err.to_string(),
5056 )?])
5057 }
5058 };
5059 timings.enter(RouteOpenPhase::Admission);
5060 let crate::forwarding::PendingRouteBindRelay {
5061 endpoint,
5062 module_sink,
5063 negotiated_ver,
5064 client_channel,
5065 client_epoch,
5066 module_channel,
5067 module_epoch,
5068 corr: relay_corr,
5069 receiver,
5070 } = pending;
5071 let mut reservation =
5072 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5073
5074 let relay_registration = self
5079 .registry
5080 .get_module_by_connection(endpoint.connection_id)
5081 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5082 let relay_capabilities = relay_registration
5083 .as_ref()
5084 .and_then(|registration| registration.manifest.capabilities.as_ref());
5085 if let Some(selector) = &scope {
5086 if let Err(refusal) = self
5089 .scopes
5090 .read()
5091 .unwrap_or_else(|poisoned| poisoned.into_inner())
5092 .check_bind_proof(&principal, selector)
5093 {
5094 reservation.release_and_disarm();
5095 return Ok(vec![self.route_open_refusal_frame(
5096 ctx,
5097 &caller,
5098 &frame,
5099 &target_module_id,
5100 refusal.code,
5101 refusal.code,
5102 refusal.message,
5103 )?]);
5104 }
5105 }
5106 if let Some(stamp) = scope_stamp
5107 .as_ref()
5108 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5109 {
5110 if let Err(refusal) = crate::scopes::check_target_flow_support(
5111 stamp,
5112 &target_module_id,
5113 relay_capabilities,
5114 )
5115 .and_then(|()| {
5116 crate::scopes::check_target_agent_run_support(
5117 stamp,
5118 &target_module_id,
5119 relay_capabilities,
5120 )
5121 }) {
5122 reservation.release_and_disarm();
5123 return Ok(vec![self.route_open_refusal_frame(
5124 ctx,
5125 &caller,
5126 &frame,
5127 &target_module_id,
5128 refusal.code,
5129 refusal.code,
5130 refusal.message,
5131 )?]);
5132 }
5133 }
5134
5135 if !relay_capabilities.is_some_and(|caps| {
5138 caps.provides
5139 .iter()
5140 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5141 }) {
5142 if let Some(stamp) = scope_stamp.as_mut() {
5143 stamp.workdir = None;
5144 stamp.project_id = None;
5145 }
5146 }
5147 let worker_run = scope_stamp
5152 .as_ref()
5153 .is_some_and(crate::scopes::is_worker_run)
5154 && relay_capabilities.is_some_and(|caps| {
5155 caps.provides
5156 .iter()
5157 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5158 });
5159 if !worker_run
5160 && !relay_capabilities.is_some_and(|caps| {
5161 caps.provides
5162 .iter()
5163 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5164 })
5165 {
5166 if let Some(stamp) = scope_stamp.as_mut() {
5167 stamp.attributes.on_behalf_of_agent = None;
5168 }
5169 }
5170 if !relay_capabilities.is_some_and(|caps| {
5171 caps.provides
5172 .iter()
5173 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5174 }) {
5175 if let Some(stamp) = scope_stamp.as_mut() {
5176 stamp.attributes.flow_version = None;
5177 stamp.attributes.flow_code_hash = None;
5178 stamp.attributes.flow_install_id = None;
5179 }
5180 }
5181 let call_mode = if worker_run
5182 || relay_capabilities
5183 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5184 {
5185 Some(derive_call_mode(scope_stamp.as_ref()))
5186 } else {
5187 if let Some(stamp) = scope_stamp.as_mut() {
5188 stamp.expires_at_ms = None;
5189 }
5190 None
5191 };
5192
5193 debug!(
5194 connection_id = ctx.connection_id.get(),
5195 client_channel,
5196 client_epoch,
5197 module_channel,
5198 module_epoch,
5199 "reserved route handle pair"
5200 );
5201 let route_principal_label = principal_label(&principal);
5204 identity.build_class = None;
5208 let relay = ModuleControlRequest::RouteBind {
5209 route_channel: module_channel,
5210 epoch: module_epoch,
5211 target,
5212 identity,
5213 principal: Some(principal),
5214 consumer_capabilities,
5215 role_versions,
5216 admission_facts,
5217 scope: scope_stamp,
5218 call_mode,
5219 };
5220 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
5221 RouterError::backend(
5222 0,
5223 frame.header.corr,
5224 format!("failed to encode route.bind request: {err}"),
5225 )
5226 })?;
5227 let relay_frame = Frame::build_with_version(
5228 negotiated_ver,
5229 FrameType::Request,
5230 control_flags(),
5231 0,
5232 0,
5233 relay_corr,
5234 relay_body,
5235 )
5236 .map_err(RouterError::FrameBuild)?;
5237
5238 timings.enter(RouteOpenPhase::BindSend);
5239 let sent = module_sink.send(relay_frame).await;
5240 timings.enter(RouteOpenPhase::Admission);
5241 if let Err(err) = sent {
5242 reservation.release_and_disarm();
5243 return Ok(vec![self.route_open_refusal_frame(
5244 ctx,
5245 &caller,
5246 &frame,
5247 &target_module_id,
5248 "relay_send_failed",
5249 "target_unavailable",
5250 err.to_string(),
5251 )?]);
5252 }
5253
5254 #[cfg(test)]
5255 if let Some(hold) = &self.route_bind_relay_sent_hold {
5256 hold.wait().await;
5260 hold.wait().await;
5261 }
5262
5263 if self
5268 .forwarding
5269 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5270 .map_err(RouterError::Forwarding)?
5271 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5272 {
5273 self.send_abandoned_route_bind_goodbye(
5274 &module_sink,
5275 negotiated_ver,
5276 module_channel,
5277 module_epoch,
5278 );
5279 }
5280
5281 timings.enter(RouteOpenPhase::BindAckWait);
5282 let outcome = timeout_at(relay_deadline, receiver).await;
5283 timings.enter(RouteOpenPhase::Admission);
5284 match outcome {
5285 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5286 reservation.disarm();
5287 if breaker.record_accepted() {
5288 info!(
5289 module_id = %target_module_id,
5290 "route.bind breaker closed: the probe was accepted"
5291 );
5292 }
5293 self.observe_route_open_accept(
5294 ctx,
5295 &target_module_id,
5296 &route_principal_label,
5297 &caller.scope,
5298 );
5299 Ok(Vec::new())
5300 }
5301 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5302 reservation.release_and_disarm();
5303 breaker.record_inconclusive();
5307 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5308 }
5309 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5310 reservation.release_and_disarm();
5311 breaker.record_inconclusive();
5312 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5313 }
5314 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5315 reservation.release_and_disarm();
5316 breaker.record_inconclusive();
5317 tracing::warn!(
5324 module_id = %target_module_id,
5325 "route.bind relay abandoned: {message}"
5326 );
5327 Ok(vec![self.route_open_refusal_frame(
5328 ctx,
5329 &caller,
5330 &frame,
5331 &target_module_id,
5332 "relay_abandoned",
5333 "target_unavailable",
5334 message,
5335 )?])
5336 }
5337 Ok(Err(_)) => {
5338 reservation.release_and_disarm();
5339 breaker.record_inconclusive();
5340 Ok(vec![self.route_open_refusal_frame(
5341 ctx,
5342 &caller,
5343 &frame,
5344 &target_module_id,
5345 "relay_waiter_canceled",
5346 "target_unavailable",
5347 "route.bind relay waiter was canceled before the module responded",
5348 )?])
5349 }
5350 Err(_) => {
5351 reservation.release_and_disarm();
5352 if let Some(opened) = breaker.record_timeout(
5356 self.route_bind_breaker_threshold,
5357 self.route_bind_breaker_cooldown,
5358 ) {
5359 warn!(
5360 module_id = %target_module_id,
5361 consecutive_timeouts = opened.consecutive_timeouts,
5362 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5363 reopened_after_probe = opened.reopened_after_probe,
5364 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5365 );
5366 }
5367 tracing::warn!(
5375 module_id = %target_module_id,
5376 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5377 "route.bind relay timed out: module did not ack within budget"
5378 );
5379 Ok(vec![self.route_open_refusal_frame(
5380 ctx,
5381 &caller,
5382 &frame,
5383 &target_module_id,
5384 "relay_timed_out",
5385 "module_timeout",
5386 format!(
5387 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5388 route_bind_relay_timeout
5389 ),
5390 )?])
5391 }
5392 }
5393 }
5394
5395 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5396 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5397 snapshot: self.supervisor.spawn_snapshot(),
5398 };
5399 Ok(vec![control_response_body_frame(
5400 &frame,
5401 &response,
5402 "ClientControlResponse::SupervisorSpawnSnapshot",
5403 )?])
5404 }
5405
5406 fn handle_supervisor_spawn_subscribe(
5407 &self,
5408 ctx: &RouteCtx,
5409 frame: Frame,
5410 since: Option<SpawnCursor>,
5411 ) -> Result<Vec<Frame>, RouterError> {
5412 match self.supervisor.subscribe_spawns(
5413 ctx.connection_id,
5414 frame.header.corr,
5415 response_version(&frame),
5416 since,
5417 ctx.egress.clone(),
5418 ) {
5419 Ok(()) => Ok(Vec::new()),
5420 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5421 Ok(vec![control_error_body_frame(
5422 &frame,
5423 ErrorBody {
5424 code: "spawn_cursor_incarnation_mismatch".to_string(),
5425 message: "spawn cursor belongs to a different daemon incarnation"
5426 .to_string(),
5427 detail: Some(serde_json::json!({
5428 "current_daemon_incarnation": current
5429 })),
5430 },
5431 )?])
5432 }
5433 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5434 &frame,
5435 ErrorBody {
5436 code: "spawn_cursor_too_old".to_string(),
5437 message: "spawn cursor predates the retained event ring".to_string(),
5438 detail: Some(serde_json::json!({
5439 "oldest_retained_cursor": oldest
5440 })),
5441 },
5442 )?]),
5443 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5444 0,
5445 frame.header.corr,
5446 format!("failed to open supervisor spawn subscription: {error}"),
5447 )),
5448 }
5449 }
5450
5451 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5452 let generation = self
5453 .registry
5454 .generation()
5455 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5456 let mut modules = Vec::new();
5457 for module in self.supervisor.list() {
5458 let status = module.status_for_control("list").map_err(|err| {
5459 RouterError::backend(
5460 0,
5461 frame.header.corr,
5462 format!("failed to read supervisor status: {err}"),
5463 )
5464 })?;
5465 let (configured, _) = module.configuration().map_err(|err| {
5466 RouterError::backend(
5467 0,
5468 frame.header.corr,
5469 format!("failed to read module configuration: {err}"),
5470 )
5471 })?;
5472 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5473 RouterError::backend(
5474 0,
5475 frame.header.corr,
5476 format!("failed to read retry time: {err}"),
5477 )
5478 })?;
5479 let image = module.running_image_agreement().await;
5481 let resources = Some(module.child_resource_usage());
5484 let mut verdict =
5485 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5486 verdict.spec = Some(launch_spec_agreement(
5487 &configured,
5488 status.process_alive,
5489 status.launch_spec_digest.as_ref(),
5490 ));
5491 let pending_reload = Some(verdict);
5492 let configured_enabled = self
5493 .capability_evaluator
5494 .configured_enabled(&status.module_id);
5495 modules.push(
5498 SupervisorEntry::new(
5499 status.module_id,
5500 status.state.to_string(),
5501 status.enabled,
5502 status.live,
5503 status.health.status,
5504 )
5505 .with_configured_enabled(configured_enabled)
5506 .with_next_retry_at_ms(next_retry_at_ms)
5507 .with_launch_nonce_env(Some(!cfg!(unix)))
5508 .with_launch_nonce_source(status.launch_nonce_source)
5509 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5510 .with_protocol(status.protocol)
5511 .with_pending_reload(pending_reload)
5512 .with_last_probe_ms(status.health.last_probe_ms)
5513 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5514 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5515 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5516 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5517 .with_restart_count(Some(status.restart_count))
5518 .with_max_restarts(Some(status.max_restarts))
5519 .with_lifetime_restarts(Some(status.lifetime_restarts))
5520 .with_spawn_generation(Some(status.spawn_generation))
5521 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5522 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5523 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5524 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5525 .with_resources(resources),
5526 );
5527 }
5528 let response = ClientControlResponse::SupervisorList {
5529 generation,
5530 modules,
5531 };
5532 Ok(vec![control_response_body_frame(
5533 &frame,
5534 &response,
5535 "ClientControlResponse::SupervisorList",
5536 )?])
5537 }
5538
5539 fn handle_supervisor_stderr_tail(
5540 &self,
5541 frame: Frame,
5542 module_id: String,
5543 max_lines: Option<u32>,
5544 max_bytes: Option<u32>,
5545 ) -> Result<Vec<Frame>, RouterError> {
5546 let Some(module) = self.supervisor.get(&module_id) else {
5547 return Ok(vec![control_error_frame(
5548 &frame,
5549 "unknown_module",
5550 format!("module_id '{module_id}' is not supervised"),
5551 )?]);
5552 };
5553
5554 let snapshot = module.stderr_tail(
5555 max_lines.map(|value| value as usize),
5556 max_bytes.map(|value| value as usize),
5557 );
5558
5559 let response = ClientControlResponse::SupervisorStderrTail {
5560 module_id,
5561 tail: StderrTail {
5562 capture: match snapshot.capture {
5563 CaptureState::Captured => StderrCaptureState::Captured,
5564 CaptureState::Incomplete { reason } => {
5565 StderrCaptureState::Incomplete { reason }
5566 }
5567 CaptureState::NotCaptured { reason } => {
5568 StderrCaptureState::NotCaptured { reason }
5569 }
5570 },
5571 entries: snapshot
5572 .entries
5573 .into_iter()
5574 .map(|entry| match entry {
5575 TailEntry::Line {
5576 text,
5577 truncated,
5578 at_ms,
5579 } => StderrTailEntry::Line {
5580 text,
5581 truncated,
5582 at_ms,
5583 },
5584 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5585 })
5586 .collect(),
5587 dropped_lines: snapshot.dropped_lines,
5588 },
5589 };
5590 Ok(vec![control_response_body_frame(
5591 &frame,
5592 &response,
5593 "ClientControlResponse::SupervisorStderrTail",
5594 )?])
5595 }
5596
5597 async fn handle_supervisor_terminals(
5598 &self,
5599 frame: Frame,
5600 module_id: String,
5601 ) -> Result<Vec<Frame>, RouterError> {
5602 let Some(module) = self.supervisor.get(&module_id) else {
5603 return Ok(vec![control_error_frame(
5604 &frame,
5605 "unknown_module",
5606 format!("module_id '{module_id}' is not supervised"),
5607 )?]);
5608 };
5609
5610 let terminals = module
5613 .read_durable_terminal_history()
5614 .await
5615 .map_err(|error| {
5616 RouterError::backend(
5617 0,
5618 frame.header.corr,
5619 format!("failed to read terminal history: {error}"),
5620 )
5621 })?;
5622 let response = ClientControlResponse::SupervisorTerminals {
5623 module_id,
5624 terminals,
5625 };
5626 Ok(vec![control_response_body_frame(
5627 &frame,
5628 &response,
5629 "ClientControlResponse::SupervisorTerminals",
5630 )?])
5631 }
5632
5633 fn handle_supervisor_routes(
5634 &self,
5635 frame: Frame,
5636 module_id: Option<String>,
5637 ) -> Result<Vec<Frame>, RouterError> {
5638 let modules = self
5639 .forwarding
5640 .route_census(module_id.as_deref())
5641 .map_err(RouterError::Forwarding)?
5642 .into_iter()
5643 .map(|(module_id, routes)| SupervisorRouteModule {
5644 module_id,
5645 routes: routes
5646 .into_iter()
5647 .map(|route| SupervisorRoute {
5648 consumer: match route.principal {
5649 Principal::Reserved { module_id } => {
5650 SupervisorRouteConsumer::Reserved { module_id }
5651 }
5652 Principal::Direct | Principal::Unverified => {
5653 SupervisorRouteConsumer::Direct {
5654 connection_id: route.goodbye_target.connection_id.get(),
5655 }
5656 }
5657 },
5658 age_ms: Instant::now()
5659 .saturating_duration_since(route.bound_at)
5660 .as_millis()
5661 .try_into()
5662 .unwrap_or(u64::MAX),
5663 draining: route.draining,
5664 drain_reason: route.drain_reason,
5665 })
5666 .collect(),
5667 })
5668 .collect();
5669 let response = ClientControlResponse::SupervisorRoutes { modules };
5670 Ok(vec![control_response_body_frame(
5671 &frame,
5672 &response,
5673 "ClientControlResponse::SupervisorRoutes",
5674 )?])
5675 }
5676
5677 async fn handle_supervisor_provenance(
5678 &self,
5679 frame: Frame,
5680 module_id: Option<String>,
5681 ) -> Result<Vec<Frame>, RouterError> {
5682 let mut selected = if let Some(module_id) = module_id {
5683 let Some(module) = self.supervisor.get(&module_id) else {
5684 return Ok(vec![control_error_frame(
5685 &frame,
5686 "unknown_module",
5687 format!("module_id '{module_id}' is not supervised"),
5688 )?]);
5689 };
5690 vec![module]
5691 } else {
5692 self.supervisor.list()
5693 };
5694
5695 let mut modules = Vec::with_capacity(selected.len());
5696 for module in selected.drain(..) {
5697 let (status, observed_image) = module
5698 .status_and_running_image_agreement()
5699 .await
5700 .map_err(|err| {
5701 RouterError::backend(
5702 0,
5703 frame.header.corr,
5704 format!("failed to read supervisor status: {err}"),
5705 )
5706 })?;
5707 let module_declared = self
5708 .registry
5709 .get_module(&status.module_id)
5710 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5711 .and_then(|registration| registration.manifest.provenance)
5712 .map(|build| ModuleDeclaredProvenance::Reported { build })
5713 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5714 #[cfg(test)]
5715 let running_image = match &self.provenance_probe_override {
5716 Some(result) => result.clone(),
5717 None => observed_image,
5718 };
5719 #[cfg(not(test))]
5720 let running_image = observed_image;
5721 modules.push(SupervisorModuleProvenance {
5722 module_id: status.module_id,
5723 module_declared,
5724 daemon_observed: SupervisorObservedProcess {
5725 launch_nonce_source: status.launch_nonce_source,
5726 pid: status.pid,
5727 spawned_at_ms: status.spawned_at_ms,
5728 spawned_from: status.spawned_from,
5729 running_image,
5730 },
5731 });
5732 }
5733 let daemon = SupervisorDaemonProvenance {
5734 daemon_build: self.daemon_provenance.build.clone(),
5735 daemon_observed: DaemonObservedProcess {
5736 pid: self.daemon_provenance.pid,
5737 started_at_ms: self
5738 .daemon_provenance
5739 .start_clock
5740 .map(|clock| clock.started_at_ms())
5741 .or(self.daemon_provenance.started_at_ms),
5742 running_image: self
5743 .daemon_provenance
5744 .probe
5745 .observe(
5746 self.daemon_provenance.pid,
5747 self.daemon_provenance.executable_path.as_deref(),
5748 self.daemon_provenance.executable_identity.clone(),
5749 self.daemon_provenance.process_start_time,
5750 )
5751 .await,
5752 },
5753 };
5754 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5755 Ok(vec![control_response_body_frame(
5756 &frame,
5757 &response,
5758 "ClientControlResponse::SupervisorProvenance",
5759 )?])
5760 }
5761
5762 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5763 self.refresh_capability_requirements();
5764 let generation = self
5765 .registry
5766 .generation()
5767 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5768 let modules = self
5769 .supervisor
5770 .list()
5771 .into_iter()
5772 .map(|module| {
5773 let status = module.status_for_control("health").map_err(|err| {
5774 RouterError::backend(
5775 0,
5776 frame.header.corr,
5777 format!("failed to read supervisor health: {err}"),
5778 )
5779 })?;
5780 let module_id = status.module_id;
5781 let capability_detail = self
5782 .capability_evaluator
5783 .required_problem_detail(&module_id);
5784 Ok(SupervisorHealthEntry {
5785 module_id,
5786 status: status.health.status,
5787 detail: append_capability_problem_detail(
5788 status.health.detail,
5789 capability_detail,
5790 ),
5791 metrics: status.health.metrics,
5792 consecutive_failures: status.health.consecutive_failures,
5793 late_answer_count: status.health.late_answer_count,
5794 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5795 last_action: status.health.last_action,
5796 last_action_ms: status.health.last_action_ms,
5797 last_probe_ms: status.health.last_probe_ms,
5798 })
5799 })
5800 .collect::<Result<Vec<_>, RouterError>>()?;
5801 let response = ClientControlResponse::SupervisorHealth {
5802 generation,
5803 modules,
5804 };
5805 Ok(vec![control_response_body_frame(
5806 &frame,
5807 &response,
5808 "ClientControlResponse::SupervisorHealth",
5809 )?])
5810 }
5811
5812 async fn handle_supervisor_restart(
5813 &self,
5814 frame: Frame,
5815 module_id: String,
5816 drain_timeout_ms: Option<u64>,
5817 requester: Principal,
5818 ) -> Result<Vec<Frame>, RouterError> {
5819 let operation_lock = self.supervisor.operation_lock();
5820 let _operation_guard = operation_lock.lock().await;
5821 let Some(module) = self.supervisor.get(&module_id) else {
5822 return Ok(vec![control_error_frame(
5823 &frame,
5824 "unknown_module",
5825 format!("module_id '{module_id}' is not supervised"),
5826 )?]);
5827 };
5828
5829 self.route_outages.mark_operator_action(&module_id);
5830 if let Err(err) = module
5831 .restart_requested(drain_timeout_ms, Some(requester))
5832 .await
5833 {
5834 self.route_outages
5835 .operator_action_ended_unrefused(&module_id);
5836 let (code, message) = match err {
5837 crate::supervise::SuperviseError::Disabled { .. } => {
5838 ("module_disabled", err.to_string())
5839 }
5840 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5841 ("swap_in_progress", err.to_string())
5842 }
5843 _ => (
5844 "target_unavailable",
5845 format!("failed to restart module_id '{module_id}': {err}"),
5846 ),
5847 };
5848 return Ok(vec![control_error_frame(&frame, code, message)?]);
5849 }
5850
5851 let response = ClientControlResponse::SupervisorAck {
5852 module_id,
5853 applied: true,
5854 };
5855 Ok(vec![control_response_body_frame(
5856 &frame,
5857 &response,
5858 "ClientControlResponse::SupervisorAck",
5859 )?])
5860 }
5861
5862 async fn handle_supervisor_swap(
5866 &self,
5867 frame: Frame,
5868 module_id: String,
5869 ready_timeout_ms: Option<u64>,
5870 requester: Principal,
5871 ) -> Result<Vec<Frame>, RouterError> {
5872 let module = {
5879 let operation_lock = self.supervisor.operation_lock();
5880 let _operation_guard = operation_lock.lock().await;
5881 self.supervisor.get(&module_id)
5882 };
5883 let Some(module) = module else {
5884 return Ok(vec![control_error_frame(
5885 &frame,
5886 "unknown_module",
5887 format!("module_id '{module_id}' is not supervised"),
5888 )?]);
5889 };
5890
5891 self.route_outages.mark_operator_action(&module_id);
5892 if let Err(err) = module
5893 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5894 .await
5895 {
5896 self.route_outages
5897 .operator_action_ended_unrefused(&module_id);
5898 use crate::supervise::SuperviseError;
5899 let message = err.to_string();
5900 let error = match err {
5901 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5902 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5903 code: "swap_refused".to_string(),
5904 message,
5905 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5906 },
5907 SuperviseError::SwapFailed {
5908 arm,
5909 candidate_exit,
5910 ..
5911 } => ErrorBody {
5912 code: "swap_failed".to_string(),
5913 message,
5914 detail: Some(serde_json::json!({
5915 "arm": arm.as_str(),
5916 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5917 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5918 })),
5919 },
5920 _ => ErrorBody::new(
5921 "target_unavailable",
5922 format!("failed to swap module_id '{module_id}': {message}"),
5923 ),
5924 };
5925 return Ok(vec![control_error_body_frame(&frame, error)?]);
5926 }
5927 self.route_outages
5931 .operator_action_ended_unrefused(&module_id);
5932
5933 let response = ClientControlResponse::SupervisorAck {
5934 module_id,
5935 applied: true,
5936 };
5937 Ok(vec![control_response_body_frame(
5938 &frame,
5939 &response,
5940 "ClientControlResponse::SupervisorAck",
5941 )?])
5942 }
5943
5944 async fn handle_supervisor_reload(
5945 &self,
5946 frame: Frame,
5947 module_id: String,
5948 requester: Principal,
5949 ) -> Result<Vec<Frame>, RouterError> {
5950 let operation_lock = self.supervisor.operation_lock();
5951 let _operation_guard = operation_lock.lock().await;
5952 let Some(module) = self.supervisor.get(&module_id) else {
5953 return Ok(vec![control_error_frame(
5954 &frame,
5955 "unknown_module",
5956 format!("module_id '{module_id}' is not supervised"),
5957 )?]);
5958 };
5959
5960 self.route_outages.mark_operator_action(&module_id);
5961 if let Err(err) = module.reload_requested(Some(requester)).await {
5962 self.route_outages
5963 .operator_action_ended_unrefused(&module_id);
5964 let (code, message) = match err {
5965 crate::supervise::SuperviseError::Disabled { .. } => {
5966 ("module_disabled", err.to_string())
5967 }
5968 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5969 ("swap_in_progress", err.to_string())
5970 }
5971 _ => (
5972 "reload_failed",
5973 format!("failed to reload module_id '{module_id}': {err}"),
5974 ),
5975 };
5976 return Ok(vec![control_error_frame(&frame, code, message)?]);
5977 }
5978
5979 let response = ClientControlResponse::SupervisorAck {
5980 module_id,
5981 applied: true,
5982 };
5983 Ok(vec![control_response_body_frame(
5984 &frame,
5985 &response,
5986 "ClientControlResponse::SupervisorAck",
5987 )?])
5988 }
5989
5990 async fn handle_supervisor_rescan(
5994 &self,
5995 frame: Frame,
5996 mode: RescanMode,
5997 requester: Principal,
5998 ) -> Result<Vec<Frame>, RouterError> {
5999 let preview = mode.previews();
6000 let Some(context) = self.rescan.clone() else {
6001 return Ok(vec![control_error_frame(
6002 &frame,
6003 "rescan_unavailable",
6004 "the daemon was not started with a reloadable config path".to_string(),
6005 )?]);
6006 };
6007
6008 let operation_lock = self.supervisor.operation_lock();
6009 let _operation_guard = operation_lock.lock().await;
6010 let loaded = match self.load_rescan_config(&mode, &context) {
6014 Ok(config) => config,
6015 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6016 };
6017 let Some((config, activity_config)) = loaded else {
6025 return Ok(vec![control_error_frame(
6026 &frame,
6027 "invalid_daemon_config",
6028 format!(
6029 "daemon config not found at {}; refusing to rescan (an absent config would \
6030 retire every supervised module)",
6031 context.config_path.display()
6032 ),
6033 )?]);
6034 };
6035 let (
6036 configured_port,
6037 storage_config,
6038 admission_facts_carrier_module_id,
6039 admission_facts_targets,
6040 scope_authority_owners,
6041 privileged_operator_prompt,
6042 modules,
6043 reserved_capabilities,
6044 ) = (
6045 config.port,
6046 config.storage,
6047 config.admission_facts_carrier_module_id,
6048 config.admission_facts_targets,
6049 config.scope_authority_owners,
6050 config.privileged_operator_prompt,
6051 config.modules,
6052 config.reserved_capabilities,
6053 );
6054
6055 let mut restart_required = Vec::new();
6064 for section in RestartRequiredSection::ALL {
6065 let changed = match section {
6066 RestartRequiredSection::Port => configured_port != context.configured_port,
6067 RestartRequiredSection::Storage => storage_config != context.storage_config,
6068 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6069 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6070 }
6071 RestartRequiredSection::AdmissionFactsTargets => {
6072 admission_facts_targets != context.admission_facts_targets
6073 }
6074 RestartRequiredSection::ScopeAuthorityOwners => {
6075 scope_authority_owners != context.scope_authority_owners
6076 }
6077 RestartRequiredSection::PrivilegedOperatorPrompt => {
6078 privileged_operator_prompt != context.privileged_operator_prompt
6079 }
6080 };
6081 if changed {
6082 restart_required.push(section.label().to_string());
6083 }
6084 }
6085 if !restart_required.is_empty() {
6086 warn!(
6087 config_path = %context.config_path.display(),
6088 sections = %restart_required.join(", "),
6089 "daemon config changed outside the modules section; restart the daemon to apply those changes"
6090 );
6091 }
6092
6093 for configured in &modules {
6094 if let Err(err) = validate_spec(&configured.module_spec()) {
6095 return Ok(vec![control_error_frame(
6096 &frame,
6097 "invalid_daemon_config",
6098 format!("supervisor rescan rejected daemon config: {err}"),
6099 )?]);
6100 }
6101 }
6102
6103 let configured_capabilities = modules
6104 .iter()
6105 .map(|module| (module.module_id.clone(), module.enabled))
6106 .collect::<Vec<_>>();
6107 let preview_capability_warnings = if preview {
6108 let (_, registrations) = self.runtime_capability_snapshot()?;
6109 let current_modules = self
6110 .supervisor
6111 .list()
6112 .into_iter()
6113 .map(|module| module.module_id().to_string())
6114 .collect::<BTreeSet<_>>();
6115 let resulting_modules = configured_capabilities.clone();
6116 let removed = current_modules
6117 .into_iter()
6118 .filter(|module_id| {
6119 !resulting_modules
6120 .iter()
6121 .any(|(configured_id, _)| configured_id == module_id)
6122 })
6123 .collect::<Vec<_>>();
6124 let disabled = match &mode {
6127 RescanMode::Candidate(_) => resulting_modules
6128 .iter()
6129 .filter(|(_, enabled)| !enabled)
6130 .map(|(module_id, _)| module_id.clone())
6131 .collect::<Vec<_>>(),
6132 _ => Vec::new(),
6133 };
6134 self.capability_evaluator.preview_warnings(
6135 resulting_modules,
6136 &removed,
6137 &disabled,
6138 ®istrations,
6139 )
6140 } else {
6141 Vec::new()
6142 };
6143 let reconciled = match &mode {
6144 RescanMode::Candidate(candidate) => self
6145 .preview_candidate(&context, modules, candidate)
6146 .map(|result| (result, Vec::new())),
6147 _ => {
6148 self.reconcile_supervised_modules(
6149 &context.supervisor,
6150 modules,
6151 preview,
6152 requester.clone(),
6153 )
6154 .await
6155 }
6156 };
6157 let (result, changed) = match reconciled {
6158 Ok(result) => result,
6159 Err(message) => {
6160 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6161 }
6162 };
6163 if !preview {
6164 if let Some(ledger) = &self.activity {
6165 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6166 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6167 }
6168 if let Err(code) = self.configure_activity(activity_config).await {
6169 warn!(
6170 code,
6171 "activity retention could not be applied during rescan"
6172 );
6173 }
6174 self.capability_evaluator
6175 .configure(configured_capabilities, reserved_capabilities);
6176 self.capability_evaluator.wake_deadline_loop();
6177 self.refresh_capability_requirements();
6178 }
6179 let mut result = result;
6180 result.restart_required = restart_required;
6181 result.capability_warnings = preview_capability_warnings;
6182 let response = ClientControlResponse::SupervisorRescan { result };
6183 Ok(vec![control_response_body_frame(
6184 &frame,
6185 &response,
6186 "ClientControlResponse::SupervisorRescan",
6187 )?])
6188 }
6189
6190 async fn handle_supervisor_release_reserved(
6191 &self,
6192 frame: Frame,
6193 module_id: String,
6194 ) -> Result<Vec<Frame>, RouterError> {
6195 let Some(context) = self.rescan.clone() else {
6196 return Ok(vec![control_error_frame(
6197 &frame,
6198 "release_unavailable",
6199 "reserved-id release requires a daemon started with a reloadable config path",
6200 )?]);
6201 };
6202 let operation_lock = self.supervisor.operation_lock();
6203 let _operation_guard = operation_lock.lock().await;
6204 let loaded = match crate::daemon_config::load(&context.config_path) {
6205 Ok(Some(config)) => config,
6206 Ok(None) => {
6207 return Ok(vec![control_error_frame(
6208 &frame,
6209 "invalid_daemon_config",
6210 format!(
6211 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6212 context.config_path.display()
6213 ),
6214 )?])
6215 }
6216 Err(err) => {
6217 return Ok(vec![control_error_frame(
6218 &frame,
6219 "invalid_daemon_config",
6220 format!("unable to verify reserved-id release against daemon config: {err}"),
6221 )?])
6222 }
6223 };
6224 if loaded
6225 .modules
6226 .iter()
6227 .any(|configured| configured.module_id == module_id)
6228 {
6229 return Ok(vec![control_error_frame(
6230 &frame,
6231 "reserved_module_configured",
6232 format!(
6233 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6234 ),
6235 )?]);
6236 }
6237 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6238 return Ok(vec![control_error_frame(
6239 &frame,
6240 "reserved_gate_not_retained",
6241 format!(
6242 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6243 ),
6244 )?]);
6245 }
6246
6247 let response = ClientControlResponse::SupervisorAck {
6248 module_id,
6249 applied: true,
6250 };
6251 Ok(vec![control_response_body_frame(
6252 &frame,
6253 &response,
6254 "ClientControlResponse::SupervisorAck",
6255 )?])
6256 }
6257
6258 fn plan_rescan(
6262 &self,
6263 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6264 ) -> Result<RescanPlan, String> {
6265 let mut current = BTreeMap::new();
6266 for module in self.supervisor.list() {
6267 let (spec, health) = module.configuration().map_err(|err| {
6268 format!(
6269 "failed to read configuration for module_id '{}': {err}",
6270 module.module_id()
6271 )
6272 })?;
6273 let enabled = module
6274 .status()
6275 .map_err(|err| {
6276 format!(
6277 "failed to read status for module_id '{}': {err}",
6278 module.module_id()
6279 )
6280 })?
6281 .enabled;
6282 current.insert(
6283 module.module_id().to_string(),
6284 (module, spec, health, enabled),
6285 );
6286 }
6287 let configured = configured_modules
6288 .into_iter()
6289 .map(|module| (module.module_id.clone(), module))
6290 .collect::<BTreeMap<_, _>>();
6291
6292 let added = configured
6293 .keys()
6294 .filter(|module_id| !current.contains_key(*module_id))
6295 .cloned()
6296 .collect::<Vec<_>>();
6297 let removed = current
6298 .keys()
6299 .filter(|module_id| !configured.contains_key(*module_id))
6300 .cloned()
6301 .collect::<Vec<_>>();
6302 let mut changed_pending_reload = Vec::new();
6303 let mut configuration_changes = BTreeSet::new();
6304 let mut enabled_changes = BTreeSet::new();
6305 let mut unchanged = 0_u32;
6306
6307 for (module_id, configured_module) in &configured {
6308 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6309 else {
6310 continue;
6311 };
6312 let launch_changed = *current_spec != configured_module.module_spec();
6316 let configuration_changed =
6317 launch_changed || *current_health != configured_module.health;
6318 let enabled_changed = *current_enabled != configured_module.enabled;
6319 if configuration_changed {
6320 configuration_changes.insert(module_id.clone());
6321 }
6322 if launch_changed {
6323 changed_pending_reload.push(module_id.clone());
6324 }
6325 if enabled_changed {
6326 enabled_changes.insert(module_id.clone());
6327 }
6328 if !configuration_changed && !enabled_changed {
6329 unchanged = unchanged.saturating_add(1);
6330 }
6331 }
6332 Ok(RescanPlan {
6333 current,
6334 configured,
6335 added,
6336 removed,
6337 changed_pending_reload,
6338 configuration_changes,
6339 enabled_changes,
6340 unchanged,
6341 })
6342 }
6343
6344 async fn reconcile_supervised_modules(
6353 &self,
6354 supervisor: &Supervisor,
6355 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6356 preview: bool,
6357 requester: Principal,
6358 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6359 let plan = self.plan_rescan(configured_modules)?;
6360
6361 if preview {
6367 return Ok((plan.preview_result(), Vec::new()));
6368 }
6369 let changed = plan
6370 .configuration_changes
6371 .union(&plan.enabled_changes)
6372 .cloned()
6373 .collect::<Vec<_>>();
6374 let RescanPlan {
6375 current,
6376 configured,
6377 added,
6378 removed,
6379 changed_pending_reload,
6380 configuration_changes,
6381 enabled_changes,
6382 unchanged,
6383 } = plan;
6384 if !(added.is_empty()
6389 && removed.is_empty()
6390 && configuration_changes.is_empty()
6391 && enabled_changes.is_empty())
6392 {
6393 if let Some(context) = &self.rescan {
6394 context
6395 .generation
6396 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6397 }
6398 }
6399
6400 for module_id in &removed {
6401 let module = ¤t
6402 .get(module_id)
6403 .expect("removed module came from current supervisor state")
6404 .0;
6405 module
6406 .retire_requested(Some(requester.clone()))
6407 .await
6408 .map_err(|err| {
6409 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6410 })?;
6411 self.supervisor.record_rescan_removal(module_id);
6445 self.supervisor.retire(module_id);
6446 self.route_outages.forget(module_id);
6447 }
6448
6449 for module_id in configured.keys() {
6450 let Some((module, _, _, _)) = current.get(module_id) else {
6451 continue;
6452 };
6453 let configured_module = configured
6454 .get(module_id)
6455 .expect("configured module id came from configured map");
6456 if configuration_changes.contains(module_id) {
6457 module
6458 .update_configuration(
6459 configured_module.module_spec(),
6460 configured_module.health.clone(),
6461 configured_module.drain_timeout_ms,
6462 )
6463 .await
6464 .map_err(|err| {
6465 format!(
6466 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6467 )
6468 })?;
6469 }
6470 if enabled_changes.contains(module_id) {
6471 self.route_outages.mark_operator_action(module_id);
6474 module
6475 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6476 .await
6477 .map_err(|err| {
6478 self.route_outages.operator_action_ended_unrefused(module_id);
6479 format!(
6480 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6481 configured_module.enabled
6482 )
6483 })?;
6484 }
6485 }
6486
6487 for module_id in &added {
6488 let configured_module = configured
6489 .get(module_id)
6490 .expect("added module id came from configured map");
6491 supervisor
6492 .supervise_configured_requested(
6493 configured_module.module_spec(),
6494 configured_module.enabled,
6495 configured_module.health.clone(),
6496 configured_module.drain_timeout_ms,
6497 configured_module.restart,
6498 configured_module.register_timeout,
6499 Some(requester.clone()),
6500 )
6501 .map_err(|err| {
6502 format!("failed to add module_id '{module_id}' during rescan: {err}")
6503 })?;
6504 }
6505
6506 Ok((
6507 SupervisorRescanResult {
6508 added,
6509 removed,
6510 changed_pending_reload,
6511 enabled_changes: enabled_changes.iter().cloned().collect(),
6512 unchanged,
6513 preview: false,
6514 restart_required: Vec::new(),
6517 capability_warnings: Vec::new(),
6518 module_actions: Vec::new(),
6519 earlier_pending: Vec::new(),
6520 expect: None,
6521 },
6522 changed,
6523 ))
6524 }
6525
6526 async fn handle_supervisor_set_enabled(
6527 &self,
6528 frame: Frame,
6529 module_id: String,
6530 enabled: bool,
6531 requester: Principal,
6532 ) -> Result<Vec<Frame>, RouterError> {
6533 let operation_lock = self.supervisor.operation_lock();
6534 let _operation_guard = operation_lock.lock().await;
6535 let Some(module) = self.supervisor.get(&module_id) else {
6536 return Ok(vec![control_error_frame(
6537 &frame,
6538 "unknown_module",
6539 format!("module_id '{module_id}' is not supervised"),
6540 )?]);
6541 };
6542
6543 self.route_outages.mark_operator_action(&module_id);
6546 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6547 Ok(applied) => applied,
6548 Err(err) => {
6549 self.route_outages
6550 .operator_action_ended_unrefused(&module_id);
6551 return Ok(vec![control_error_frame(
6552 &frame,
6553 "target_unavailable",
6554 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6555 )?]);
6556 }
6557 };
6558 if !applied {
6559 self.route_outages
6562 .operator_action_ended_unrefused(&module_id);
6563 }
6564
6565 self.capability_evaluator.wake_deadline_loop();
6566 self.refresh_capability_requirements();
6567 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6568 Ok(vec![control_response_body_frame(
6569 &frame,
6570 &response,
6571 "ClientControlResponse::SupervisorAck",
6572 )?])
6573 }
6574
6575 async fn handle_supervisor_health_probe(
6576 &self,
6577 frame: Frame,
6578 module_id: String,
6579 ) -> Result<Vec<Frame>, RouterError> {
6580 self.refresh_capability_requirements();
6581 let Some(registration) = self
6582 .registry
6583 .get_module(&module_id)
6584 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6585 else {
6586 return Ok(vec![control_error_frame(
6587 &frame,
6588 "unknown_module",
6589 format!("module_id '{module_id}' is not registered"),
6590 )?]);
6591 };
6592
6593 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6606 {
6607 return Ok(vec![control_error_frame(
6608 &frame,
6609 "health_not_advertised",
6610 format!("module_id '{module_id}' did not advertise health.check"),
6611 )?]);
6612 }
6613
6614 let deadline = Instant::now() + self.health_probe_timeout;
6615 let pending = match self.forwarding.begin_module_control_rpc_for(
6616 &module_id,
6617 MODULE_CONTROL_OP_HEALTH_CHECK,
6618 deadline,
6619 ) {
6620 Ok(pending) => pending,
6621 Err(err) => {
6622 return Ok(vec![control_error_frame(
6623 &frame,
6624 forwarding_error_code(&err),
6625 err.to_string(),
6626 )?])
6627 }
6628 };
6629
6630 let PendingModuleControlRpc {
6631 endpoint,
6632 module_sink,
6633 negotiated_ver,
6634 corr: probe_corr,
6635 receiver,
6636 } = pending;
6637 let mut guard =
6638 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6639 let probe_body =
6640 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6641 RouterError::backend(
6642 0,
6643 frame.header.corr,
6644 format!("failed to encode health.check request: {err}"),
6645 )
6646 })?;
6647 let probe_frame = Frame::build_with_version(
6648 negotiated_ver,
6649 FrameType::Request,
6650 control_flags(),
6651 0,
6652 0,
6653 probe_corr,
6654 probe_body,
6655 )
6656 .map_err(RouterError::FrameBuild)?;
6657
6658 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6661 Ok(Ok(())) => {}
6662 Ok(Err(err)) => {
6663 return Ok(vec![control_error_frame(
6664 &frame,
6665 "target_unavailable",
6666 err.to_string(),
6667 )?]);
6668 }
6669 Err(_) => {
6670 return Ok(vec![control_error_frame(
6671 &frame,
6672 "module_timeout",
6673 format!(
6674 "module_id '{module_id}' did not answer health.check within {:?}",
6675 self.health_probe_timeout
6676 ),
6677 )?]);
6678 }
6679 }
6680
6681 match timeout_at(deadline, receiver).await {
6682 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6683 guard.disarm();
6684 let Some(report) = response.health_report() else {
6685 return Ok(vec![control_error_frame(
6686 &frame,
6687 "invalid_control_body",
6688 "health.check RPC returned a non-health response",
6689 )?]);
6690 };
6691 let HealthReport {
6696 status,
6697 detail,
6698 metrics,
6699 } = report;
6700 let capability_detail = self
6701 .capability_evaluator
6702 .required_problem_detail(&module_id);
6703 let response = ClientControlResponse::SupervisorHealthProbe {
6704 module_id,
6705 status,
6706 detail: append_capability_problem_detail(detail, capability_detail),
6707 metrics,
6708 };
6709 Ok(vec![control_response_body_frame(
6710 &frame,
6711 &response,
6712 "ClientControlResponse::SupervisorHealthProbe",
6713 )?])
6714 }
6715 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6716 guard.disarm();
6717 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
6720 }
6721 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6722 guard.disarm();
6723 Ok(vec![control_error_frame(
6724 &frame,
6725 "target_unavailable",
6726 message,
6727 )?])
6728 }
6729 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6730 guard.disarm();
6731 Ok(vec![control_error_frame(
6732 &frame,
6733 "invalid_control_body",
6734 message,
6735 )?])
6736 }
6737 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6738 guard.disarm();
6739 Ok(vec![control_error_frame(
6740 &frame,
6741 "invalid_control_body",
6742 format!("expected module-control op '{expected}', got '{actual}'"),
6743 )?])
6744 }
6745 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6746 guard.disarm();
6747 Ok(vec![control_error_frame(
6748 &frame,
6749 "module_timeout",
6750 format!(
6751 "module_id '{module_id}' answered health.check after {:?}",
6752 self.health_probe_timeout
6753 ),
6754 )?])
6755 }
6756 Ok(Err(_)) => Ok(vec![control_error_frame(
6757 &frame,
6758 "target_unavailable",
6759 "health.check waiter was canceled before the module responded",
6760 )?]),
6761 Err(_) => Ok(vec![control_error_frame(
6762 &frame,
6763 "module_timeout",
6764 format!(
6765 "module_id '{module_id}' did not answer health.check within {:?}",
6766 self.health_probe_timeout
6767 ),
6768 )?]),
6769 }
6770 }
6771
6772 fn supervisor_status(
6773 &self,
6774 module_id: &str,
6775 corr: u64,
6776 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6777 self.supervisor
6778 .get(module_id)
6779 .map(|module| {
6780 let warming = module.is_warming_for_control("status").map_err(|err| {
6781 RouterError::backend(
6782 0,
6783 corr,
6784 format!(
6785 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6786 ),
6787 )
6788 })?;
6789 module.status_for_control("status").map_err(|err| {
6790 RouterError::backend(
6791 0,
6792 corr,
6793 format!(
6794 "failed to read supervisor status for module_id '{module_id}': {err}"
6795 ),
6796 )
6797 }).map(|status| (status, warming))
6798 })
6799 .transpose()
6800 }
6801
6802 fn guard_module_control_op(
6803 &self,
6804 frame: &Frame,
6805 module_id: &str,
6806 op: &str,
6807 ) -> Result<Option<Frame>, RouterError> {
6808 if self.module_grants_op(module_id, op, frame.header.corr)? {
6809 return Ok(None);
6810 }
6811
6812 Ok(Some(control_error_frame(
6813 frame,
6814 "op_not_allowed",
6815 format!("module_id '{module_id}' did not grant control op '{op}'"),
6816 )?))
6817 }
6818
6819 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6820 let Some(registration) = self
6821 .registry
6822 .get_module(module_id)
6823 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6824 else {
6825 return Ok(false);
6826 };
6827 Ok(module_registration_grants_op(®istration.control_ops, op))
6828 }
6829
6830 fn handle_status_update(
6831 &self,
6832 endpoint: ModuleEndpointId,
6833 frame: Frame,
6834 ) -> Result<Vec<Frame>, RouterError> {
6835 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6836 Ok(update) => update,
6837 Err(err) => {
6838 if is_known_module_push_op(&frame.body) {
6843 return Ok(vec![control_error_frame(
6844 &frame,
6845 "invalid_control_body",
6846 format!("malformed module control push body: {err}"),
6847 )?]);
6848 }
6849 return Ok(Vec::new());
6850 }
6851 };
6852
6853 match update {
6854 ModuleControlPush::RouteStatus {
6855 route_channel,
6856 route_epoch,
6857 status,
6858 } => {
6859 self.forwarding
6860 .cache_status(endpoint, route_channel, route_epoch, status)
6861 .map_err(RouterError::Forwarding)?;
6862 }
6863 }
6864 Ok(Vec::new())
6865 }
6866
6867 fn handle_route_poll(
6868 &self,
6869 ctx: &RouteCtx,
6870 frame: Frame,
6871 route_channel: u16,
6872 route_epoch: u32,
6873 kind: PollKind,
6874 ) -> Result<Vec<Frame>, RouterError> {
6875 let snapshot = self
6876 .forwarding
6877 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6878 .map_err(RouterError::Forwarding)?;
6879 let response = match (kind, snapshot) {
6880 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6881 ClientControlResponse::RoutePoll {
6882 route_channel,
6883 route_epoch,
6884 status,
6885 live: None,
6886 }
6887 }
6888 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6889 route_channel,
6890 route_epoch,
6891 status: None,
6892 live: None,
6893 },
6894 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6895 let live = self
6921 .process_liveness
6922 .as_ref()
6923 .and_then(|source| source.process_live(&module_id))
6924 .unwrap_or(true);
6925 ClientControlResponse::RoutePoll {
6926 route_channel,
6927 route_epoch,
6928 status: None,
6929 live: Some(live),
6930 }
6931 }
6932 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6933 route_channel,
6934 route_epoch,
6935 status: None,
6936 live: Some(false),
6937 },
6938 };
6939
6940 Ok(vec![control_response_body_frame(
6941 &frame,
6942 &response,
6943 "ClientControlResponse::RoutePoll",
6944 )?])
6945 }
6946
6947 pub(crate) fn observe_module_control_completion(
6948 &self,
6949 completion: ModuleControlRpcCompletion,
6950 ) -> bool {
6951 match completion {
6952 ModuleControlRpcCompletion::Unknown => false,
6953 ModuleControlRpcCompletion::Settled => true,
6954 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6955 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6956 info!(
6957 module_id = %module_id,
6958 latency_ms,
6959 "late health.check answer proves the module is alive"
6960 );
6961 match self
6962 .supervisor
6963 .record_late_health_answer(&module_id, latency_ms)
6964 {
6965 Ok(true) => {}
6966 Ok(false) => debug!(
6967 module_id = %module_id,
6968 latency_ms,
6969 "late health.check answer has no active supervisor snapshot"
6970 ),
6971 Err(err) => warn!(
6972 module_id = %module_id,
6973 latency_ms,
6974 error = %err,
6975 "failed to record late health.check answer"
6976 ),
6977 }
6978 true
6979 }
6980 }
6981 }
6982
6983 fn refuse_to_end_module_connection_for_a_client(
7004 &self,
7005 module_connection_id: ConnectionId,
7006 corr: u64,
7007 err: ForwardingError,
7008 ) -> Result<(), RouterError> {
7009 if let ForwardingError::ConnectionClosing { connection_id } = err {
7010 if connection_id != module_connection_id {
7011 warn!(
7012 module_connection_id = module_connection_id.get(),
7013 client_connection_id = connection_id.get(),
7014 corr,
7015 "dropping a route.bind response for a closing client; the module connection keeps serving"
7016 );
7017 return Ok(());
7018 }
7019 }
7020 Err(RouterError::Forwarding(err))
7021 }
7022
7023 fn handle_module_relay_response(
7024 &self,
7025 connection_id: ConnectionId,
7026 frame: Frame,
7027 ) -> Result<Vec<Frame>, RouterError> {
7028 let mut secondary_error = None;
7029 let outcome = match frame.header.ty {
7030 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7031 Ok(probe) if probe.op == "route.bind" => {
7032 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7033 Ok(ModuleControlResponse::RouteBindAck {}) => {
7034 RouteBindRelayOutcome::Accepted
7035 }
7036 Ok(other) => {
7037 let message =
7038 format!("route.bind response carried unexpected body: {other:?}");
7039 secondary_error = Some(control_error_frame(
7040 &frame,
7041 "invalid_control_body",
7042 message.clone(),
7043 )?);
7044 RouteBindRelayOutcome::ModuleGone(message)
7045 }
7046 Err(err) => {
7047 let message = format!("malformed route.bind response body: {err}");
7048 secondary_error = Some(control_error_frame(
7049 &frame,
7050 "invalid_control_body",
7051 message.clone(),
7052 )?);
7053 RouteBindRelayOutcome::ModuleGone(message)
7054 }
7055 }
7056 }
7057 Ok(probe) => {
7058 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7059 {
7060 Ok(response) => ModuleControlRpcOutcome::Response(response),
7061 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7062 "malformed {} response body: {err}",
7063 probe.op
7064 )),
7065 };
7066 let completion = self
7067 .forwarding
7068 .complete_module_control_rpc(
7069 connection_id,
7070 frame.header.corr,
7071 Some(&probe.op),
7072 outcome,
7073 )
7074 .map_err(RouterError::Forwarding)?;
7075 if !self.observe_module_control_completion(completion) {
7076 debug!(
7077 connection_id = connection_id.get(),
7078 corr = frame.header.corr,
7079 op = %probe.op,
7080 "dropping late or unknown module-control RPC response"
7081 );
7082 }
7083 return Ok(Vec::new());
7084 }
7085 Err(err) => {
7086 if let Some(expected_op) = self
7087 .forwarding
7088 .pending_module_control_op(connection_id, frame.header.corr)
7089 .map_err(RouterError::Forwarding)?
7090 {
7091 let completion = self
7092 .forwarding
7093 .complete_module_control_rpc(
7094 connection_id,
7095 frame.header.corr,
7096 None,
7097 ModuleControlRpcOutcome::MalformedResponse(format!(
7098 "malformed {expected_op} response body: {err}"
7099 )),
7100 )
7101 .map_err(RouterError::Forwarding)?;
7102 if !self.observe_module_control_completion(completion) {
7103 debug!(
7104 connection_id = connection_id.get(),
7105 corr = frame.header.corr,
7106 "dropping late malformed module-control RPC response"
7107 );
7108 }
7109 return Ok(Vec::new());
7110 }
7111 let message = format!("malformed route.bind response body: {err}");
7112 secondary_error = Some(control_error_frame(
7113 &frame,
7114 "invalid_control_body",
7115 message.clone(),
7116 )?);
7117 RouteBindRelayOutcome::ModuleGone(message)
7118 }
7119 },
7120 FrameType::Error => {
7121 if self
7122 .forwarding
7123 .pending_module_control_op(connection_id, frame.header.corr)
7124 .map_err(RouterError::Forwarding)?
7125 .is_some()
7126 {
7127 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7128 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7129 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7130 "malformed module-control ERROR body: {err}"
7131 )),
7132 };
7133 let completion = self
7134 .forwarding
7135 .complete_module_control_rpc(
7136 connection_id,
7137 frame.header.corr,
7138 None,
7139 outcome,
7140 )
7141 .map_err(RouterError::Forwarding)?;
7142 if !self.observe_module_control_completion(completion) {
7143 debug!(
7144 connection_id = connection_id.get(),
7145 corr = frame.header.corr,
7146 "dropping late or unknown module-control RPC error"
7147 );
7148 }
7149 return Ok(Vec::new());
7150 }
7151 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7152 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7153 Err(err) => {
7154 let message = format!("malformed route.bind ERROR body: {err}");
7155 secondary_error = Some(control_error_frame(
7156 &frame,
7157 "invalid_control_body",
7158 message.clone(),
7159 )?);
7160 RouteBindRelayOutcome::ModuleGone(message)
7161 }
7162 }
7163 }
7164 ty => {
7165 return Ok(vec![control_error_frame(
7166 &frame,
7167 "unsupported_control_frame",
7168 format!("unsupported module channel-0 frame {ty:?}"),
7169 )?])
7170 }
7171 };
7172
7173 let settled =
7174 self.forwarding
7175 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7176 let completion = match settled {
7177 Ok(completion) => completion,
7178 Err(err) => {
7179 self.refuse_to_end_module_connection_for_a_client(
7180 connection_id,
7181 frame.header.corr,
7182 err,
7183 )?;
7184 return Ok(secondary_error.into_iter().collect());
7185 }
7186 };
7187 if let Some(target) = completion.abandoned.as_ref() {
7188 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7189 }
7190 if !completion.settled {
7191 debug!(
7192 connection_id = connection_id.get(),
7193 corr = frame.header.corr,
7194 frame_type = ?frame.header.ty,
7195 "dropping late or unknown route.bind relay response"
7196 );
7197 }
7198 Ok(secondary_error.into_iter().collect())
7199 }
7200
7201 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7202 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7203 self.cleanup_connection_with_end_reason(
7207 connection_id,
7208 RegistrationEndReason::ExplicitGoodbye,
7209 )
7210 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7211 Ok(Vec::new())
7212 }
7213}
7214
7215impl Default for ControlHandler {
7216 fn default() -> Self {
7217 Self::new(Arc::new(Registry::default()))
7218 }
7219}
7220
7221impl crate::supervise::SwapPromotionObserver for ControlHandler {
7222 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7223 self.apply_registration_capabilities(registration);
7224 }
7225}
7226
7227fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7228 CapabilityRequirementStatus {
7229 consumer: status.consumer,
7230 capability: status.capability,
7231 need: match status.need {
7232 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7233 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7234 },
7235 verdict: status.verdict.as_str().to_string(),
7236 episode_seq: status.episode_seq,
7237 config_satisfiable: status.config_satisfiable,
7238 runtime_available: status.runtime_available,
7239 detail: status.detail,
7240 }
7241}
7242
7243fn append_capability_problem_detail(
7244 detail: Option<String>,
7245 capability_detail: Option<String>,
7246) -> Option<String> {
7247 match (detail, capability_detail) {
7248 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7249 (Some(detail), None) => Some(detail),
7250 (None, Some(capability_detail)) => Some(capability_detail),
7251 (None, None) => None,
7252 }
7253}
7254
7255fn subc_ops() -> Vec<String> {
7256 SUBC_CONTROL_OPS
7257 .iter()
7258 .map(|op| (*op).to_string())
7259 .collect()
7260}
7261
7262fn module_subc_ops() -> Vec<String> {
7263 SUBC_CONTROL_OPS
7264 .iter()
7265 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7266 .map(|op| (*op).to_string())
7267 .collect()
7268}
7269
7270#[cfg(test)]
7271fn module_baseline_control_ops() -> Vec<String> {
7272 MODULE_BASELINE_CONTROL_OPS
7273 .iter()
7274 .map(|op| (*op).to_string())
7275 .collect()
7276}
7277
7278fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7279 let mut seen = HashSet::new();
7280 let mut effective = Vec::new();
7281 for op in MODULE_BASELINE_CONTROL_OPS {
7282 if seen.insert((*op).to_string()) {
7283 effective.push((*op).to_string());
7284 }
7285 }
7286 for op in declared.unwrap_or_default() {
7287 if seen.insert(op.clone()) {
7288 effective.push(op);
7289 }
7290 }
7291 effective
7292}
7293
7294fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7295 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7296}
7297
7298fn target_module_id(target: &RouteTarget) -> &str {
7299 match target {
7300 RouteTarget::ToolProvider { module_id }
7301 | RouteTarget::ManagementSurface { module_id }
7302 | RouteTarget::InternalService { module_id, .. } => module_id,
7303 }
7304}
7305
7306fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7307 roles.iter().any(|role| match (target, role) {
7308 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7309 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7310 (
7311 RouteTarget::InternalService { service_id, .. },
7312 ProviderRole::InternalService {
7313 service_id: provided,
7314 ..
7315 },
7316 ) => service_id == provided,
7317 _ => false,
7318 })
7319}
7320
7321fn is_routable_role(role: &ProviderRole) -> bool {
7322 matches!(
7323 role,
7324 ProviderRole::ToolProvider { .. }
7325 | ProviderRole::ManagementSurface { .. }
7326 | ProviderRole::InternalService { .. }
7327 )
7328}
7329
7330#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7331enum ControlRequestBodyError {
7332 UnknownOp,
7333 InvalidBody,
7334}
7335
7336#[derive(Debug, Deserialize)]
7337struct ControlOpProbe {
7338 op: String,
7339}
7340
7341const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7344
7345fn is_known_module_push_op(body: &[u8]) -> bool {
7346 serde_json::from_slice::<ControlOpProbe>(body)
7347 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7348 .unwrap_or(false)
7349}
7350
7351fn is_known_module_request_op(body: &[u8]) -> bool {
7352 serde_json::from_slice::<ControlOpProbe>(body)
7353 .map(|probe| is_module_to_subc_op(&probe.op))
7354 .unwrap_or(false)
7355}
7356
7357fn is_module_to_subc_op(op: &str) -> bool {
7358 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7359}
7360
7361fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7362 debug!(
7363 op = %op,
7364 connection_id = connection_id.get(),
7365 corr,
7366 "control dispatch"
7367 );
7368}
7369
7370fn log_slow_control_dispatch(
7371 dispatch_started_at: Option<StdInstant>,
7372 op: &'static str,
7373 connection_id: ConnectionId,
7374 corr: u64,
7375 route_open_timings: Option<&RouteOpenTimings>,
7376) {
7377 let Some(dispatch_started_at) = dispatch_started_at else {
7378 return;
7379 };
7380 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7382 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7383 if let Some(timings) = route_open_timings {
7384 warn!(
7385 op = %op,
7386 connection_id = connection_id.get(),
7387 corr,
7388 elapsed_ms = elapsed.as_millis() as u64,
7389 module_id = ?timings.module_id,
7390 principal = %timings.principal,
7391 admission_ms = timings.admission.as_millis() as u64,
7392 project_root_ms = timings.project_root.as_millis() as u64,
7393 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7394 bind_send_ms = timings.bind_send.as_millis() as u64,
7395 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7396 "slow control dispatch"
7397 );
7398 return;
7399 }
7400 warn!(
7401 op = %op,
7402 connection_id = connection_id.get(),
7403 corr,
7404 elapsed_ms = elapsed.as_millis() as u64,
7405 "slow control dispatch"
7406 );
7407 }
7408}
7409
7410#[cfg(test)]
7412type TestCallerLookup =
7413 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7414
7415fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7417 match request {
7418 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7419 Some(PrivilegedOperation::Restart { module_id })
7420 }
7421 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7422 Some(PrivilegedOperation::Swap { module_id })
7423 }
7424 ClientControlRequest::SupervisorReload { module_id } => {
7425 Some(PrivilegedOperation::Reload { module_id })
7426 }
7427 ClientControlRequest::SupervisorRescan { preview } => {
7428 if *preview {
7429 None
7430 } else {
7431 Some(PrivilegedOperation::Rescan)
7432 }
7433 }
7434 ClientControlRequest::SupervisorRescanChecked {
7439 preview,
7440 candidate,
7441 expect,
7442 } => (!*preview && candidate.is_none() && expect.is_some())
7443 .then_some(PrivilegedOperation::Rescan),
7444 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7445 Some(PrivilegedOperation::SetEnabled {
7446 module_id,
7447 enabled: *enabled,
7448 })
7449 }
7450 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7451 Some(PrivilegedOperation::ReleaseReserved { module_id })
7452 }
7453 ClientControlRequest::ServerDescribe {}
7454 | ClientControlRequest::CatalogList { .. }
7455 | ClientControlRequest::RouteOpen { .. }
7456 | ClientControlRequest::RoutePoll { .. }
7457 | ClientControlRequest::SupervisorList {}
7458 | ClientControlRequest::SupervisorHealth {}
7459 | ClientControlRequest::SupervisorHealthProbe { .. }
7460 | ClientControlRequest::SupervisorRoutes { .. }
7461 | ClientControlRequest::SupervisorProvenance { .. }
7462 | ClientControlRequest::SupervisorStderrTail { .. }
7463 | ClientControlRequest::SupervisorTerminals { .. }
7464 | ClientControlRequest::SupervisorSpawnSnapshot {}
7465 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7466 }
7467}
7468
7469fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7470 match request {
7471 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7472 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7473 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7474 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7475 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7476 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7477 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7478 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7479 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7480 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7481 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7482 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7483 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7484 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7485 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7486 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7487 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7488 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7489 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7490 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7491 }
7492}
7493
7494struct RouteOpenCaller {
7502 principal: String,
7506 scope: String,
7509 harness: String,
7512}
7513
7514impl RouteOpenCaller {
7515 fn new(
7516 principal: Option<&Principal>,
7517 scope: Option<&ScopeSelector>,
7518 identity: &BindIdentity,
7519 ) -> Self {
7520 Self {
7521 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7522 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7523 harness: bounded_log_text(&identity.harness),
7524 }
7525 }
7526
7527 fn unknown() -> Self {
7530 Self {
7531 principal: "unknown".to_string(),
7532 scope: "unknown".to_string(),
7533 harness: "unknown".to_string(),
7534 }
7535 }
7536}
7537
7538const MAX_LOGGED_TEXT_CHARS: usize = 64;
7541
7542fn bounded_log_text(value: &str) -> String {
7546 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7547 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7548 bounded.push('…');
7549 }
7550 format!("{bounded:?}")
7551}
7552
7553fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7559 use subc_protocol::scope::ScopeKind;
7560 let kind = match stamp.kind {
7561 ScopeKind::Head => "head",
7562 ScopeKind::Worker => "worker",
7563 ScopeKind::Ephemeral => "ephemeral",
7564 };
7565 format!(
7566 "{kind}/{}/{}@{}",
7567 principal_label(&stamp.owner),
7568 bounded_log_text(&stamp.scope_ref),
7569 stamp.scope_epoch
7570 )
7571}
7572
7573fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7580 let epoch = selector
7581 .scope_epoch
7582 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7583 format!(
7584 "requested/{}/{}@{epoch}",
7585 bounded_log_text(&principal_label(&selector.owner)),
7586 bounded_log_text(&selector.scope_ref),
7587 )
7588}
7589
7590fn principal_label(principal: &Principal) -> String {
7591 match principal {
7592 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7593 Principal::Direct => "direct".to_string(),
7594 other => format!("{other:?}"),
7595 }
7596}
7597
7598fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7599 match request {
7600 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7601 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7602 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7603 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7604 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7605 }
7606}
7607
7608fn parse_client_control_request(
7609 body: &[u8],
7610) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7611 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7612 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7613 Ok(probe)
7614 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7615 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7616 {
7617 ControlRequestBodyError::InvalidBody
7618 }
7619 Ok(_) => ControlRequestBodyError::UnknownOp,
7620 Err(_) => ControlRequestBodyError::InvalidBody,
7621 };
7622 (err, classification)
7623 })
7624}
7625
7626fn parse_module_control_request_from_module(
7627 body: &[u8],
7628) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7629 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7630 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7631 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7632 Ok(_) => ControlRequestBodyError::UnknownOp,
7633 Err(_) => ControlRequestBodyError::InvalidBody,
7634 };
7635 (err, classification)
7636 })
7637}
7638
7639#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7640enum ProviderRoleKind {
7641 ToolProvider,
7642 PipelineStage,
7643 ManagementSurface,
7644 InternalService,
7645}
7646
7647fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7648 match role {
7649 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7650 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7651 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7652 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7653 }
7654}
7655
7656fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7657 roles.iter().map(provider_role_kind).collect()
7658}
7659
7660const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7663
7664#[derive(Debug, Default, PartialEq, Eq)]
7666struct ScopeOutcomeCounts {
7667 created: usize,
7668 replaced: usize,
7669 updated: usize,
7670 unchanged: usize,
7671 refused: usize,
7672}
7673
7674impl ScopeOutcomeCounts {
7675 fn of(results: &[ScopeRecordResult]) -> Self {
7676 let mut counts = Self::default();
7677 for result in results {
7678 let slot = match result.outcome {
7679 ScopeRecordOutcome::Created => &mut counts.created,
7680 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7681 ScopeRecordOutcome::Updated => &mut counts.updated,
7682 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7683 ScopeRecordOutcome::Refused => &mut counts.refused,
7684 };
7685 *slot += 1;
7686 }
7687 counts
7688 }
7689}
7690
7691#[cfg(test)]
7692mod scope_outcome_count_tests {
7693 use super::*;
7694
7695 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7696 ScopeRecordResult {
7697 scope_ref: "r".to_string(),
7698 scope_epoch: 1,
7699 outcome,
7700 code: None,
7701 message: None,
7702 version: None,
7703 parent_state: None,
7704 }
7705 }
7706
7707 #[test]
7710 fn every_outcome_is_counted_in_its_own_field() {
7711 let results = [
7712 result(ScopeRecordOutcome::Created),
7713 result(ScopeRecordOutcome::Created),
7714 result(ScopeRecordOutcome::Replaced),
7715 result(ScopeRecordOutcome::Updated),
7716 result(ScopeRecordOutcome::Unchanged),
7717 result(ScopeRecordOutcome::Refused),
7718 result(ScopeRecordOutcome::Refused),
7719 result(ScopeRecordOutcome::Refused),
7720 ];
7721 assert_eq!(
7722 ScopeOutcomeCounts::of(&results),
7723 ScopeOutcomeCounts {
7724 created: 2,
7725 replaced: 1,
7726 updated: 1,
7727 unchanged: 1,
7728 refused: 3,
7729 }
7730 );
7731 }
7732}
7733
7734fn capability_census_trigger(
7738 old: Option<&CapabilityDeclarations>,
7739 new: Option<&CapabilityDeclarations>,
7740) -> bool {
7741 let old_provides = old
7742 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7743 .unwrap_or_default();
7744 let old_denies = old
7745 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7746 .unwrap_or_default();
7747 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7748 provides: Vec::new(),
7749 requires: Vec::new(),
7750 must_never_reach: Vec::new(),
7751 });
7752
7753 new.provides
7754 .iter()
7755 .any(|capability| !old_provides.contains(capability))
7756 || new
7757 .must_never_reach
7758 .iter()
7759 .any(|capability| !old_denies.contains(capability))
7760}
7761
7762fn denied_capability<'a>(
7765 opening_manifest: &'a ModuleManifest,
7766 target_manifest: &ModuleManifest,
7767) -> Option<&'a str> {
7768 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7769 let target_capabilities = target_manifest.capabilities.as_ref()?;
7770 opening_capabilities
7771 .must_never_reach
7772 .iter()
7773 .find(|denied| {
7774 target_capabilities
7775 .provides
7776 .iter()
7777 .any(|provided| provided == *denied)
7778 })
7779 .map(String::as_str)
7780}
7781
7782fn catalog_update_frozen_field_message(
7783 registered: &ModuleManifest,
7784 provides: &[ProviderRole],
7785) -> Option<String> {
7786 let old_has_provides = !registered.provides.is_empty();
7787 let new_has_provides = !provides.is_empty();
7788 if old_has_provides != new_has_provides {
7789 return Some(format!(
7790 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7791 registered.module_id
7792 ));
7793 }
7794
7795 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7796 return Some(format!(
7797 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7798 registered.module_id
7799 ));
7800 }
7801
7802 let registered_concurrency = manifest_concurrency(registered);
7803 let mut candidate = registered.clone();
7804 candidate.provides = provides.to_vec();
7805 let candidate_concurrency = manifest_concurrency(&candidate);
7806 if candidate_concurrency != registered_concurrency {
7807 return Some(format!(
7808 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7809 registered.module_id, registered_concurrency, candidate_concurrency
7810 ));
7811 }
7812
7813 None
7816}
7817
7818fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7819 manifest.provides.iter().any(is_routable_role)
7820}
7821
7822fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7828 manifest
7829 .provides
7830 .iter()
7831 .find_map(|provider| match provider {
7832 ProviderRole::ToolProvider { concurrency, .. }
7833 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7834 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7835 })
7836 .unwrap_or(Concurrency::ModuleManaged)
7837}
7838
7839fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7846 let has_management_surface = manifest
7847 .provides
7848 .iter()
7849 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7850 if !has_management_surface {
7851 return false;
7852 }
7853 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7854 return false;
7855 };
7856 let Some(provides) = raw
7857 .get("manifest")
7858 .and_then(|manifest| manifest.get("provides"))
7859 .and_then(serde_json::Value::as_array)
7860 else {
7861 return false;
7862 };
7863 provides.iter().any(|role| {
7868 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7869 && role.get("concurrency").is_none()
7870 })
7871}
7872
7873fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7874 if peer_version != PROTOCOL_VERSION {
7875 return Err(format!(
7876 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7877 ));
7878 }
7879 Ok(PROTOCOL_VERSION)
7880}
7881
7882fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7883 Frame::build_with_version(
7884 response_version(frame),
7885 FrameType::Pong,
7886 frame.header.flags,
7887 0,
7888 0,
7889 frame.header.corr,
7890 Vec::new(),
7891 )
7892 .map_err(RouterError::FrameBuild)
7893}
7894
7895pub(crate) fn control_error_frame(
7896 frame: &Frame,
7897 code: &'static str,
7898 message: impl Into<String>,
7899) -> Result<Frame, RouterError> {
7900 control_error_body_frame(
7901 frame,
7902 ErrorBody {
7903 code: code.to_string(),
7904 message: message.into(),
7905 detail: None,
7906 },
7907 )
7908}
7909
7910fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7918 control_error_frame_with_flags(frame, error, daemon_error_flags())
7919}
7920
7921fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7927 control_error_frame_with_flags(frame, error, control_flags())
7928}
7929
7930fn control_error_frame_with_flags(
7931 frame: &Frame,
7932 error: ErrorBody,
7933 flags: Flags,
7934) -> Result<Frame, RouterError> {
7935 let body = serde_json::to_vec(&error).map_err(|err| {
7936 RouterError::backend(
7937 0,
7938 frame.header.corr,
7939 format!("failed to encode control ERROR: {err}"),
7940 )
7941 })?;
7942
7943 Frame::build_with_version(
7944 response_version(frame),
7945 FrameType::Error,
7946 flags,
7947 0,
7948 0,
7949 frame.header.corr,
7950 body,
7951 )
7952 .map_err(RouterError::FrameBuild)
7953}
7954
7955fn control_response_body_frame<T: Serialize>(
7956 frame: &Frame,
7957 reply: &T,
7958 label: &'static str,
7959) -> Result<Frame, RouterError> {
7960 let body = serde_json::to_vec(reply).map_err(|err| {
7961 RouterError::backend(
7962 0,
7963 frame.header.corr,
7964 format!("failed to encode {label}: {err}"),
7965 )
7966 })?;
7967
7968 Frame::build_with_version(
7969 response_version(frame),
7970 FrameType::Response,
7971 control_flags(),
7972 0,
7973 0,
7974 frame.header.corr,
7975 body,
7976 )
7977 .map_err(RouterError::FrameBuild)
7978}
7979
7980fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8010 match err {
8011 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8012 ForwardingError::NoModuleConnection => "target_unavailable",
8013 ForwardingError::ModuleReloading { .. } => "module_reloading",
8014 ForwardingError::ClientRouteChannelExhausted { .. }
8015 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8016 ForwardingError::StaleModuleEndpoint
8017 | ForwardingError::UnknownReservation { .. }
8018 | ForwardingError::ConnectionClosing { .. }
8019 | ForwardingError::ClientEgressClosed { .. }
8020 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8021 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8024 ForwardingError::RelayCorrelationExhausted
8025 | ForwardingError::RouteOpenBuild(_)
8026 | ForwardingError::Poisoned => "forwarding_error",
8027 }
8028}
8029
8030fn response_version(frame: &Frame) -> u8 {
8031 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8032 frame.header.ver
8033 } else {
8034 PROTOCOL_VERSION
8035 }
8036}
8037
8038fn control_flags() -> Flags {
8039 Flags::new(false, Priority::Passive, false)
8040}
8041
8042fn send_goodbye_target_best_effort(
8047 counters: &DaemonCounters,
8048 target: &GoodbyeTarget,
8049 context: &'static str,
8050) {
8051 let Ok(frame) = Frame::build_with_version(
8052 target.negotiated_ver,
8053 FrameType::Goodbye,
8054 control_flags(),
8055 target.channel,
8056 target.epoch,
8057 0,
8058 Vec::new(),
8059 ) else {
8060 return;
8061 };
8062 crate::forwarding::send_module_route_goodbye(
8063 counters,
8064 &target.sink,
8065 frame,
8066 target.module_id.as_deref(),
8067 context,
8068 );
8069}
8070
8071pub(crate) fn send_route_control_pushes(
8072 forwarding: &ForwardingTable,
8073 routes: Vec<EndpointRoute>,
8074 push: ClientControlPush,
8075) {
8076 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8077 for route in routes {
8078 let target = route.goodbye_target;
8079 if let Some((existing, channels)) = targets
8080 .iter_mut()
8081 .find(|(existing, _)| existing.connection_id == target.connection_id)
8082 {
8083 debug_assert_eq!(
8084 existing.negotiated_ver, target.negotiated_ver,
8085 "one connection cannot negotiate multiple frame versions"
8086 );
8087 if !channels.contains(&target.channel) {
8088 channels.push(target.channel);
8089 }
8090 continue;
8091 }
8092 let channel = target.channel;
8093 targets.push((target, vec![channel]));
8094 }
8095 for (target, mut channels) in targets {
8096 channels.sort_unstable();
8097 let mut push = push.clone();
8098 match &mut push {
8099 ClientControlPush::RouteClosing {
8100 channels: covered, ..
8101 }
8102 | ClientControlPush::RouteClosed {
8103 channels: covered, ..
8104 } => *covered = channels,
8105 }
8106 let body = match serde_json::to_vec(&push) {
8107 Ok(body) => body,
8108 Err(err) => {
8109 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8110 continue;
8111 }
8112 };
8113 let frame = match Frame::build_with_version(
8114 target.negotiated_ver,
8115 FrameType::Push,
8116 control_flags(),
8117 0,
8118 0,
8119 0,
8120 body.clone(),
8121 ) {
8122 Ok(frame) => frame,
8123 Err(err) => {
8124 warn!(
8125 route_channel = target.channel,
8126 error = %err,
8127 "failed to build route lifecycle control PUSH frame"
8128 );
8129 continue;
8130 }
8131 };
8132 if let Err(err) = target.sink.try_send(frame) {
8133 if target.close_on_delivery_failure() {
8134 warn!(
8135 target_connection_id = target.connection_id.get(),
8136 route_channel = target.channel,
8137 error = %err,
8138 "route lifecycle control PUSH was not delivered to client; closing target connection"
8139 );
8140 let _ = forwarding.escalate_client_delivery_failure(
8141 target.connection_id,
8142 target.channel,
8143 target.epoch,
8144 CloseReason::new(
8145 "route_lifecycle_push_delivery_failed",
8146 format!(
8147 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8148 target.channel
8149 ),
8150 ),
8151 crate::forwarding::UndeliveredFrame {
8152 module_id: target.module_id.as_deref(),
8153 sink: &target.sink,
8154 },
8155 );
8156 }
8157 }
8158 }
8159}
8160
8161#[cfg(test)]
8162mod tests {
8163 mod activity_read {
8164 include!("activity_read_tests.rs");
8165 }
8166
8167 mod lifecycle_activity {
8168 include!("control_lifecycle_tests.rs");
8169 }
8170 #[cfg(unix)]
8171 #[tokio::test]
8172 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8173 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8174 let path = dir.join("subc.jsonc");
8175 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8176 "program":"/bin/sleep","args":["60"],"protocol":"none",
8177 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8178 }}}).to_string()).unwrap();
8179 let mut configured = crate::daemon_config::load(&path)
8180 .unwrap()
8181 .unwrap()
8182 .modules
8183 .pop()
8184 .unwrap();
8185 let registry = std::sync::Arc::new(crate::Registry::default());
8186 let handle = crate::SupervisorHandle::new();
8187 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8188 .with_handle(handle.clone());
8189 let module = supervisor
8190 .supervise_configured_with_health(
8191 configured.module_spec(),
8192 true,
8193 configured.health.clone(),
8194 None,
8195 configured.restart,
8196 )
8197 .unwrap();
8198 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8199 let before = module.status().unwrap().pid;
8200 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8201 configured.health.cadence = std::time::Duration::from_secs(3600);
8202 let health_only = handler
8203 .reconcile_supervised_modules(
8204 &supervisor,
8205 vec![configured.clone()],
8206 false,
8207 Principal::Direct,
8208 )
8209 .await
8210 .unwrap();
8211 assert!(
8212 health_only.0.changed_pending_reload.is_empty(),
8213 "health policy is already applied live"
8214 );
8215 assert_eq!(module.status().unwrap().pid, before);
8216 assert_eq!(
8217 module.configuration().unwrap().1.http,
8218 configured.health.http
8219 );
8220 configured.args = vec!["61".into()];
8221 let launch = handler
8222 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8223 .await
8224 .unwrap();
8225 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8226 assert_eq!(
8227 module.status().unwrap().pid,
8228 before,
8229 "a launch edit is stored until reload"
8230 );
8231 module.drain().await.unwrap();
8232 }
8233 use cortexkit_test_support::ScratchDir;
8234 use std::{
8235 collections::BTreeMap,
8236 fmt,
8237 path::PathBuf,
8238 sync::{Arc, Mutex},
8239 time::Duration,
8240 };
8241
8242 use serde_json::{json, Value};
8243 use subc_protocol::{
8244 manifest::{
8245 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8246 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8247 },
8248 session::HealthStatus,
8249 FrameType,
8250 };
8251
8252 use super::*;
8253 use crate::{
8254 forwarding::{DataRoute, DataRouteState},
8255 registry::ChannelState,
8256 router::FrameSink,
8257 stderr_tail::DEFAULT_MAX_LINE_BYTES,
8258 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8259 RouteCtx, Router,
8260 };
8261 use tokio::{
8262 sync::mpsc,
8263 time::{sleep, Instant},
8264 };
8265 use tracing::{
8266 field::{Field, Visit},
8267 Event, Subscriber,
8268 };
8269 use tracing_subscriber::{layer::Context, prelude::*, Layer};
8270
8271 fn fake_aft_stub_path() -> PathBuf {
8290 let mut path = std::env::current_exe().expect("current_exe available in tests");
8291 path.pop(); path.pop(); path.push(if cfg!(windows) {
8294 "fake-aft-stub.exe"
8295 } else {
8296 "fake-aft-stub"
8297 });
8298 assert!(
8299 path.exists(),
8300 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8301 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8302 path.display()
8303 );
8304 path
8305 }
8306
8307 fn client_retries(code: &str) -> bool {
8312 subc_protocol::error_codes::is_retryable_route_open(code)
8313 }
8314
8315 #[test]
8326 fn retryability_of_forwarding_codes_matches_the_failure() {
8327 let transient = [
8330 ForwardingError::NoModuleConnection,
8331 ForwardingError::ModuleReloading {
8332 module_id: "m".into(),
8333 },
8334 ForwardingError::StaleModuleEndpoint,
8335 ForwardingError::UnknownReservation {
8336 client_channel: 1,
8337 module_channel: 1,
8338 },
8339 ForwardingError::ConnectionClosing {
8340 connection_id: ConnectionId::new(1),
8341 },
8342 ForwardingError::ClientEgressClosed {
8343 connection_id: ConnectionId::new(1),
8344 },
8345 ForwardingError::ModuleEgressUnavailable {
8346 connection_id: ConnectionId::new(1),
8347 },
8348 ];
8349 for err in transient {
8350 let code = forwarding_error_code(&err);
8351 assert!(
8352 client_retries(code),
8353 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8354 );
8355 }
8356
8357 let permanent = [
8362 ForwardingError::ConnectionRoleConflict {
8363 connection_id: ConnectionId::new(1),
8364 },
8365 ForwardingError::ClientRouteChannelExhausted {
8366 connection_id: ConnectionId::new(1),
8367 },
8368 ForwardingError::ModuleRouteChannelExhausted {
8369 endpoint: ModuleEndpointId {
8370 connection_id: ConnectionId::new(1),
8371 generation: 1,
8372 },
8373 },
8374 ForwardingError::RelayCorrelationExhausted,
8375 ForwardingError::RouteOpenBuild("x".into()),
8376 ForwardingError::Poisoned,
8377 ];
8378 for err in permanent {
8379 let code = forwarding_error_code(&err);
8380 assert!(
8381 !client_retries(code),
8382 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8383 );
8384 }
8385 }
8386
8387 #[tokio::test]
8415 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8416 let handler = ControlHandler::default();
8417 let frame =
8418 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8419
8420 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8424 assert_eq!(
8425 stamped,
8426 Principal::Direct,
8427 "a caller that proved nothing must not be stamped as a supervised module"
8428 );
8429
8430 let forged = handler
8435 .route_open_principal(
8436 &frame,
8437 Some(ConsumerIdentity {
8438 module_id: "aft".to_string(),
8439 launch_nonce: "not-a-real-nonce".to_string(),
8440 }),
8441 )
8442 .unwrap();
8443 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8444 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8445 }
8446
8447 #[test]
8464 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8465 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8466 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8467 let ClientControlRequest::RouteOpen {
8468 consumer_identity, ..
8469 } = parsed
8470 else {
8471 panic!("route.open body must parse as RouteOpen");
8472 };
8473 assert_eq!(
8474 consumer_identity,
8475 Some(ConsumerIdentity {
8476 module_id: "aft".to_string(),
8477 launch_nonce: "n".to_string(),
8478 }),
8479 "the wire field name must reach the value route_open_principal reads"
8480 );
8481 }
8482
8483 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8484 ModuleManifest::builder(module_id, "0.1.0")
8485 .protocol_ver(protocol_ver)
8486 .provides(vec![ProviderRole::ToolProvider {
8487 tools: vec![Tool {
8488 name: "read".to_string(),
8489 description: None,
8490 execution_mode: ExecutionMode::Pure,
8491 schema: json!({"type": "object"}),
8492 }],
8493 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8494 concurrency: Concurrency::ModuleManaged,
8495 emits_push: true,
8496 sub_supervises: true,
8497 }])
8498 .build()
8499 }
8500
8501 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8502 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8503 }
8504
8505 fn hello_frame_with_control_ops(
8506 module_id: &str,
8507 protocol_ver: u8,
8508 corr: u64,
8509 control_ops: Option<Vec<String>>,
8510 ) -> Frame {
8511 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8512 }
8513
8514 fn hello_frame_with_nonce(
8515 module_id: &str,
8516 protocol_ver: u8,
8517 corr: u64,
8518 launch_nonce: Option<&str>,
8519 ) -> Frame {
8520 hello_frame_full(
8521 module_id,
8522 protocol_ver,
8523 corr,
8524 None,
8525 launch_nonce.map(ToOwned::to_owned),
8526 )
8527 }
8528
8529 fn hello_frame_full(
8530 module_id: &str,
8531 protocol_ver: u8,
8532 corr: u64,
8533 control_ops: Option<Vec<String>>,
8534 launch_nonce: Option<String>,
8535 ) -> Frame {
8536 let body = serde_json::to_vec(&ModuleHelloBody {
8537 manifest: manifest(module_id, protocol_ver),
8538 protocol_ver,
8539 control_ops,
8540 launch_nonce,
8541 })
8542 .unwrap();
8543 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8544 }
8545
8546 fn hello_frame_with_rich_ui(
8547 module_id: &str,
8548 protocol_ver: u8,
8549 corr: u64,
8550 launch_nonce: Option<&str>,
8551 rich_ui: Value,
8552 ) -> Frame {
8553 let mut body = serde_json::to_value(ModuleHelloBody {
8554 manifest: manifest(module_id, protocol_ver),
8555 protocol_ver,
8556 control_ops: None,
8557 launch_nonce: launch_nonce.map(str::to_string),
8558 })
8559 .unwrap();
8560 body["manifest"]["rich_ui"] = rich_ui;
8561 Frame::build(
8562 FrameType::Hello,
8563 control_flags(),
8564 0,
8565 0,
8566 corr,
8567 serde_json::to_vec(&body).unwrap(),
8568 )
8569 .unwrap()
8570 }
8571
8572 fn non_routable_hello_frame_with_control_ops(
8573 module_id: &str,
8574 corr: u64,
8575 control_ops: Option<Vec<String>>,
8576 ) -> Frame {
8577 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8578 manifest.provides.clear();
8579 let body = serde_json::to_vec(&ModuleHelloBody {
8580 manifest,
8581 protocol_ver: PROTOCOL_VERSION,
8582 control_ops,
8583 launch_nonce: None,
8584 })
8585 .unwrap();
8586 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8587 }
8588
8589 fn capability_grammar_hello_frame(
8590 capabilities: Value,
8591 runtime_computed: Option<Value>,
8592 corr: u64,
8593 ) -> Frame {
8594 let mut body = serde_json::to_value(ModuleHelloBody {
8595 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8596 protocol_ver: PROTOCOL_VERSION,
8597 control_ops: None,
8598 launch_nonce: None,
8599 })
8600 .expect("HELLO body serializes");
8601 body["manifest"]["capabilities"] = capabilities;
8602 if let Some(runtime_computed) = runtime_computed {
8603 body["runtime_computed"] = runtime_computed;
8604 }
8605 Frame::build(
8606 FrameType::Hello,
8607 control_flags(),
8608 0,
8609 0,
8610 corr,
8611 serde_json::to_vec(&body).expect("HELLO body reserializes"),
8612 )
8613 .expect("HELLO frame builds")
8614 }
8615
8616 fn channel_request(channel: u16, corr: u64) -> Frame {
8617 Frame::build(
8618 FrameType::Request,
8619 Flags::new(true, Priority::Interactive, false),
8620 channel,
8621 0,
8622 corr,
8623 b"opaque".to_vec(),
8624 )
8625 .unwrap()
8626 }
8627
8628 fn route_ctx(
8629 connection_id: ConnectionId,
8630 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8631 let (tx, rx) = mpsc::channel(8);
8632 (
8633 RouteCtx {
8634 connection_id,
8635 egress: FrameSink::new(tx),
8636 },
8637 rx,
8638 )
8639 }
8640
8641 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8642 serde_json::from_slice(&frame.body).unwrap()
8643 }
8644
8645 async fn hello_via_sink(
8650 handler: &ControlHandler,
8651 ctx: &RouteCtx,
8652 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8653 hello: Frame,
8654 ) -> Frame {
8655 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8656 assert!(
8657 replies.is_empty(),
8658 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8659 );
8660 let ack = rx
8661 .try_recv()
8662 .expect("HELLO_ACK is queued on the module sink")
8663 .frame;
8664 assert_eq!(ack.header.ty, FrameType::HelloAck);
8665 ack
8666 }
8667
8668 fn parse_error(frame: &Frame) -> Value {
8669 serde_json::from_slice(&frame.body).unwrap()
8670 }
8671
8672 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8673 serde_json::from_slice(&frame.body).unwrap()
8674 }
8675
8676 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8677 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8678 route_channel,
8679 route_epoch: 0,
8680 kind,
8681 })
8682 .unwrap();
8683 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8684 }
8685
8686 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8687 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8688 module_id: module_id.to_string(),
8689 })
8690 .unwrap();
8691 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8692 }
8693
8694 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8695 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8696 }
8697
8698 fn route_open_frame_with_consumer_capabilities(
8699 corr: u64,
8700 module_id: &str,
8701 project_root: ScratchDir,
8702 consumer_capabilities: Option<Vec<String>>,
8703 ) -> Frame {
8704 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8705 target: RouteTarget::ToolProvider {
8706 module_id: module_id.to_string(),
8707 },
8708 identity: BindIdentity::new(
8709 project_root.path().to_path_buf(),
8710 "unit".to_string(),
8711 "session".to_string(),
8712 ),
8713 consumer_identity: None,
8714 consumer_capabilities,
8715 role_versions: None,
8716 admission_facts: None,
8717 scope: None,
8718 })
8719 .unwrap();
8720 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8721 }
8722
8723 fn route_open_frame_with_role_versions(
8724 corr: u64,
8725 module_id: &str,
8726 project_root: ScratchDir,
8727 role_versions: Option<BTreeMap<String, String>>,
8728 ) -> Frame {
8729 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8730 target: RouteTarget::ToolProvider {
8731 module_id: module_id.to_string(),
8732 },
8733 identity: BindIdentity::new(
8734 project_root.path().to_path_buf(),
8735 "unit".to_string(),
8736 format!("session-{corr}"),
8737 ),
8738 consumer_identity: None,
8739 consumer_capabilities: None,
8740 role_versions,
8741 admission_facts: None,
8742 scope: None,
8743 })
8744 .unwrap();
8745 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8746 }
8747
8748 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8749 entries
8750 .iter()
8751 .map(|(role, version)| (role.to_string(), version.to_string()))
8752 .collect()
8753 }
8754
8755 fn route_open_frame_with_admission_facts(
8756 corr: u64,
8757 module_id: &str,
8758 project_root: ScratchDir,
8759 consumer_identity: Option<subc_control::ConsumerIdentity>,
8760 facts: Option<Value>,
8761 ) -> Frame {
8762 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8763 target: RouteTarget::ToolProvider {
8764 module_id: module_id.to_string(),
8765 },
8766 identity: BindIdentity::new(
8767 project_root.path().to_path_buf(),
8768 "unit".to_string(),
8769 format!("session-{corr}"),
8770 ),
8771 consumer_identity,
8772 consumer_capabilities: None,
8773 role_versions: None,
8774 admission_facts: facts,
8775 scope: None,
8776 })
8777 .unwrap();
8778 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8779 }
8780
8781 #[derive(Clone, Default)]
8782 struct EventCapture {
8783 events: Arc<Mutex<Vec<CapturedEvent>>>,
8784 }
8785
8786 #[derive(Clone, Debug)]
8787 struct CapturedEvent {
8788 target: String,
8789 level: tracing::Level,
8790 fields: BTreeMap<String, String>,
8791 }
8792
8793 impl EventCapture {
8794 fn events(&self) -> Vec<CapturedEvent> {
8795 self.events.lock().unwrap().clone()
8796 }
8797 }
8798
8799 impl<S> Layer<S> for EventCapture
8800 where
8801 S: Subscriber,
8802 {
8803 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8804 let mut visitor = EventFieldVisitor::default();
8805 event.record(&mut visitor);
8806 self.events.lock().unwrap().push(CapturedEvent {
8807 target: event.metadata().target().to_string(),
8808 level: *event.metadata().level(),
8809 fields: visitor.fields,
8810 });
8811 }
8812 }
8813
8814 #[derive(Default)]
8815 struct EventFieldVisitor {
8816 fields: BTreeMap<String, String>,
8817 }
8818
8819 impl Visit for EventFieldVisitor {
8820 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8821 self.fields
8822 .insert(field.name().to_string(), format!("{value:?}"));
8823 }
8824 }
8825
8826 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8827 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8828 status,
8829 detail: Some("warming".to_string()),
8830 metrics: Some(json!({"queue_depth": 3})),
8831 })
8832 .unwrap();
8833 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8834 }
8835
8836 fn route_bind_ack(corr: u64) -> Frame {
8837 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8838 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8839 }
8840
8841 fn unique_project_root(label: &str) -> ScratchDir {
8842 ScratchDir::new(label)
8843 }
8844
8845 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8846 match parse_route_poll(frame) {
8847 ClientControlResponse::RoutePoll {
8848 status: None,
8849 live: Some(live),
8850 ..
8851 } => assert_eq!(live, expected_live),
8852 other => panic!("unexpected route.poll response: {other:?}"),
8853 }
8854 }
8855
8856 fn bind_liveness_route(
8857 registry: &Registry,
8858 forwarding: &ForwardingTable,
8859 module_id: &str,
8860 ) -> (RouteCtx, u16, u32) {
8861 let module_connection = ConnectionId::new(101);
8862 let client_connection = ConnectionId::new(202);
8863 let registration = registry
8864 .register_with_control_ops(
8865 manifest(module_id, PROTOCOL_VERSION),
8866 PROTOCOL_VERSION,
8867 module_connection,
8868 module_baseline_control_ops(),
8869 )
8870 .unwrap();
8871 let (module_tx, _module_rx) = mpsc::channel(8);
8872 let endpoint = forwarding
8873 .register_module_connection(
8874 module_connection,
8875 module_id.to_string(),
8876 PROTOCOL_VERSION,
8877 manifest_concurrency(®istration.manifest),
8878 FrameSink::new(module_tx),
8879 )
8880 .unwrap();
8881 let (client_ctx, _client_rx) = route_ctx(client_connection);
8882 let pending = forwarding
8883 .begin_route_bind_relay_for_test(
8884 client_connection,
8885 client_ctx.egress.clone(),
8886 1,
8887 module_id,
8888 )
8889 .unwrap();
8890 assert_eq!(pending.endpoint, endpoint);
8891 let route_channel = pending.client_channel;
8892 let route_epoch = pending.client_epoch;
8893 forwarding
8894 .complete_pending_relay(
8895 module_connection,
8896 pending.corr,
8897 RouteBindRelayOutcome::Accepted,
8898 )
8899 .unwrap();
8900 (client_ctx, route_channel, route_epoch)
8901 }
8902
8903 struct FakeProcessLiveness {
8904 live: Option<bool>,
8905 }
8906
8907 impl ModuleProcessLiveness for FakeProcessLiveness {
8908 fn process_live(&self, _module_id: &str) -> Option<bool> {
8909 self.live
8910 }
8911 }
8912
8913 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8914 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8915 {
8916 let registry = Arc::new(Registry::default());
8917 let supervisor_handle = SupervisorHandle::new();
8918 let supervisor = Supervisor::new_for_test(
8919 Arc::clone(®istry),
8920 RestartPolicy::new(1, Duration::from_millis(10)),
8921 )
8922 .with_handle(supervisor_handle.clone());
8923 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8924 let module = supervisor
8925 .spawn(ModuleSpec {
8926 module_id: "stderr-tail-wire".to_string(),
8927 program: fake_aft_stub_path(),
8928 args: Vec::new(),
8929 env: vec![
8930 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8931 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8932 ],
8933 reserved: false,
8934 reserved_prefixes: Vec::new(),
8935 protocol: ModuleProtocol::Subc,
8936 overlap: Default::default(),
8937 })
8938 .unwrap();
8939
8940 let deadline = Instant::now() + Duration::from_secs(5);
8941 loop {
8942 let tail = module.stderr_tail(None, None);
8943 if tail
8944 .entries
8945 .iter()
8946 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8947 && tail.entries.iter().any(|entry| {
8948 matches!(
8949 entry,
8950 TailEntry::Line {
8951 truncated: true,
8952 ..
8953 }
8954 )
8955 })
8956 {
8957 break;
8958 }
8959 assert!(
8960 Instant::now() < deadline,
8961 "module did not produce a truncated line and restart boundary: {tail:?}"
8962 );
8963 sleep(Duration::from_millis(10)).await;
8964 }
8965
8966 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8967 let request = ClientControlRequest::SupervisorStderrTail {
8968 module_id: "stderr-tail-wire".to_string(),
8969 max_lines: None,
8970 max_bytes: None,
8971 };
8972 let frame = Frame::build(
8973 FrameType::Request,
8974 control_flags(),
8975 0,
8976 0,
8977 1,
8978 serde_json::to_vec(&request).unwrap(),
8979 )
8980 .unwrap();
8981 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8982 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8983 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8984 serde_json::from_slice(&responses[0].body).unwrap()
8985 else {
8986 panic!("expected supervisor.stderr_tail response");
8987 };
8988
8989 assert!(
8990 tail.entries
8991 .iter()
8992 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8993 "the control response lost the restart boundary"
8994 );
8995 let Some(StderrTailEntry::Line {
8996 text,
8997 truncated,
8998 at_ms,
8999 }) = tail.entries.iter().find(|entry| {
9000 matches!(
9001 entry,
9002 StderrTailEntry::Line {
9003 truncated: true,
9004 ..
9005 }
9006 )
9007 })
9008 else {
9009 panic!("the control response lost the truncated line");
9010 };
9011 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9012 assert!(*truncated);
9013 assert!(
9014 at_ms.is_some(),
9015 "the control response lost the line's capture time"
9016 );
9017 }
9018
9019 #[tokio::test(flavor = "current_thread")]
9024 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9025 let dir = ScratchDir::new("terminals-off-worker");
9026 let journal_path = dir.join("terminals.jsonl");
9027 let registry = Arc::new(Registry::default());
9028 let supervisor_handle = SupervisorHandle::new();
9029 let supervisor =
9030 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9031 .with_handle(supervisor_handle.clone())
9032 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9033 let module = supervisor
9034 .spawn(ModuleSpec {
9035 module_id: "terminal-off-worker".to_string(),
9036 program: fake_aft_stub_path(),
9037 args: Vec::new(),
9038 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9039 reserved: false,
9040 reserved_prefixes: Vec::new(),
9041 protocol: ModuleProtocol::Subc,
9042 overlap: Default::default(),
9043 })
9044 .unwrap();
9045 let deadline = Instant::now() + Duration::from_secs(5);
9046 while module.terminal_history().entries.len() != 2 {
9047 assert!(Instant::now() < deadline, "module did not record two exits");
9048 sleep(Duration::from_millis(10)).await;
9049 }
9050
9051 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9052 let handler =
9053 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
9054 let frame = Frame::build(
9055 FrameType::Request,
9056 control_flags(),
9057 0,
9058 0,
9059 1,
9060 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9061 module_id: "terminal-off-worker".to_string(),
9062 })
9063 .unwrap(),
9064 )
9065 .unwrap();
9066 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9067 let spawned_at = std::time::Instant::now();
9068 let read = tokio::spawn({
9069 let handler = Arc::clone(&handler);
9070 async move { handler.handle_control_frame(&ctx, frame).await }
9071 });
9072 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9075 .await
9076 .unwrap()
9077 .expect("the history read reached its pause");
9078 let elapsed = spawned_at.elapsed();
9079 assert!(
9080 elapsed < Duration::from_secs(2) && !read.is_finished(),
9081 "this task could not run while the history read was paused \
9082 (resumed after {elapsed:?}, read finished: {})",
9083 read.is_finished()
9084 );
9085
9086 drop(release);
9087 let responses = read.await.unwrap().unwrap();
9088 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9089 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9090 panic!("expected supervisor.terminals response");
9091 };
9092 assert_eq!(terminals.entries.len(), 2);
9093 assert_eq!(terminals.journal_skipped_lines, 0);
9094 assert_eq!(terminals.journal_read_errors, 0);
9095 }
9096
9097 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9098 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9099 let registry = Arc::new(Registry::default());
9100 let supervisor_handle = SupervisorHandle::new();
9101 let supervisor =
9102 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9103 .with_handle(supervisor_handle.clone());
9104 let module = supervisor
9105 .spawn(ModuleSpec {
9106 module_id: "terminal-golden".to_string(),
9107 program: fake_aft_stub_path(),
9108 args: Vec::new(),
9109 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9110 reserved: false,
9111 reserved_prefixes: Vec::new(),
9112 protocol: ModuleProtocol::Subc,
9113 overlap: Default::default(),
9114 })
9115 .unwrap();
9116
9117 let deadline = Instant::now() + Duration::from_secs(5);
9118 while module.terminal_history().entries.len() != 2 {
9119 assert!(
9120 Instant::now() < deadline,
9121 "module did not retain two terminal exits: {:?}",
9122 module.terminal_history()
9123 );
9124 sleep(Duration::from_millis(10)).await;
9125 }
9126
9127 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9128 let request = ClientControlRequest::SupervisorTerminals {
9129 module_id: "terminal-golden".to_string(),
9130 };
9131 let frame = Frame::build(
9132 FrameType::Request,
9133 control_flags(),
9134 0,
9135 0,
9136 1,
9137 serde_json::to_vec(&request).unwrap(),
9138 )
9139 .unwrap();
9140 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9141 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9142 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9143 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9144 panic!("expected supervisor.terminals response");
9145 };
9146 assert_eq!(terminals.entries.len(), 2);
9147 assert_eq!(terminals.dropped, 0);
9148
9149 let mut rendered = serde_json::to_value(response).unwrap();
9150 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9153 for (index, entry) in rendered["entries"]
9154 .as_array_mut()
9155 .expect("terminal response entries array")
9156 .iter_mut()
9157 .enumerate()
9158 {
9159 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9160 }
9161
9162 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9163 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9164 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9165 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9166 std::fs::write(&golden_path, &serialized).unwrap();
9167 }
9168 let expected: Value =
9169 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9170 assert_eq!(rendered, expected);
9171 }
9172
9173 #[test]
9174 fn hello_registers_manifest_and_returns_ack() {
9175 let registry = Arc::new(Registry::default());
9176 let handler = ControlHandler::new(Arc::clone(®istry));
9177 let conn = ConnectionId::new(1);
9178
9179 let responses = handler
9180 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9181 .unwrap();
9182
9183 assert_eq!(responses.len(), 1);
9184 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9185 assert_eq!(responses[0].header.channel, 0);
9186 assert_eq!(responses[0].header.corr, 7);
9187 let ack = parse_ack(&responses[0]);
9188 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9189 assert!(ack
9190 .subc_capabilities
9191 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9192 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9193 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9194 assert!(ack
9195 .subc_ops
9196 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9197 assert!(ack
9198 .subc_ops
9199 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9200
9201 let registration = registry.get_module("aft").unwrap().unwrap();
9202 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9203 assert_eq!(registration.state, ChannelState::Active);
9204 assert_eq!(registration.connection_id, conn);
9205 assert_eq!(registration.control_ops, module_baseline_control_ops());
9206 }
9207
9208 #[test]
9209 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9210 let invalid_identifiers = [
9211 ("case_change", "credentials-Provider/v1"),
9212 ("leading_zero", "credentials-provider/v01"),
9213 ("trailing_hyphen", "credentials-provider-/v1"),
9214 ("consecutive_hyphens", "credentials--provider/v1"),
9215 ("uppercase", "Credentials-provider/v1"),
9216 ("missing_v", "credentials-provider/1"),
9217 ("whitespace", "credentials provider/v1"),
9218 ("zero_version", "credentials-provider/v0"),
9219 ("out_of_range_version", "credentials-provider/v4294967296"),
9220 (
9221 "overlength_name",
9222 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9223 ),
9224 ];
9225 let mut cases = invalid_identifiers
9226 .into_iter()
9227 .map(|(name, identifier)| {
9228 (
9229 format!("identifier_{name}"),
9230 "capabilities.provides[0]".to_string(),
9231 identifier.to_string(),
9232 json!({ "provides": [identifier] }),
9233 None,
9234 )
9235 })
9236 .collect::<Vec<_>>();
9237 cases.extend([
9238 (
9239 "unknown_need".to_string(),
9240 "capabilities.requires[0].need".to_string(),
9241 "deferred".to_string(),
9242 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9243 None,
9244 ),
9245 (
9246 "duplicate_provides".to_string(),
9247 "capabilities.provides[1]".to_string(),
9248 "credentials-provider/v1".to_string(),
9249 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9250 None,
9251 ),
9252 (
9253 "duplicate_must_never_reach".to_string(),
9254 "capabilities.must_never_reach[1]".to_string(),
9255 "credentials-provider/v1".to_string(),
9256 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9257 None,
9258 ),
9259 (
9260 "duplicate_requires_same_need".to_string(),
9261 "capabilities.requires[1]".to_string(),
9262 "credentials-provider/v1".to_string(),
9263 json!({ "requires": [
9264 { "capability": "credentials-provider/v1", "need": "required" },
9265 { "capability": "credentials-provider/v1", "need": "required" }
9266 ] }),
9267 None,
9268 ),
9269 (
9270 "duplicate_requires_conflicting_need".to_string(),
9271 "capabilities.requires[1]".to_string(),
9272 "credentials-provider/v1".to_string(),
9273 json!({ "requires": [
9274 { "capability": "credentials-provider/v1", "need": "required" },
9275 { "capability": "credentials-provider/v1", "need": "optional" }
9276 ] }),
9277 None,
9278 ),
9279 (
9280 "capabilities_root_pointer".to_string(),
9281 "runtime_computed[0]".to_string(),
9282 "/capabilities".to_string(),
9283 json!({}),
9284 Some(json!(["/capabilities"])),
9285 ),
9286 (
9287 "capabilities_descendant_pointer".to_string(),
9288 "runtime_computed[0]".to_string(),
9289 "/capabilities/provides".to_string(),
9290 json!({}),
9291 Some(json!(["/capabilities/provides"])),
9292 ),
9293 (
9294 "malformed_pointer_without_leading_slash".to_string(),
9295 "runtime_computed[0]".to_string(),
9296 "capabilities".to_string(),
9297 json!({}),
9298 Some(json!(["capabilities"])),
9299 ),
9300 (
9301 "malformed_pointer_escape".to_string(),
9302 "runtime_computed[0]".to_string(),
9303 "/roles/~2/tools".to_string(),
9304 json!({}),
9305 Some(json!(["/roles/~2/tools"])),
9306 ),
9307 (
9308 "unknown_capabilities_field".to_string(),
9309 "capabilities.future".to_string(),
9310 "<array>".to_string(),
9311 json!({ "future": [] }),
9312 None,
9313 ),
9314 ]);
9315
9316 for (index, (name, field, value, capabilities, runtime_computed)) in
9317 cases.into_iter().enumerate()
9318 {
9319 let registry = Arc::new(Registry::default());
9320 let handler = ControlHandler::new(Arc::clone(®istry));
9321 let response = handler
9322 .handle_control(
9323 ConnectionId::new((index + 1) as u64),
9324 capability_grammar_hello_frame(
9325 capabilities,
9326 runtime_computed,
9327 index as u64 + 1,
9328 ),
9329 )
9330 .expect("invalid HELLO returns a refusal");
9331
9332 assert_eq!(response.len(), 1, "{name} must emit one refusal");
9333 let error = parse_error(&response[0]);
9334 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9335 let message = error["message"]
9336 .as_str()
9337 .expect("error message is a string");
9338 assert!(
9339 message.contains(&field),
9340 "{name}: field missing from {message}"
9341 );
9342 assert!(
9343 message.contains(&value),
9344 "{name}: value missing from {message}"
9345 );
9346 assert_eq!(
9347 registry
9348 .active_registration_count()
9349 .expect("registry reads"),
9350 0,
9351 "{name}: refused HELLO must not create a catalog entry"
9352 );
9353 }
9354 }
9355
9356 #[test]
9357 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9358 let registry = Arc::new(Registry::default());
9359 let handler = ControlHandler::new(Arc::clone(®istry));
9360 let capabilities = json!({
9361 "provides": ["credentials-provider/v1"],
9362 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9363 "must_never_reach": ["federation-transport/v1"]
9364 });
9365 let response = handler
9366 .handle_control(
9367 ConnectionId::new(99),
9368 capability_grammar_hello_frame(
9369 capabilities.clone(),
9370 Some(json!(["/roles/0/tools"])),
9371 99,
9372 ),
9373 )
9374 .expect("valid HELLO registers");
9375 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9376
9377 let request = Frame::build(
9378 FrameType::Request,
9379 control_flags(),
9380 0,
9381 0,
9382 100,
9383 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9384 .expect("catalog request serializes"),
9385 )
9386 .expect("catalog request frame builds");
9387 let response = handler
9388 .handle_catalog_list(request, None)
9389 .expect("catalog list succeeds");
9390 let ClientControlResponse::CatalogList { modules, .. } =
9391 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9392 else {
9393 panic!("catalog request must return catalog.list");
9394 };
9395 assert_eq!(modules.len(), 1);
9396 assert_eq!(
9397 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9398 capabilities
9399 );
9400 }
9401
9402 #[test]
9403 fn catalog_list_mirrors_management_operation_description() {
9404 let registry = Arc::new(Registry::default());
9405 let handler = ControlHandler::new(Arc::clone(®istry));
9406 let description = "List managed records and return their identifiers and metadata.";
9407 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9408 manifest.provides = vec![ProviderRole::ManagementSurface {
9409 operations: vec![ManagementOperation {
9410 name: "records.list".to_string(),
9411 kind: ManagementOperationKind::Query,
9412 description: Some(description.to_string()),
9413 }],
9414 config_schema: json!({"type": "object"}),
9415 observability: vec![ObservabilitySurface {
9416 name: "records.stats".to_string(),
9417 kind: ObservabilityKind::Snapshot,
9418 }],
9419 identity_scope: vec![IdentityScope::Project],
9420 concurrency: Concurrency::ModuleManaged,
9421 }];
9422 registry
9423 .register_with_control_ops(
9424 manifest,
9425 PROTOCOL_VERSION,
9426 ConnectionId::new(99),
9427 Vec::new(),
9428 )
9429 .expect("described management manifest registers");
9430
9431 let request = Frame::build(
9432 FrameType::Request,
9433 control_flags(),
9434 0,
9435 0,
9436 100,
9437 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9438 .expect("catalog request serializes"),
9439 )
9440 .expect("catalog request frame builds");
9441 let response = handler
9442 .handle_catalog_list(request, None)
9443 .expect("catalog list succeeds");
9444 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9445 assert_eq!(
9446 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9447 "catalog.list must preserve the declared operation description verbatim"
9448 );
9449 }
9450
9451 #[test]
9452 fn catalog_list_mirrors_registered_operation_delivery() {
9453 let registry = Arc::new(Registry::default());
9454 let handler = ControlHandler::new(Arc::clone(®istry));
9455 let delivery =
9456 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9457 let mut body = serde_json::to_value(ModuleHelloBody {
9458 manifest: manifest("watch-management", PROTOCOL_VERSION),
9459 protocol_ver: PROTOCOL_VERSION,
9460 control_ops: None,
9461 launch_nonce: None,
9462 })
9463 .unwrap();
9464 body["manifest"]["provides"] = json!([{
9465 "role": "management_surface",
9466 "operations": [
9467 {"name": "status", "kind": "query"},
9468 {"name": "future", "kind": "query"},
9469 {"name": "reset", "kind": "mutate"}
9470 ],
9471 "config_schema": {}, "observability": [], "identity_scope": []
9472 }]);
9473 body["manifest"]["operation_delivery"] = delivery.clone();
9474 let hello = Frame::build(
9475 FrameType::Hello,
9476 control_flags(),
9477 0,
9478 0,
9479 99,
9480 serde_json::to_vec(&body).unwrap(),
9481 )
9482 .unwrap();
9483 let response = handler
9484 .handle_control(ConnectionId::new(99), hello)
9485 .unwrap();
9486 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9487
9488 let (_, registrations) = registry.list_modules().unwrap();
9489 let registered = ®istrations[0].manifest;
9490 assert!(registered.is_watch_operation("status"));
9491 assert!(!registered.is_watch_operation("future"));
9492 assert!(!registered.is_watch_operation("reset"));
9493 assert!(!registered.is_watch_operation("missing"));
9494
9495 let request = Frame::build(
9496 FrameType::Request,
9497 control_flags(),
9498 0,
9499 0,
9500 100,
9501 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9502 )
9503 .unwrap();
9504 let response = handler.handle_catalog_list(request, None).unwrap();
9505 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9506 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9507 let ClientControlResponse::CatalogList { modules, .. } =
9508 serde_json::from_slice(&response[0].body).unwrap()
9509 else {
9510 panic!("expected catalog.list");
9511 };
9512 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9513 }
9514
9515 #[test]
9516 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9517 let registry = Arc::new(Registry::default());
9518 let supervisor = SupervisorHandle::new();
9519 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9520 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9521 let descriptor = json!({
9522 "schema": "ckios.richui.manifest/1",
9523 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9524 "bytes": 4096,
9525 "chunks": 4
9526 });
9527 let connection = ConnectionId::new(201);
9528 let hello = handler
9529 .handle_control(
9530 connection,
9531 hello_frame_with_rich_ui(
9532 "vault",
9533 PROTOCOL_VERSION,
9534 201,
9535 Some("the-real-nonce"),
9536 descriptor.clone(),
9537 ),
9538 )
9539 .unwrap();
9540 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9541
9542 let request =
9543 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9544 let provides = registry
9545 .get_module_by_connection(connection)
9546 .unwrap()
9547 .unwrap()
9548 .manifest
9549 .provides;
9550 handler
9551 .handle_catalog_update(connection, request, provides, None, Some(false))
9552 .unwrap();
9553
9554 let request = Frame::build(
9555 FrameType::Request,
9556 control_flags(),
9557 0,
9558 0,
9559 203,
9560 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9561 )
9562 .unwrap();
9563 let response = handler.handle_catalog_list(request, None).unwrap();
9564 let ClientControlResponse::CatalogList { modules, .. } =
9565 serde_json::from_slice(&response[0].body).unwrap()
9566 else {
9567 panic!("expected catalog.list");
9568 };
9569 assert_eq!(
9570 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9571 descriptor
9572 );
9573 assert!(modules[0].nonce_verified);
9574 }
9575
9576 fn hello_with_role_declarations(declarations: Value) -> Frame {
9577 let mut body = serde_json::to_value(ModuleHelloBody {
9578 manifest: manifest("role-test", PROTOCOL_VERSION),
9579 protocol_ver: PROTOCOL_VERSION,
9580 control_ops: None,
9581 launch_nonce: None,
9582 })
9583 .unwrap();
9584 body["manifest"]["role_declarations"] = declarations;
9585 Frame::build(
9586 FrameType::Hello,
9587 control_flags(),
9588 0,
9589 0,
9590 301,
9591 serde_json::to_vec(&body).unwrap(),
9592 )
9593 .unwrap()
9594 }
9595
9596 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9597 let request = Frame::build(
9598 FrameType::Request,
9599 control_flags(),
9600 0,
9601 0,
9602 302,
9603 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9604 )
9605 .unwrap();
9606 let response = handler.handle_catalog_list(request, None).unwrap();
9607 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9608 let ClientControlResponse::CatalogList { mut modules, .. } =
9609 serde_json::from_slice(&response[0].body).unwrap()
9610 else {
9611 panic!("expected catalog.list");
9612 };
9613 (raw["modules"][0].clone(), modules.remove(0))
9614 }
9615
9616 #[test]
9617 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9618 let registry = Arc::new(Registry::default());
9619 let handler = ControlHandler::new(Arc::clone(®istry));
9620 let declarations = json!({
9621 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9622 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9623 });
9624 let connection = ConnectionId::new(301);
9625 let response = handler
9626 .handle_control(
9627 connection,
9628 hello_with_role_declarations(declarations.clone()),
9629 )
9630 .unwrap();
9631 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9632 let registered = registry
9633 .get_module_by_connection(connection)
9634 .unwrap()
9635 .unwrap()
9636 .manifest;
9637 assert_eq!(
9641 serde_json::to_value(®istered.role_declarations).unwrap(),
9642 declarations
9643 );
9644 let (raw, entry) = role_declarations_catalog(&handler);
9645 assert_eq!(raw["role_declarations"], declarations);
9646 assert_eq!(entry.role_declarations, registered.role_declarations);
9647
9648 let request =
9649 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
9650 handler
9651 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
9652 .unwrap();
9653 let (raw, entry) = role_declarations_catalog(&handler);
9654 assert_eq!(raw["role_declarations"], declarations);
9655 assert_eq!(entry.role_declarations, registered.role_declarations);
9656 }
9657
9658 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
9659 let registry = Arc::new(Registry::default());
9660 let handler = ControlHandler::new(Arc::clone(®istry));
9661 let capture = EventCapture::default();
9662 let _subscriber =
9663 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9664 let response = handler
9665 .handle_control(
9666 ConnectionId::new(301),
9667 hello_with_role_declarations(declarations),
9668 )
9669 .unwrap();
9670 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9671 assert!(registry
9672 .get_module_by_connection(ConnectionId::new(301))
9673 .unwrap()
9674 .unwrap()
9675 .manifest
9676 .role_declarations
9677 .is_empty());
9678 let (raw, entry) = role_declarations_catalog(&handler);
9679 assert!(raw.get("role_declarations").is_none());
9680 assert!(entry.role_declarations.is_empty());
9681 let warnings: Vec<_> = capture
9682 .events()
9683 .into_iter()
9684 .filter(|event| {
9685 event.fields.get("message").map(String::as_str)
9686 == Some("dropping whole role_declarations map from HELLO")
9687 })
9688 .collect();
9689 assert_eq!(warnings.len(), 1);
9690 assert_eq!(warnings[0].level, tracing::Level::WARN);
9691 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
9692 assert_eq!(warnings[0].fields["reason"], reason);
9693 }
9694
9695 #[test]
9696 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
9697 for value in [
9698 json!(null),
9699 json!(false),
9700 json!(0),
9701 json!("opaque"),
9702 json!([]),
9703 ] {
9704 assert_hello_drops_role_declarations_with_warning(
9705 json!({"routing/v1": {}, "broken/v1": value}),
9706 "role_declarations values must be objects",
9707 );
9708 }
9709 assert_hello_drops_role_declarations_with_warning(
9710 json!([]),
9711 "role_declarations must be an object",
9712 );
9713 }
9714
9715 #[test]
9716 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
9717 assert_hello_drops_role_declarations_with_warning(
9718 json!({"routing/v1": {}, "bad key": {}}),
9719 "role_declarations key must use capability-name grammar",
9720 );
9721 }
9722
9723 #[test]
9724 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
9725 assert_hello_drops_role_declarations_with_warning(
9726 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
9727 "role_declarations exceeds 16384 compact JSON bytes",
9728 );
9729 }
9730
9731 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
9732 let mut body = serde_json::to_value(ModuleHelloBody {
9733 manifest: manifest("flow-ops", PROTOCOL_VERSION),
9734 protocol_ver: PROTOCOL_VERSION,
9735 control_ops: None,
9736 launch_nonce: None,
9737 })
9738 .unwrap();
9739 body["manifest"]["provides"] = json!([{
9740 "role": "management_surface",
9741 "operations": [
9742 {"name": "diagnostics.findings", "kind": "query"},
9743 {"name": "reset", "kind": "mutate"}
9744 ],
9745 "config_schema": {}, "observability": [], "identity_scope": []
9746 }]);
9747 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
9748 if let Some(flow_operations) = flow_operations {
9749 body["manifest"]["flow_operations"] = flow_operations;
9750 }
9751 Frame::build(
9752 FrameType::Hello,
9753 control_flags(),
9754 0,
9755 0,
9756 401,
9757 serde_json::to_vec(&body).unwrap(),
9758 )
9759 .unwrap()
9760 }
9761
9762 #[test]
9763 fn catalog_list_mirrors_flow_operations_unchanged() {
9764 for (capability, flow_operations) in [
9767 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
9768 ("call-modes/v1", json!(["diagnostics.findings"])),
9769 ("flow-scopes/v1", json!([])),
9770 ] {
9771 let registry = Arc::new(Registry::default());
9772 let handler = ControlHandler::new(Arc::clone(®istry));
9773 let connection = ConnectionId::new(401);
9774 let response = handler
9775 .handle_control(
9776 connection,
9777 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
9778 )
9779 .unwrap();
9780 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9781 let registered = registry
9782 .get_module_by_connection(connection)
9783 .unwrap()
9784 .unwrap()
9785 .manifest;
9786 assert_eq!(
9787 serde_json::to_value(®istered.flow_operations).unwrap(),
9788 flow_operations
9789 );
9790 let (raw, entry) = role_declarations_catalog(&handler);
9791 assert_eq!(raw["flow_operations"], flow_operations);
9792 assert_eq!(entry.flow_operations, registered.flow_operations);
9793 }
9794
9795 let registry = Arc::new(Registry::default());
9796 let handler = ControlHandler::new(Arc::clone(®istry));
9797 let response = handler
9798 .handle_control(
9799 ConnectionId::new(401),
9800 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
9801 )
9802 .unwrap();
9803 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9804 let (raw, entry) = role_declarations_catalog(&handler);
9805 assert!(raw.get("flow_operations").is_none());
9806 assert_eq!(entry.flow_operations, None);
9807 }
9808
9809 fn assert_hello_drops_flow_operations_with_warning(
9810 capabilities: Value,
9811 flow_operations: Value,
9812 reason: &str,
9813 ) {
9814 let registry = Arc::new(Registry::default());
9815 let handler = ControlHandler::new(Arc::clone(®istry));
9816 let capture = EventCapture::default();
9817 let _subscriber =
9818 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9819 let response = handler
9820 .handle_control(
9821 ConnectionId::new(401),
9822 hello_with_flow_operations(capabilities, Some(flow_operations)),
9823 )
9824 .unwrap();
9825 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9826 let registered = registry
9827 .get_module_by_connection(ConnectionId::new(401))
9828 .unwrap()
9829 .expect("the module still registers")
9830 .manifest;
9831 assert_eq!(registered.module_id, "flow-ops");
9832 assert_eq!(registered.flow_operations, None);
9833 let (raw, entry) = role_declarations_catalog(&handler);
9834 assert_eq!(raw["module_id"], "flow-ops");
9835 assert!(raw.get("flow_operations").is_none());
9836 assert_eq!(entry.flow_operations, None);
9837 let warnings: Vec<_> = capture
9838 .events()
9839 .into_iter()
9840 .filter(|event| {
9841 event.fields.get("message").map(String::as_str)
9842 == Some("dropping flow_operations from HELLO")
9843 })
9844 .collect();
9845 assert_eq!(warnings.len(), 1);
9846 assert_eq!(warnings[0].level, tracing::Level::WARN);
9847 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
9848 assert_eq!(warnings[0].fields["reason"], reason);
9849 }
9850
9851 #[test]
9852 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
9853 assert_hello_drops_flow_operations_with_warning(
9854 json!(["flow-scopes/v1"]),
9855 json!(["diagnostics.findings", "missing"]),
9856 "flow_operations names 'missing', which is not a declared management operation",
9857 );
9858 }
9859
9860 #[test]
9861 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
9862 assert_hello_drops_flow_operations_with_warning(
9863 json!([]),
9864 json!(["diagnostics.findings"]),
9865 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
9866 );
9867 assert_hello_drops_flow_operations_with_warning(
9869 json!(["agent-run-scopes/v1"]),
9870 json!([]),
9871 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
9872 );
9873 }
9874
9875 #[test]
9876 fn hello_malformed_flow_operations_is_dropped_with_warning() {
9877 assert_hello_drops_flow_operations_with_warning(
9878 json!(["flow-scopes/v1"]),
9879 json!(["reset", "reset"]),
9880 "flow_operations names 'reset' more than once",
9881 );
9882 assert_hello_drops_flow_operations_with_warning(
9883 json!(["flow-scopes/v1"]),
9884 json!(vec!["reset"; 257]),
9885 "flow_operations lists 257 names, more than 256",
9886 );
9887 assert_hello_drops_flow_operations_with_warning(
9888 json!(["flow-scopes/v1"]),
9889 json!("reset"),
9890 "flow_operations must be an array of operation names",
9891 );
9892 }
9893
9894 #[test]
9895 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
9896 let registry = Arc::new(Registry::default());
9897 let handler = ControlHandler::new(registry.clone());
9898 let descriptor = json!({
9899 "schema": "ckios.richui.manifest/1",
9900 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9901 "bytes": 128,
9902 "chunks": 2
9903 });
9904 let malformed = [
9905 ("schema", json!("")),
9906 ("schema", json!("a".repeat(65))),
9907 ("schema", json!("Bad schema")),
9908 (
9909 "manifest_digest",
9910 json!(format!("sha256:{}", "A".repeat(64))),
9911 ),
9912 ("manifest_digest", json!("sha256:abc")),
9913 ("bytes", json!(0)),
9914 ("bytes", json!(16_777_217)),
9915 ("bytes", json!("128")),
9916 ("chunks", json!(0)),
9917 ("chunks", json!(1025)),
9918 ("chunks", json!("2")),
9919 ];
9920 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
9921 let mut invalid = descriptor.clone();
9922 invalid[field] = invalid_value;
9923 let response = handler
9924 .handle_control(
9925 ConnectionId::new(210 + index as u64),
9926 hello_frame_with_rich_ui(
9927 &format!("bad-rich-ui-{index}"),
9928 PROTOCOL_VERSION,
9929 210 + index as u64,
9930 None,
9931 invalid,
9932 ),
9933 )
9934 .unwrap();
9935 assert_eq!(response[0].header.ty, FrameType::Error);
9936 let error = parse_error(&response[0]);
9937 assert_eq!(error["code"], "invalid_manifest");
9938 assert!(error["message"]
9939 .as_str()
9940 .unwrap()
9941 .contains(&format!("rich_ui.{field}")));
9942 assert!(registry
9943 .get_module(&format!("bad-rich-ui-{index}"))
9944 .unwrap()
9945 .is_none());
9946 }
9947 }
9948
9949 #[test]
9950 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
9951 let registry = Arc::new(Registry::default());
9952 let handler = ControlHandler::new(Arc::clone(®istry));
9953 let connection = ConnectionId::new(219);
9954 registry
9955 .register_with_control_ops(
9956 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
9957 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
9958 "bad schema",
9959 format!("sha256:{}", "a".repeat(64)),
9960 128,
9961 2,
9962 )))
9963 .build(),
9964 PROTOCOL_VERSION,
9965 connection,
9966 Vec::new(),
9967 )
9968 .unwrap();
9969 let request =
9970 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
9971 let response = handler
9972 .handle_catalog_update(connection, request, Vec::new(), None, None)
9973 .unwrap();
9974 assert_eq!(response[0].header.ty, FrameType::Error);
9975 let error = parse_error(&response[0]);
9976 assert_eq!(error["code"], "invalid_manifest");
9977 assert!(error["message"]
9978 .as_str()
9979 .unwrap()
9980 .contains("rich_ui.schema"));
9981 }
9982
9983 #[test]
9984 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
9985 let registry = Arc::new(Registry::default());
9986 let supervisor = SupervisorHandle::new();
9987 supervisor.open_swap("provider", "candidate-nonce".to_string());
9988 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9989 let connection = ConnectionId::new(218);
9990 let hello = handler
9991 .handle_control(
9992 connection,
9993 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
9994 )
9995 .unwrap();
9996 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9997 assert!(registry.list_modules().unwrap().1.is_empty());
9998
9999 registry.promote_candidate("provider").unwrap().unwrap();
10000 let request = Frame::build(
10001 FrameType::Request,
10002 control_flags(),
10003 0,
10004 0,
10005 219,
10006 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10007 )
10008 .unwrap();
10009 let response = handler.handle_catalog_list(request, None).unwrap();
10010 let ClientControlResponse::CatalogList { modules, .. } =
10011 serde_json::from_slice(&response[0].body).unwrap()
10012 else {
10013 panic!("expected catalog.list");
10014 };
10015 assert_eq!(modules.len(), 1);
10016 assert!(modules[0].nonce_verified);
10017 }
10018
10019 #[test]
10020 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
10021 let registry = Arc::new(Registry::default());
10022 let handler = ControlHandler::new(Arc::clone(®istry));
10023 let hello = Frame::build(
10024 FrameType::Hello,
10025 control_flags(),
10026 0,
10027 0,
10028 220,
10029 serde_json::to_vec(&ModuleHelloBody {
10030 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
10031 protocol_ver: PROTOCOL_VERSION,
10032 control_ops: None,
10033 launch_nonce: Some("unverified".to_string()),
10034 })
10035 .unwrap(),
10036 )
10037 .unwrap();
10038 assert_eq!(
10039 handler
10040 .handle_control(ConnectionId::new(220), hello)
10041 .unwrap()[0]
10042 .header
10043 .ty,
10044 FrameType::HelloAck
10045 );
10046 let request = Frame::build(
10047 FrameType::Request,
10048 control_flags(),
10049 0,
10050 0,
10051 221,
10052 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10053 )
10054 .unwrap();
10055 let response = handler.handle_catalog_list(request, None).unwrap();
10056 let ClientControlResponse::CatalogList { modules, .. } =
10057 serde_json::from_slice(&response[0].body).unwrap()
10058 else {
10059 panic!("expected catalog.list");
10060 };
10061 assert_eq!(
10062 serde_json::to_vec(&modules[0]).unwrap(),
10063 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
10064 );
10065 assert!(!modules[0].nonce_verified);
10066 }
10067
10068 #[test]
10069 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
10070 let registry = Arc::new(Registry::default());
10071 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10072 [("vault".to_string(), true), ("squatter".to_string(), true)],
10073 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10074 );
10075 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
10076 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10077 provides: vec!["credentials-provider/v1".to_string()],
10078 requires: Vec::new(),
10079 must_never_reach: Vec::new(),
10080 });
10081 let frame = Frame::build(
10082 FrameType::Hello,
10083 control_flags(),
10084 0,
10085 0,
10086 77,
10087 serde_json::to_vec(&ModuleHelloBody {
10088 manifest: squatter,
10089 protocol_ver: PROTOCOL_VERSION,
10090 control_ops: None,
10091 launch_nonce: None,
10092 })
10093 .expect("HELLO serializes"),
10094 )
10095 .expect("HELLO frame builds");
10096 let response = handler
10097 .handle_control(ConnectionId::new(77), frame)
10098 .expect("reserved claim receives a typed refusal");
10099 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10100 assert_eq!(
10101 registry
10102 .active_registration_count()
10103 .expect("registry reads"),
10104 0,
10105 "a reserved capability refusal must not leave a catalog entry"
10106 );
10107 }
10108
10109 #[test]
10110 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10111 for settlement in ["timeout", "inconclusive", "drop"] {
10112 let breakers = RouteBindBreakers::default();
10113 let RouteBindAdmission::Admitted {
10114 guard: mut old,
10115 probe: false,
10116 } = breakers.admit("prov")
10117 else {
10118 panic!("ordinary relay admitted")
10119 };
10120 let RouteBindAdmission::Admitted {
10121 guard: mut opener, ..
10122 } = breakers.admit("prov")
10123 else {
10124 panic!("second relay admitted")
10125 };
10126 assert!(
10127 !opener
10128 .record_timeout(1, Duration::ZERO)
10129 .unwrap()
10130 .reopened_after_probe
10131 );
10132 let RouteBindAdmission::Admitted {
10133 guard: mut probe,
10134 probe: true,
10135 } = breakers.admit("prov")
10136 else {
10137 panic!("one cooldown probe admitted")
10138 };
10139 match settlement {
10140 "timeout" => assert!(
10141 !old.record_timeout(1, Duration::ZERO)
10142 .unwrap()
10143 .reopened_after_probe
10144 ),
10145 "inconclusive" => old.record_inconclusive(),
10146 "drop" => drop(old),
10147 _ => unreachable!(),
10148 }
10149 assert!(
10150 matches!(
10151 breakers.admit("prov"),
10152 RouteBindAdmission::Refused {
10153 probe_in_flight: true,
10154 ..
10155 }
10156 ),
10157 "{settlement} of a pre-open relay cannot release the real probe"
10158 );
10159 assert!(
10160 probe
10161 .record_timeout(1, Duration::ZERO)
10162 .unwrap()
10163 .reopened_after_probe
10164 );
10165 assert!(matches!(
10166 breakers.admit("prov"),
10167 RouteBindAdmission::Admitted { probe: true, .. }
10168 ));
10169 }
10170 let breakers = RouteBindBreakers::default();
10171 let admit = || match breakers.admit("prov") {
10172 RouteBindAdmission::Admitted { guard, .. } => guard,
10173 _ => panic!("relay admitted"),
10174 };
10175 admit().record_timeout(1, Duration::ZERO);
10176 let mut old_probe = admit();
10177 breakers.reset_for_new_module_connection("prov");
10178 admit().record_timeout(1, Duration::ZERO);
10179 let _new_probe = admit();
10180 old_probe.record_inconclusive();
10181 assert!(matches!(
10182 breakers.admit("prov"),
10183 RouteBindAdmission::Refused {
10184 probe_in_flight: true,
10185 ..
10186 }
10187 ));
10188 }
10189
10190 #[tokio::test]
10191 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
10192 for candidate in [false, true] {
10193 let registry = Arc::new(Registry::default());
10194 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10195 [("vault".to_string(), true), ("squatter".to_string(), true)],
10196 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10197 );
10198 let conn = ConnectionId::new(77);
10199 let (ctx, mut rx) = route_ctx(conn);
10200 let initial = capability_manifest("squatter", &[], &[]);
10201 if candidate {
10202 registry
10203 .register_candidate_with_control_ops(
10204 initial.clone(),
10205 PROTOCOL_VERSION,
10206 conn,
10207 module_baseline_control_ops(),
10208 )
10209 .unwrap();
10210 handler
10211 .forwarding
10212 .register_candidate_module_connection(
10213 conn,
10214 "squatter".to_string(),
10215 PROTOCOL_VERSION,
10216 manifest_concurrency(&initial),
10217 ctx.egress.clone(),
10218 )
10219 .unwrap();
10220 } else {
10221 hello_via_sink(
10222 &handler,
10223 &ctx,
10224 &mut rx,
10225 hello_frame_with_manifest(initial.clone(), 1),
10226 )
10227 .await;
10228 }
10229 let response = handler
10230 .handle_control_frame(
10231 &ctx,
10232 catalog_update_with_capabilities_frame(
10233 2,
10234 capability_manifest("squatter", &["credentials-provider/v1"], &[])
10235 .capabilities
10236 .unwrap(),
10237 ),
10238 )
10239 .await
10240 .unwrap();
10241 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10242 assert_eq!(
10243 registry
10244 .get_module_by_connection(conn)
10245 .unwrap()
10246 .unwrap()
10247 .manifest,
10248 initial
10249 );
10250 }
10251 }
10252
10253 #[test]
10254 fn server_describe_surfaces_required_capability_verdict_fields() {
10255 let registry = Arc::new(Registry::default());
10256 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10257 [
10258 ("consumer".to_string(), true),
10259 ("provider".to_string(), false),
10260 ],
10261 BTreeMap::new(),
10262 );
10263 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10264 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10265 provides: Vec::new(),
10266 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10267 capability: "credentials-provider/v1".to_string(),
10268 need: subc_protocol::manifest::CapabilityNeed::Required,
10269 }],
10270 must_never_reach: Vec::new(),
10271 });
10272 let hello = Frame::build(
10273 FrameType::Hello,
10274 control_flags(),
10275 0,
10276 0,
10277 78,
10278 serde_json::to_vec(&ModuleHelloBody {
10279 manifest: consumer,
10280 protocol_ver: PROTOCOL_VERSION,
10281 control_ops: None,
10282 launch_nonce: None,
10283 })
10284 .expect("HELLO serializes"),
10285 )
10286 .expect("HELLO frame builds");
10287 handler
10288 .handle_control(ConnectionId::new(78), hello)
10289 .expect("consumer registers");
10290 let describe = Frame::build(
10291 FrameType::Request,
10292 control_flags(),
10293 0,
10294 0,
10295 79,
10296 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10297 .expect("request serializes"),
10298 )
10299 .expect("describe frame builds");
10300 let response = handler
10301 .handle_server_describe(describe)
10302 .expect("server.describe succeeds");
10303 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10304 let requirement = &rendered["capability_requirements"][0];
10305 assert_eq!(requirement["consumer"], "consumer");
10306 assert_eq!(requirement["verdict"], "never_provided");
10307 assert_eq!(requirement["episode_seq"], 1);
10308 assert_eq!(requirement["config_satisfiable"], false);
10309 assert_eq!(requirement["runtime_available"], false);
10310 assert!(requirement["detail"]
10311 .as_str()
10312 .expect("detail string")
10313 .contains("credentials-provider/v1"));
10314 }
10315
10316 fn deregister_required_provider(
10322 daemon_shutdown_begun: bool,
10323 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10324 const CAPABILITY: &str = "project-identity/v1";
10325 let registry = Arc::new(Registry::default());
10326 let daemon_shutdown = DaemonShutdownFlag::default();
10327 let handler = ControlHandler::new(Arc::clone(®istry))
10328 .with_capability_config(
10329 [
10330 ("consumer".to_string(), true),
10331 ("provider".to_string(), true),
10332 ],
10333 BTreeMap::new(),
10334 )
10335 .with_daemon_shutdown_flag(daemon_shutdown.clone());
10336 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10337 consumer.capabilities = Some(CapabilityDeclarations {
10338 provides: Vec::new(),
10339 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10340 capability: CAPABILITY.to_string(),
10341 need: subc_protocol::manifest::CapabilityNeed::Required,
10342 }],
10343 must_never_reach: Vec::new(),
10344 });
10345 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10346 let provider_conn = ConnectionId::new(182);
10347 for (conn, manifest, corr) in [
10348 (ConnectionId::new(181), consumer, 181),
10349 (provider_conn, provider, 182),
10350 ] {
10351 let hello = handler
10352 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10353 .expect("module registers");
10354 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10355 }
10356 assert_eq!(
10357 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10358 Some(CapabilityVerdict::Provided),
10359 "the registered provider must satisfy the requirement first"
10360 );
10361 if daemon_shutdown_begun {
10362 daemon_shutdown.set_for_test();
10363 }
10364
10365 let capture = EventCapture::default();
10366 let _subscriber =
10367 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10368 handler
10369 .cleanup_connection(provider_conn)
10370 .expect("provider deregisters");
10371 let events = capture
10372 .events()
10373 .into_iter()
10374 .filter(|event| {
10375 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10376 })
10377 .collect();
10378 (
10379 events,
10380 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10381 )
10382 }
10383
10384 #[test]
10385 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10386 let (events, verdict) = deregister_required_provider(true);
10387 assert_eq!(
10388 verdict,
10389 Some(CapabilityVerdict::NeverProvided),
10390 "shutdown must not change the verdict route.open refusals read"
10391 );
10392 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10393 let event = &events[0];
10394 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10395 assert_eq!(
10396 event.fields.get("cause").map(String::as_str),
10397 Some("\"daemon_shutdown\""),
10398 "{event:?}"
10399 );
10400 assert_eq!(
10401 event.fields.get("verdict").map(String::as_str),
10402 Some("\"never_provided\""),
10403 "{event:?}"
10404 );
10405 }
10406
10407 #[test]
10408 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10409 let (events, verdict) = deregister_required_provider(false);
10410 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10411 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10412 let event = &events[0];
10413 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10414 assert_eq!(event.fields.get("cause"), None, "{event:?}");
10415 assert_eq!(
10416 event.fields.get("verdict").map(String::as_str),
10417 Some("\"never_provided\""),
10418 "{event:?}"
10419 );
10420 }
10421
10422 #[test]
10423 fn catalog_list_omits_capabilities_for_legacy_manifest() {
10424 let registry = Arc::new(Registry::default());
10425 let handler = ControlHandler::new(Arc::clone(®istry));
10426 let hello = handler
10427 .handle_control(
10428 ConnectionId::new(101),
10429 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10430 )
10431 .expect("legacy HELLO registers");
10432 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10433
10434 let request = Frame::build(
10435 FrameType::Request,
10436 control_flags(),
10437 0,
10438 0,
10439 102,
10440 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10441 .expect("catalog request serializes"),
10442 )
10443 .expect("catalog request frame builds");
10444 let response = handler
10445 .handle_catalog_list(request, None)
10446 .expect("catalog list succeeds");
10447 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10448 assert!(
10449 body["modules"][0].get("capabilities").is_none(),
10450 "legacy manifest must retain an absent capabilities field on catalog.list"
10451 );
10452 }
10453
10454 #[test]
10455 fn hello_ack_omits_storage_when_no_storage_config() {
10456 let registry = Arc::new(Registry::default());
10457 let handler = ControlHandler::new(Arc::clone(®istry));
10458 let responses = handler
10459 .handle_control(
10460 ConnectionId::new(1),
10461 hello_frame("aft", PROTOCOL_VERSION, 7),
10462 )
10463 .unwrap();
10464 let ack = parse_ack(&responses[0]);
10465 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10466 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10467 }
10468
10469 #[tokio::test]
10470 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10471 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10472 let registry = Arc::new(Registry::default());
10473 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
10474 let responses = handler
10475 .handle_control(
10476 ConnectionId::new(1),
10477 hello_frame("aft", PROTOCOL_VERSION, 7),
10478 )
10479 .unwrap();
10480 let ack = parse_ack(&responses[0]);
10481 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10482
10483 let described = handler
10484 .handle_control_frame(
10485 &route_ctx(ConnectionId::new(2)).0,
10486 Frame::build(
10487 FrameType::Request,
10488 control_flags(),
10489 0,
10490 0,
10491 9,
10492 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10493 )
10494 .unwrap(),
10495 )
10496 .await
10497 .unwrap();
10498 let ClientControlResponse::ServerDescribe { machine_id, .. } =
10499 serde_json::from_slice(&described[0].body).unwrap()
10500 else {
10501 panic!("server.describe answered with another shape");
10502 };
10503 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10504 }
10505
10506 #[test]
10507 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10508 let registry = Arc::new(Registry::default());
10511 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
10512 crate::daemon_config::StorageConfig::Sqlite {
10513 data_home: std::path::PathBuf::from("/data"),
10514 },
10515 ));
10516
10517 let responses = handler
10518 .handle_control(
10519 ConnectionId::new(1),
10520 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10521 )
10522 .unwrap();
10523 let ack = parse_ack(&responses[0]);
10524 assert_eq!(
10525 ack.storage,
10526 Some(serde_json::json!({
10527 "module_id": "alfonso-routing",
10528 "storage_namespace": "default",
10529 "isolation": { "kind": "module" },
10530 "backend": {
10531 "backend": "sqlite",
10532 "path": "/data/cortexkit/alfonso-routing/store.db"
10533 }
10534 })),
10535 "the delivered descriptor is the module's own sqlite store path"
10536 );
10537 }
10538
10539 #[test]
10540 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10541 let registry = Arc::new(Registry::default());
10542 let handler = ControlHandler::new(Arc::clone(®istry));
10543 let conn = ConnectionId::new(1);
10544 let responses = handler
10545 .handle_control(
10546 conn,
10547 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10548 )
10549 .unwrap();
10550 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10551 let registration = registry.get_module("aft").unwrap().unwrap();
10552 assert_eq!(registration.control_ops, module_baseline_control_ops());
10553
10554 let frame =
10555 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10556 assert!(handler
10557 .guard_module_control_op(&frame, "aft", "route.bind")
10558 .unwrap()
10559 .is_none());
10560 let error = handler
10561 .guard_module_control_op(&frame, "aft", "test.synthetic")
10562 .unwrap()
10563 .expect("synthetic ungranted op should be rejected");
10564 assert_eq!(error.header.ty, FrameType::Error);
10565 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10566 }
10567
10568 #[test]
10569 fn hello_control_ops_some_adds_optional_grants() {
10570 let registry = Arc::new(Registry::default());
10571 let handler = ControlHandler::new(Arc::clone(®istry));
10572 handler
10573 .handle_control(
10574 ConnectionId::new(1),
10575 hello_frame_with_control_ops(
10576 "aft",
10577 PROTOCOL_VERSION,
10578 7,
10579 Some(vec![
10580 "future.synthetic".to_string(),
10581 "route.bind".to_string(),
10582 ]),
10583 ),
10584 )
10585 .unwrap();
10586 let registration = registry.get_module("aft").unwrap().unwrap();
10587 assert_eq!(
10588 registration.control_ops,
10589 vec![
10590 "route.bind".to_string(),
10591 "route.status".to_string(),
10592 "future.synthetic".to_string(),
10593 ]
10594 );
10595 let frame =
10596 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10597 assert!(handler
10598 .guard_module_control_op(&frame, "aft", "future.synthetic")
10599 .unwrap()
10600 .is_none());
10601 }
10602
10603 #[tokio::test]
10604 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10605 let registry = Arc::new(Registry::default());
10606 let forwarding = Arc::new(ForwardingTable::default());
10607 let handler =
10608 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10609 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10610 hello_via_sink(
10611 &handler,
10612 &module_ctx,
10613 &mut module_rx,
10614 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10615 )
10616 .await;
10617
10618 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10619 let responses = handler
10620 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10621 .await
10622 .unwrap();
10623 assert_eq!(responses.len(), 1);
10624 assert_eq!(responses[0].header.ty, FrameType::Error);
10625 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10626 assert!(module_rx.try_recv().is_err());
10627 }
10628
10629 #[tokio::test]
10630 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10631 let registry = Arc::new(Registry::default());
10632 let forwarding = Arc::new(ForwardingTable::default());
10633 let handler =
10634 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10635 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10636 hello_via_sink(
10637 &handler,
10638 &module_ctx,
10639 &mut module_rx,
10640 hello_frame_with_control_ops(
10641 "aft",
10642 PROTOCOL_VERSION,
10643 7,
10644 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10645 ),
10646 )
10647 .await;
10648
10649 let project_root = unique_project_root("demux");
10650 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10651 let route_handler = handler.clone();
10652 let route_task = tokio::spawn(async move {
10653 route_handler
10654 .handle_control_frame(
10655 &route_client_ctx,
10656 route_open_frame(100, "aft", project_root),
10657 )
10658 .await
10659 .unwrap()
10660 });
10661 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10662 .await
10663 .unwrap()
10664 .unwrap();
10665 assert!(matches!(
10666 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10667 ModuleControlRequest::RouteBind { .. }
10668 ));
10669
10670 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10671 let health_handler = handler.clone();
10672 let health_task = tokio::spawn(async move {
10673 health_handler
10674 .handle_control_frame(
10675 &health_client_ctx,
10676 supervisor_health_probe_frame(101, "aft"),
10677 )
10678 .await
10679 .unwrap()
10680 });
10681 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10682 .await
10683 .unwrap()
10684 .unwrap();
10685 assert_eq!(
10686 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10687 ModuleControlRequest::HealthCheck {}
10688 );
10689
10690 handler
10691 .handle_control_frame(
10692 &module_ctx,
10693 health_response(health_frame.header.corr, HealthStatus::Degraded),
10694 )
10695 .await
10696 .unwrap();
10697 let health_response = health_task.await.unwrap();
10698 assert_eq!(health_response.len(), 1);
10699 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10700 ClientControlResponse::SupervisorHealthProbe {
10701 module_id,
10702 status,
10703 detail,
10704 metrics,
10705 } => {
10706 assert_eq!(module_id, "aft");
10707 assert_eq!(status, HealthStatus::Degraded);
10708 assert_eq!(detail.as_deref(), Some("warming"));
10709 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10710 }
10711 other => panic!("unexpected health response: {other:?}"),
10712 }
10713
10714 handler
10715 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10716 .await
10717 .unwrap();
10718 let route_response = route_task.await.unwrap();
10719 assert!(route_response.is_empty());
10720 let published = route_client_rx.recv().await.unwrap();
10721 assert!(matches!(
10722 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10723 ClientControlResponse::RouteOpen { .. }
10724 ));
10725 }
10726
10727 async fn relay_route_open(
10732 handler: &ControlHandler,
10733 client_connection: ConnectionId,
10734 client_egress: &FrameSink,
10735 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10736 corr: u64,
10737 module_id: &str,
10738 project_root_label: &str,
10739 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10740 let ctx = RouteCtx {
10741 connection_id: client_connection,
10742 egress: client_egress.clone(),
10743 };
10744 let handler = handler.clone();
10745 let project_root = unique_project_root(project_root_label);
10746 let module_id = module_id.to_string();
10747 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10748 let task = tokio::spawn(async move {
10749 let _guard = tracing::dispatcher::set_default(&dispatch);
10750 handler
10751 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10752 .await
10753 .unwrap()
10754 });
10755 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10756 .await
10757 .expect("module receives the relayed route.bind")
10758 .expect("module egress is open");
10759 (task, bind.frame)
10760 }
10761
10762 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10763 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10764 ModuleControlRequest::RouteBind {
10765 route_channel,
10766 epoch,
10767 ..
10768 } => (route_channel, epoch),
10769 other => panic!("expected a route.bind request, got {other:?}"),
10770 }
10771 }
10772
10773 fn published_route(frame: &Frame) -> (u16, u32) {
10774 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10775 ClientControlResponse::RouteOpen {
10776 route_channel,
10777 route_epoch,
10778 } => (route_channel, route_epoch),
10779 other => panic!("expected a route.open response, got {other:?}"),
10780 }
10781 }
10782
10783 #[tokio::test]
10803 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10804 let registry = Arc::new(Registry::default());
10805 let forwarding = Arc::new(ForwardingTable::default());
10806 let handler =
10807 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10808
10809 let module_connection = ConnectionId::new(30);
10810 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10811 hello_via_sink(
10812 &handler,
10813 &module_ctx,
10814 &mut module_rx,
10815 hello_frame("aft", PROTOCOL_VERSION, 7),
10816 )
10817 .await;
10818
10819 let dying_client = ConnectionId::new(31);
10820 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10821
10822 let (first_task, first_bind) = relay_route_open(
10825 &handler,
10826 dying_client,
10827 &dying_ctx.egress,
10828 &mut module_rx,
10829 100,
10830 "aft",
10831 "closing-first",
10832 )
10833 .await;
10834 handler
10835 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10836 .await
10837 .unwrap();
10838 assert!(first_task.await.unwrap().is_empty());
10839 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10840
10841 let (second_task, second_bind) = relay_route_open(
10843 &handler,
10844 dying_client,
10845 &dying_ctx.egress,
10846 &mut module_rx,
10847 101,
10848 "aft",
10849 "closing-second",
10850 )
10851 .await;
10852 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10853
10854 assert!(forwarding
10857 .escalate_client_delivery_failure(
10858 dying_client,
10859 first_channel,
10860 first_epoch,
10861 CloseReason::new(
10862 "module_to_client_delivery_failed",
10863 "client egress refused a module frame",
10864 ),
10865 crate::forwarding::UndeliveredFrame {
10866 module_id: None,
10867 sink: &dying_ctx.egress,
10868 },
10869 )
10870 .unwrap());
10871 assert!(!dying_ctx.egress.is_closed());
10872
10873 let ack = handler
10875 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
10876 .await;
10877 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
10878 if module_loop_error.is_some() {
10879 handler.cleanup_connection(module_connection).unwrap();
10883 }
10884 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10888 .await
10889 .ok()
10890 .flatten();
10891
10892 assert!(
10894 registry
10895 .get_module_by_connection(module_connection)
10896 .unwrap()
10897 .is_some(),
10898 "one client's closing connection ended the shared module connection: \
10899 {module_loop_error:?}"
10900 );
10901 let cotenant = ConnectionId::new(32);
10903 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
10904 let (cotenant_task, cotenant_bind) = relay_route_open(
10905 &handler,
10906 cotenant,
10907 &cotenant_ctx.egress,
10908 &mut module_rx,
10909 102,
10910 "aft",
10911 "closing-cotenant",
10912 )
10913 .await;
10914 handler
10915 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
10916 .await
10917 .unwrap();
10918 assert!(cotenant_task.await.unwrap().is_empty());
10919 let (cotenant_channel, cotenant_epoch) =
10920 published_route(&cotenant_rx.recv().await.unwrap());
10921 assert!(matches!(
10922 forwarding
10923 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
10924 .unwrap(),
10925 DataRoute::Client(DataRouteState::Bound(_))
10926 ));
10927
10928 let goodbye = post_ack_module_frame
10931 .expect("module receives a GOODBYE for the abandoned route channel");
10932 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10933 assert_eq!(goodbye.header.channel, abandoned_channel);
10934 assert_eq!(goodbye.header.epoch, abandoned_epoch);
10935
10936 assert!(dying_rx.try_recv().is_err());
10940 let second_response = second_task.await.unwrap();
10941 assert_eq!(second_response.len(), 1);
10942 assert_eq!(
10943 parse_error(&second_response[0])["code"],
10944 "target_unavailable"
10945 );
10946 }
10947
10948 async fn catalog_snapshot(
10951 handler: &ControlHandler,
10952 client_ctx: &RouteCtx,
10953 corr: u64,
10954 ) -> (u64, Vec<(String, bool)>) {
10955 let body =
10956 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
10957 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
10958 let reply = handler
10959 .handle_control_frame(client_ctx, request)
10960 .await
10961 .unwrap()
10962 .pop()
10963 .unwrap();
10964 let ClientControlResponse::CatalogList {
10965 generation,
10966 modules,
10967 ..
10968 } = serde_json::from_slice(&reply.body).unwrap()
10969 else {
10970 panic!("not a catalog.list reply");
10971 };
10972 (
10973 generation,
10974 modules
10975 .into_iter()
10976 .map(|entry| (entry.module_id, entry.ready))
10977 .collect(),
10978 )
10979 }
10980
10981 fn hello_held_before_forwarding() -> (
10987 ControlHandler,
10988 Arc<std::sync::Barrier>,
10989 std::thread::JoinHandle<Vec<Frame>>,
10990 RouteCtx,
10991 mpsc::Receiver<crate::router::OutboundFrame>,
10992 ) {
10993 let hold = Arc::new(std::sync::Barrier::new(2));
10994 let handler = ControlHandler::with_forwarding(
10995 Arc::new(Registry::default()),
10996 Arc::new(ForwardingTable::default()),
10997 )
10998 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
10999 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11000 let hello = {
11001 let handler = handler.clone();
11002 let module_ctx = module_ctx.clone();
11003 std::thread::spawn(move || {
11004 handler
11005 .handle_hello(
11006 module_ctx.connection_id,
11007 Some(module_ctx.egress.clone()),
11008 hello_frame("aft", PROTOCOL_VERSION, 7),
11009 )
11010 .unwrap()
11011 })
11012 };
11013 (handler, hold, hello, module_ctx, module_rx)
11014 }
11015
11016 #[tokio::test]
11025 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
11026 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
11027 let client = ConnectionId::new(41);
11028 let (client_ctx, mut client_rx) = route_ctx(client);
11029
11030 hold.wait();
11031 assert!(handler.registry.get_module("aft").unwrap().is_some());
11033 assert!(!handler
11034 .forwarding
11035 .has_live_module_connection("aft")
11036 .unwrap());
11037 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
11038 assert!(
11039 listed.is_empty(),
11040 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
11041 );
11042 let refused = handler
11043 .handle_control_frame(
11044 &client_ctx,
11045 route_open_frame(301, "aft", unique_project_root("listed-window")),
11046 )
11047 .await
11048 .unwrap();
11049 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
11050 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
11051 let duplicate = handler
11052 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
11053 .await
11054 .unwrap();
11055 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11056
11057 hold.wait();
11058 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11059 let ack = module_rx.try_recv().unwrap().frame;
11060 assert_eq!(ack.header.ty, FrameType::HelloAck);
11061 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11062 assert_eq!(listed, vec![("aft".to_string(), true)]);
11063
11064 let (open_task, bind) = relay_route_open(
11065 &handler,
11066 client,
11067 &client_ctx.egress,
11068 &mut module_rx,
11069 303,
11070 "aft",
11071 "listed-after",
11072 )
11073 .await;
11074 assert!(handler
11075 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11076 .await
11077 .unwrap()
11078 .is_empty());
11079 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
11080 published_route(&client_rx.try_recv().unwrap().frame);
11081 }
11082
11083 #[tokio::test]
11090 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11091 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11092 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11093 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11094
11095 hold.wait();
11096 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11097 assert!(listed.is_empty());
11098 assert_eq!(
11099 during, before,
11100 "the generation moved while the module was not listed"
11101 );
11102
11103 hold.wait();
11104 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11105 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11106 assert_eq!(listed, vec![("aft".to_string(), true)]);
11107 assert_eq!(
11108 after,
11109 before + 1,
11110 "the module appeared without exactly one generation bump"
11111 );
11112 }
11113
11114 #[tokio::test]
11119 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11120 let registry = Arc::new(Registry::default());
11121 let handler = ControlHandler::with_forwarding(
11122 Arc::clone(®istry),
11123 Arc::new(ForwardingTable::default()),
11124 );
11125 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11126 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11127 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11128 drop(module_rx);
11129
11130 let replies = handler
11131 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
11132 .await
11133 .unwrap();
11134 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
11135 assert!(registry.get_module("aft").unwrap().is_none());
11136 assert!(registry
11137 .get_module_by_connection(module_ctx.connection_id)
11138 .unwrap()
11139 .is_none());
11140 assert_eq!(
11141 catalog_snapshot(&handler, &client_ctx, 301).await,
11142 (before, Vec::new())
11143 );
11144 }
11145
11146 async fn route_open_held_after_bind_send() -> (
11150 ControlHandler,
11151 Arc<tokio::sync::Barrier>,
11152 RouteCtx,
11153 mpsc::Receiver<crate::router::OutboundFrame>,
11154 ) {
11155 let hold = Arc::new(tokio::sync::Barrier::new(2));
11156 let handler = ControlHandler::with_forwarding(
11157 Arc::new(Registry::default()),
11158 Arc::new(ForwardingTable::default()),
11159 )
11160 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
11161 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11162 hello_via_sink(
11163 &handler,
11164 &module_ctx,
11165 &mut module_rx,
11166 hello_frame("aft", PROTOCOL_VERSION, 7),
11167 )
11168 .await;
11169 (handler, hold, module_ctx, module_rx)
11170 }
11171
11172 #[tokio::test]
11179 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
11180 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11181 let client = ConnectionId::new(41);
11182 let (client_ctx, mut client_rx) = route_ctx(client);
11183
11184 let (open_task, bind) = relay_route_open(
11185 &handler,
11186 client,
11187 &client_ctx.egress,
11188 &mut module_rx,
11189 200,
11190 "aft",
11191 "acked-before-mark",
11192 )
11193 .await;
11194 let (module_channel, module_epoch) = route_bind_channel(&bind);
11195 hold.wait().await;
11197
11198 assert!(handler
11200 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11201 .await
11202 .unwrap()
11203 .is_empty());
11204 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
11205
11206 hold.wait().await;
11207 assert!(
11208 open_task.await.unwrap().is_empty(),
11209 "route.open answers accepted"
11210 );
11211
11212 match module_rx.try_recv() {
11215 Err(mpsc::error::TryRecvError::Empty) => {}
11216 Ok(outbound) => panic!(
11217 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
11218 outbound.frame.header.ty,
11219 outbound.frame.header.channel,
11220 outbound.frame.header.epoch
11221 ),
11222 Err(err) => panic!("module egress closed: {err}"),
11223 }
11224
11225 let router = Router::with_control_handler(Arc::new(handler.clone()));
11228 let request = Frame::build(
11229 FrameType::Request,
11230 Flags::new(true, Priority::Interactive, false),
11231 client_channel,
11232 client_epoch,
11233 201,
11234 b"first request".to_vec(),
11235 )
11236 .unwrap();
11237 router
11238 .route_for_connection(&client_ctx, request)
11239 .await
11240 .unwrap();
11241 let delivered = module_rx.try_recv().unwrap().frame;
11242 assert_eq!(delivered.header.ty, FrameType::Request);
11243 assert_eq!(
11244 (delivered.header.channel, delivered.header.epoch),
11245 (module_channel, module_epoch)
11246 );
11247 assert_eq!(delivered.body, b"first request");
11248 }
11249
11250 #[tokio::test]
11256 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
11257 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11258 let client = ConnectionId::new(42);
11259 let (client_ctx, _client_rx) = route_ctx(client);
11260
11261 let (open_task, bind) = relay_route_open(
11262 &handler,
11263 client,
11264 &client_ctx.egress,
11265 &mut module_rx,
11266 300,
11267 "aft",
11268 "released-before-mark",
11269 )
11270 .await;
11271 let (module_channel, module_epoch) = route_bind_channel(&bind);
11272 hold.wait().await;
11273
11274 handler.cleanup_connection(client).unwrap();
11275 assert!(
11276 module_rx.try_recv().is_err(),
11277 "client teardown must not GOODBYE a bind route.open is still queueing"
11278 );
11279
11280 hold.wait().await;
11281 let response = open_task.await.unwrap();
11282 assert_eq!(response.len(), 1);
11283 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11284
11285 let goodbye = module_rx
11286 .try_recv()
11287 .expect("module is told to drop the binding the daemon abandoned")
11288 .frame;
11289 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11290 assert_eq!(
11291 (goodbye.header.channel, goodbye.header.epoch),
11292 (module_channel, module_epoch)
11293 );
11294 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11295 }
11296
11297 #[tokio::test]
11302 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11303 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11304 let client = ConnectionId::new(43);
11305 let (client_ctx, client_rx) = route_ctx(client);
11306
11307 let (open_task, bind) = relay_route_open(
11308 &handler,
11309 client,
11310 &client_ctx.egress,
11311 &mut module_rx,
11312 400,
11313 "aft",
11314 "gone-client-before-mark",
11315 )
11316 .await;
11317 let (module_channel, module_epoch) = route_bind_channel(&bind);
11318 hold.wait().await;
11319
11320 drop(client_rx);
11321 assert!(client_ctx.egress.is_closed());
11322 handler
11323 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11324 .await
11325 .unwrap();
11326
11327 hold.wait().await;
11328 let response = open_task.await.unwrap();
11329 assert_eq!(response.len(), 1);
11330 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11331
11332 let goodbye = module_rx
11333 .try_recv()
11334 .expect("module is told to drop the binding it acked")
11335 .frame;
11336 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11337 assert_eq!(
11338 (goodbye.header.channel, goodbye.header.epoch),
11339 (module_channel, module_epoch)
11340 );
11341 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11342 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11343 }
11344
11345 #[test]
11352 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11353 let handler = ControlHandler::default();
11354 let module_connection = ConnectionId::new(30);
11355 let client_connection = ConnectionId::new(31);
11356
11357 handler
11358 .refuse_to_end_module_connection_for_a_client(
11359 module_connection,
11360 77,
11361 ForwardingError::ConnectionClosing {
11362 connection_id: client_connection,
11363 },
11364 )
11365 .expect("a closing client must never end the module connection");
11366
11367 assert!(matches!(
11368 handler.refuse_to_end_module_connection_for_a_client(
11369 module_connection,
11370 78,
11371 ForwardingError::ConnectionClosing {
11372 connection_id: module_connection,
11373 },
11374 ),
11375 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11376 connection_id
11377 })) if connection_id == module_connection
11378 ));
11379 assert!(matches!(
11380 handler.refuse_to_end_module_connection_for_a_client(
11381 module_connection,
11382 79,
11383 ForwardingError::Poisoned,
11384 ),
11385 Err(RouterError::Forwarding(ForwardingError::Poisoned))
11386 ));
11387 assert!(matches!(
11388 handler.refuse_to_end_module_connection_for_a_client(
11389 module_connection,
11390 80,
11391 ForwardingError::StaleModuleEndpoint,
11392 ),
11393 Err(RouterError::Forwarding(
11394 ForwardingError::StaleModuleEndpoint
11395 ))
11396 ));
11397 }
11398
11399 #[tokio::test]
11412 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11413 let registry = Arc::new(Registry::default());
11414 let forwarding = Arc::new(ForwardingTable::default());
11415 let supervisor = SupervisorHandle::new();
11416 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11417 let handler =
11418 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11419 .with_supervisor(supervisor);
11420
11421 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11422 hello_via_sink(
11423 &handler,
11424 &target_ctx,
11425 &mut target_rx,
11426 hello_frame("target", PROTOCOL_VERSION, 1),
11427 )
11428 .await;
11429
11430 let wrong_nonce = handler
11434 .handle_control_frame(
11435 &route_ctx(ConnectionId::new(91)).0,
11436 route_open_frame_with_admission_facts(
11437 20,
11438 "target",
11439 unique_project_root("admission-facts"),
11440 Some(subc_control::ConsumerIdentity {
11441 module_id: "fed".to_string(),
11442 launch_nonce: "not-the-real-nonce".to_string(),
11443 }),
11444 None,
11445 ),
11446 )
11447 .await
11448 .unwrap();
11449 assert_eq!(
11450 parse_error(&wrong_nonce[0])["code"],
11451 "bad_consumer_identity",
11452 "a mismatched launch nonce must be refused, not stamped Reserved"
11453 );
11454
11455 let never_spawned = handler
11459 .handle_control_frame(
11460 &route_ctx(ConnectionId::new(92)).0,
11461 route_open_frame_with_admission_facts(
11462 21,
11463 "target",
11464 unique_project_root("admission-facts"),
11465 Some(subc_control::ConsumerIdentity {
11466 module_id: "never-spawned".to_string(),
11467 launch_nonce: "any-nonce".to_string(),
11468 }),
11469 None,
11470 ),
11471 )
11472 .await
11473 .unwrap();
11474 assert_eq!(
11475 parse_error(&never_spawned[0])["code"],
11476 "bad_consumer_identity",
11477 "an unspawned module_id must be refused rather than accepted for lack of a record"
11478 );
11479 }
11480
11481 #[tokio::test]
11504 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11505 let registry = Arc::new(Registry::default());
11506 let forwarding = Arc::new(ForwardingTable::default());
11507 let supervisor = SupervisorHandle::new();
11508 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11509 let handler =
11510 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11511 .with_supervisor(supervisor);
11512
11513 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11514 hello_via_sink(
11515 &handler,
11516 &target_ctx,
11517 &mut target_rx,
11518 hello_frame("target", PROTOCOL_VERSION, 1),
11519 )
11520 .await;
11521
11522 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11523 let route_handler = handler.clone();
11524 let route_task = tokio::spawn(async move {
11525 route_handler
11526 .handle_control_frame(
11527 &client_ctx,
11528 route_open_frame_with_admission_facts(
11529 30,
11530 "target",
11531 unique_project_root("admission-facts"),
11532 Some(subc_control::ConsumerIdentity {
11533 module_id: "fed".to_string(),
11534 launch_nonce: "fed-nonce".to_string(),
11535 }),
11536 None,
11537 ),
11538 )
11539 .await
11540 .unwrap()
11541 });
11542
11543 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11550 .await
11551 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11552 .expect("module control channel closed before route.bind");
11553 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11554 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11555 panic!("expected route.bind")
11556 };
11557 assert_eq!(
11558 principal,
11559 Some(Principal::Reserved {
11560 module_id: "fed".to_string()
11561 }),
11562 "a correctly attested consumer must be stamped Reserved for its own id"
11563 );
11564
11565 handler
11566 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11567 .await
11568 .unwrap();
11569 assert!(route_task.await.unwrap().is_empty());
11570 assert!(
11571 matches!(
11572 serde_json::from_slice::<ClientControlResponse>(
11573 &client_rx.recv().await.unwrap().body
11574 )
11575 .unwrap(),
11576 ClientControlResponse::RouteOpen { .. }
11577 ),
11578 "the route must actually open, not merely avoid an error"
11579 );
11580 }
11581
11582 #[tokio::test(start_paused = true)]
11583 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11584 let registry = Arc::new(Registry::default());
11585 let forwarding = Arc::new(ForwardingTable::default());
11586 let supervisor = SupervisorHandle::new();
11587 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11588 let handler =
11589 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11590 .with_supervisor(supervisor);
11591
11592 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11593 hello_via_sink(
11594 &handler,
11595 &target_ctx,
11596 &mut target_rx,
11597 hello_frame("target", PROTOCOL_VERSION, 1),
11598 )
11599 .await;
11600
11601 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11602 let direct_handler = handler.clone();
11603 let direct_open = tokio::spawn(async move {
11604 direct_handler
11605 .handle_control_frame(
11606 &direct_ctx,
11607 route_open_frame(2, "target", unique_project_root("route-census-direct")),
11608 )
11609 .await
11610 .unwrap()
11611 });
11612 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11613 .await
11614 .expect("no direct route.bind within 5s")
11615 .expect("target control channel closed before direct route.bind");
11616 handler
11617 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11618 .await
11619 .unwrap();
11620 assert!(direct_open.await.unwrap().is_empty());
11621 let _ = direct_rx.recv().await.unwrap();
11622
11623 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11624 let reserved_handler = handler.clone();
11625 let reserved_open = tokio::spawn(async move {
11626 reserved_handler
11627 .handle_control_frame(
11628 &reserved_ctx,
11629 route_open_frame_with_admission_facts(
11630 3,
11631 "target",
11632 unique_project_root("admission-facts"),
11633 Some(ConsumerIdentity {
11634 module_id: "fed".to_string(),
11635 launch_nonce: "fed-nonce".to_string(),
11636 }),
11637 None,
11638 ),
11639 )
11640 .await
11641 .unwrap()
11642 });
11643 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11644 .await
11645 .expect("no reserved route.bind within 5s")
11646 .expect("target control channel closed before reserved route.bind");
11647 handler
11648 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11649 .await
11650 .unwrap();
11651 assert!(reserved_open.await.unwrap().is_empty());
11652 let _ = reserved_rx.recv().await.unwrap();
11653
11654 forwarding
11655 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11656 .unwrap();
11657 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11658 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11659 module_id: Some("target".to_string()),
11660 })
11661 .unwrap();
11662 let census_frame =
11663 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11664 let response = handler
11665 .handle_control_frame(&census_ctx, census_frame)
11666 .await
11667 .unwrap()
11668 .pop()
11669 .unwrap();
11670 let actual: Value = serde_json::from_slice(&response.body).unwrap();
11671 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11672 assert!(matches!(
11673 decoded,
11674 ClientControlResponse::SupervisorRoutes { .. }
11675 ));
11676 let routes = actual["modules"][0]["routes"].as_array().unwrap();
11677 assert_eq!(routes.len(), 2);
11678 assert!(routes.iter().all(|route| route["draining"] == true));
11679 assert!(
11682 routes.iter().all(|route| route["drain_reason"] == "reload"),
11683 "draining routes must name the drain's reason: {routes:?}"
11684 );
11685 assert!(routes.iter().any(|route| {
11686 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11687 }));
11688 assert!(routes.iter().any(|route| {
11689 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11690 }));
11691
11692 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11693 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11694 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11695 std::fs::write(
11696 &golden_path,
11697 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11698 )
11699 .unwrap();
11700 }
11701 let expected: Value =
11702 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11703 assert_eq!(actual, expected);
11704 }
11705
11706 async fn query_live_roots(
11707 handler: &ControlHandler,
11708 module_ctx: &RouteCtx,
11709 ) -> ModuleControlResponseToModule {
11710 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11711 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11712 let response = handler
11713 .handle_control_frame(module_ctx, frame)
11714 .await
11715 .unwrap()
11716 .pop()
11717 .unwrap();
11718 serde_json::from_slice(&response.body).unwrap()
11719 }
11720
11721 #[tokio::test(start_paused = true)]
11722 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11723 let registry = Arc::new(Registry::default());
11724 let forwarding = Arc::new(ForwardingTable::default());
11725 let handler = ControlHandler::with_forwarding(registry, forwarding);
11726 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11727 hello_via_sink(
11728 &handler,
11729 &target_ctx,
11730 &mut target_rx,
11731 hello_frame("target", PROTOCOL_VERSION, 1),
11732 )
11733 .await;
11734 let root = unique_project_root("live-roots-known");
11735 let path = ProjectRootId::from_path_allowing_missing(root.path())
11736 .unwrap()
11737 .as_path()
11738 .to_path_buf();
11739 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11740 let open_handler = handler.clone();
11741 let opened = tokio::spawn(async move {
11742 open_handler
11743 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11744 .await
11745 .unwrap()
11746 });
11747 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11748 .await
11749 .unwrap()
11750 .unwrap();
11751 handler
11752 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11753 .await
11754 .unwrap();
11755 assert!(opened.await.unwrap().is_empty());
11756 let _ = client_rx.recv().await.unwrap();
11757
11758 let root = unique_project_root("live-roots-pending");
11759 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11760 .unwrap()
11761 .as_path()
11762 .to_path_buf();
11763 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11764 let open_handler = handler.clone();
11765 let pending = tokio::spawn(async move {
11766 open_handler
11767 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11768 .await
11769 .unwrap()
11770 });
11771 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11772 .await
11773 .unwrap()
11774 .unwrap();
11775 let actual = query_live_roots(&handler, &target_ctx).await;
11776 let ModuleControlResponseToModule::LiveRoots {
11777 roots,
11778 unknown_root_bindings,
11779 total_bindings,
11780 } = actual
11781 else {
11782 panic!("expected live roots")
11783 };
11784 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11785 assert_eq!(unknown_root_bindings, 0);
11786 assert_eq!(
11787 roots.len(),
11788 2,
11789 "root-known arm must retain each canonical root"
11790 );
11791 assert_eq!(
11792 total_bindings,
11793 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11794 );
11795 let counts = roots
11796 .iter()
11797 .map(|root| (root.project_root.clone(), root.bound, root.pending))
11798 .collect::<Vec<_>>();
11799 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11800 expected.sort_by(|a, b| a.0.cmp(&b.0));
11801 assert_eq!(
11802 counts, expected,
11803 "roots must sort by path and count pending separately"
11804 );
11805 handler
11806 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11807 .await
11808 .unwrap();
11809 assert!(pending.await.unwrap().is_empty());
11810 }
11811
11812 #[tokio::test(start_paused = true)]
11813 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11814 let forwarding = Arc::new(ForwardingTable::default());
11815 let handler =
11816 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11817 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11818 hello_via_sink(
11819 &handler,
11820 &target_ctx,
11821 &mut target_rx,
11822 hello_frame("target", PROTOCOL_VERSION, 1),
11823 )
11824 .await;
11825 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11826 let pending = forwarding
11827 .begin_route_bind_relay_for_test(
11828 client_ctx.connection_id,
11829 client_ctx.egress.clone(),
11830 2,
11831 "target",
11832 )
11833 .unwrap();
11834 forwarding
11835 .complete_pending_relay(
11836 target_ctx.connection_id,
11837 pending.corr,
11838 RouteBindRelayOutcome::Accepted,
11839 )
11840 .unwrap();
11841 let actual = query_live_roots(&handler, &target_ctx).await;
11842 let ModuleControlResponseToModule::LiveRoots {
11843 roots,
11844 unknown_root_bindings,
11845 total_bindings,
11846 } = actual
11847 else {
11848 panic!("expected live roots")
11849 };
11850 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11851 assert_eq!(
11852 unknown_root_bindings, 1,
11853 "unknown-root arm must not read as no bindings"
11854 );
11855 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11856 assert_eq!(
11857 total_bindings,
11858 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11859 );
11860 }
11861
11862 #[tokio::test(start_paused = true)]
11870 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
11871 let forwarding = Arc::new(ForwardingTable::default());
11872 let handler =
11873 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11874 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
11875 let replies = handler
11876 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
11877 .await
11878 .unwrap();
11879 let queued_by_hello = module_rx.len();
11880
11881 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
11882 let open_handler = handler.clone();
11883 let open = tokio::spawn(async move {
11884 open_handler
11885 .handle_control_frame(
11886 &client_ctx,
11887 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
11888 )
11889 .await
11890 .unwrap()
11891 });
11892 let mut spins = 0;
11894 while module_rx.len() == queued_by_hello {
11895 spins += 1;
11896 assert!(spins < 10_000, "route.open never queued a route.bind");
11897 tokio::task::yield_now().await;
11898 }
11899
11900 for reply in replies {
11902 module_ctx.egress.send(reply).await.unwrap();
11903 }
11904
11905 let first = module_rx.recv().await.unwrap().frame;
11906 assert_eq!(
11907 first.header.ty,
11908 FrameType::HelloAck,
11909 "the first frame a registering module reads must be its HELLO_ACK"
11910 );
11911 assert_eq!(first.header.corr, 7);
11912 let second = module_rx.recv().await.unwrap().frame;
11913 assert_eq!(second.header.ty, FrameType::Request);
11914 assert!(
11915 matches!(
11916 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
11917 ModuleControlRequest::RouteBind { .. }
11918 ),
11919 "the route.bind follows the ack"
11920 );
11921 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
11922
11923 handler
11924 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
11925 .await
11926 .unwrap();
11927 assert!(open.await.unwrap().is_empty());
11928 let _ = client_rx.recv().await.unwrap();
11929 }
11930
11931 #[tokio::test(start_paused = true)]
11932 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
11933 let handler = ControlHandler::with_forwarding(
11934 Arc::new(Registry::default()),
11935 Arc::new(ForwardingTable::default()),
11936 );
11937 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
11938 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
11939 hello_via_sink(
11940 &handler,
11941 &first_ctx,
11942 &mut first_rx,
11943 hello_frame("first", PROTOCOL_VERSION, 1),
11944 )
11945 .await;
11946 hello_via_sink(
11947 &handler,
11948 &second_ctx,
11949 &mut second_rx,
11950 hello_frame("second", PROTOCOL_VERSION, 2),
11951 )
11952 .await;
11953 let root = unique_project_root("second-only");
11954 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
11955 let cloned = handler.clone();
11956 let open = tokio::spawn(async move {
11957 cloned
11958 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
11959 .await
11960 .unwrap()
11961 });
11962 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
11963 .await
11964 .unwrap()
11965 .unwrap();
11966 let first = query_live_roots(&handler, &first_ctx).await;
11967 let second = query_live_roots(&handler, &second_ctx).await;
11968 assert!(
11969 matches!(
11970 first,
11971 ModuleControlResponseToModule::LiveRoots {
11972 total_bindings: 0,
11973 ..
11974 }
11975 ),
11976 "cross-module scope must not expose another module's roots"
11977 );
11978 assert!(
11979 matches!(
11980 second,
11981 ModuleControlResponseToModule::LiveRoots {
11982 total_bindings: 1,
11983 ..
11984 }
11985 ),
11986 "second module must see its pending route"
11987 );
11988 handler
11989 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
11990 .await
11991 .unwrap();
11992 assert!(open.await.unwrap().is_empty());
11993 }
11994
11995 #[tokio::test(start_paused = true)]
11996 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
11997 let handler = ControlHandler::with_forwarding(
11998 Arc::new(Registry::default()),
11999 Arc::new(ForwardingTable::default()),
12000 );
12001 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
12002 hello_via_sink(
12003 &handler,
12004 &target_ctx,
12005 &mut target_rx,
12006 hello_frame("target", PROTOCOL_VERSION, 1),
12007 )
12008 .await;
12009 let actual = query_live_roots(&handler, &target_ctx).await;
12010 let ModuleControlResponseToModule::LiveRoots {
12011 roots,
12012 unknown_root_bindings,
12013 total_bindings,
12014 } = actual
12015 else {
12016 panic!("expected live roots")
12017 };
12018 assert!(roots.is_empty());
12019 assert_eq!(unknown_root_bindings, 0);
12020 assert_eq!(total_bindings, 0);
12021 assert_eq!(
12022 total_bindings,
12023 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12024 );
12025 }
12026
12027 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
12034 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12035 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
12036 let text = std::fs::read_to_string(&path)
12037 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
12038 let vectors: Vec<(String, Value)> = text
12039 .lines()
12040 .filter(|line| !line.trim().is_empty())
12041 .map(|line| {
12042 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
12043 let id = entry["corpus_id"]
12044 .as_str()
12045 .expect("every vector carries a corpus_id")
12046 .to_string();
12047 (id, entry["package"].clone())
12048 })
12049 .collect();
12050 assert_eq!(
12054 vectors.len(),
12055 3,
12056 "vendored fed corpus changed size; re-sync from subc-federation"
12057 );
12058
12059 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
12074 let richest = vectors
12075 .iter()
12076 .filter_map(|(_, package)| package.as_object())
12077 .map(|object| {
12078 object
12079 .keys()
12080 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
12081 .count()
12082 })
12083 .max()
12084 .unwrap_or(0);
12085 assert!(
12086 richest >= 2,
12087 "vendored corpus no longer carries a package with unmodelled fields, \
12088 so the relay test can no longer distinguish a verbatim relay from a lossy one"
12089 );
12090
12091 vectors
12092 }
12093
12094 #[tokio::test]
12102 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12103 for (corpus_id, package) in fed_admission_facts_vectors() {
12104 let registry = Arc::new(Registry::default());
12105 let forwarding = Arc::new(ForwardingTable::default());
12106 let supervisor = SupervisorHandle::new();
12107 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12108 let handler =
12109 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12110 .with_supervisor(supervisor)
12111 .with_admission_facts_config(
12112 Some("fed".to_string()),
12113 Some(vec!["target".to_string()]),
12114 );
12115
12116 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12117 hello_via_sink(
12118 &handler,
12119 &target_ctx,
12120 &mut target_rx,
12121 hello_frame("target", PROTOCOL_VERSION, 1),
12122 )
12123 .await;
12124
12125 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12126 let route_handler = handler.clone();
12127 let expected = package.clone();
12128 let route_task = tokio::spawn(async move {
12129 route_handler
12130 .handle_control_frame(
12131 &client_ctx,
12132 route_open_frame_with_admission_facts(
12133 20,
12134 "target",
12135 unique_project_root("admission-facts"),
12136 Some(subc_control::ConsumerIdentity {
12137 module_id: "fed".to_string(),
12138 launch_nonce: "fed-nonce".to_string(),
12139 }),
12140 Some(package),
12141 ),
12142 )
12143 .await
12144 .unwrap()
12145 });
12146
12147 let bind_frame = target_rx.recv().await.unwrap();
12148 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12149 let ModuleControlRequest::RouteBind {
12150 admission_facts, ..
12151 } = bind
12152 else {
12153 panic!("{corpus_id}: expected route.bind")
12154 };
12155 assert_eq!(
12156 admission_facts,
12157 Some(expected),
12158 "{corpus_id}: relay must not add, drop or reshape any field"
12159 );
12160
12161 handler
12162 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12163 .await
12164 .unwrap();
12165 route_task.await.unwrap();
12166 }
12167 }
12168
12169 #[tokio::test]
12170 async fn route_bind_never_relays_the_openers_build_class() {
12171 let registry = Arc::new(Registry::default());
12172 let forwarding = Arc::new(ForwardingTable::default());
12173 let handler =
12174 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12175 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
12176 hello_via_sink(
12177 &handler,
12178 &target_ctx,
12179 &mut target_rx,
12180 hello_frame("target", PROTOCOL_VERSION, 1),
12181 )
12182 .await;
12183
12184 let root = unique_project_root("build-class");
12185 let mut identity = BindIdentity::new(
12186 root.path().to_path_buf(),
12187 "unit".to_string(),
12188 "s".to_string(),
12189 );
12190 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
12191 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12192 target: RouteTarget::ToolProvider {
12193 module_id: "target".to_string(),
12194 },
12195 identity,
12196 consumer_identity: None,
12197 consumer_capabilities: None,
12198 role_versions: None,
12199 admission_facts: None,
12200 scope: None,
12201 })
12202 .unwrap();
12203 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
12204
12205 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
12206 let route_handler = handler.clone();
12207 let route_task = tokio::spawn(async move {
12208 route_handler
12209 .handle_control_frame(&client_ctx, open)
12210 .await
12211 .unwrap()
12212 });
12213 let bind_frame = target_rx.recv().await.unwrap();
12214 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12215 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
12216 panic!("expected route.bind")
12217 };
12218 assert_eq!(
12219 identity.build_class, None,
12220 "route.bind carried the build class"
12221 );
12222 assert!(
12223 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
12224 "the route.bind bytes still name build_class"
12225 );
12226 handler
12227 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12228 .await
12229 .unwrap();
12230 route_task.await.unwrap();
12231 }
12232
12233 #[tokio::test]
12234 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
12235 let registry = Arc::new(Registry::default());
12236 let forwarding = Arc::new(ForwardingTable::default());
12237 let supervisor = SupervisorHandle::new();
12238 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12239 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
12240 let handler =
12241 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12242 .with_supervisor(supervisor)
12243 .with_admission_facts_config(
12244 Some("fed".to_string()),
12245 Some(vec!["target".to_string()]),
12246 );
12247
12248 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
12249 hello_via_sink(
12250 &handler,
12251 &target_ctx,
12252 &mut target_rx,
12253 hello_frame("target", PROTOCOL_VERSION, 1),
12254 )
12255 .await;
12256 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
12257 hello_via_sink(
12258 &handler,
12259 &other_ctx,
12260 &mut other_rx,
12261 hello_frame("other", PROTOCOL_VERSION, 2),
12262 )
12263 .await;
12264
12265 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
12266 let expected_facts = facts.clone();
12267 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
12268 let route_handler = handler.clone();
12269 let route_task = tokio::spawn(async move {
12270 route_handler
12271 .handle_control_frame(
12272 &client_ctx,
12273 route_open_frame_with_admission_facts(
12274 10,
12275 "target",
12276 unique_project_root("admission-facts"),
12277 Some(subc_control::ConsumerIdentity {
12278 module_id: "fed".to_string(),
12279 launch_nonce: "fed-nonce".to_string(),
12280 }),
12281 Some(facts.clone()),
12282 ),
12283 )
12284 .await
12285 .unwrap()
12286 });
12287 let bind_frame = target_rx.recv().await.unwrap();
12288 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12289 let ModuleControlRequest::RouteBind {
12290 admission_facts, ..
12291 } = bind
12292 else {
12293 panic!("expected route.bind")
12294 };
12295 assert_eq!(admission_facts, Some(expected_facts));
12296 handler
12297 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12298 .await
12299 .unwrap();
12300 assert!(route_task.await.unwrap().is_empty());
12301 assert!(matches!(
12302 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
12303 .unwrap(),
12304 ClientControlResponse::RouteOpen { .. }
12305 ));
12306
12307 let direct = handler
12308 .handle_control_frame(
12309 &route_ctx(ConnectionId::new(73)).0,
12310 route_open_frame_with_admission_facts(
12311 11,
12312 "target",
12313 unique_project_root("admission-facts"),
12314 None,
12315 Some(json!({"x": 1})),
12316 ),
12317 )
12318 .await
12319 .unwrap();
12320 assert_eq!(
12321 parse_error(&direct[0])["code"],
12322 "admission_facts_not_permitted"
12323 );
12324
12325 let different_reserved = handler
12326 .handle_control_frame(
12327 &route_ctx(ConnectionId::new(77)).0,
12328 route_open_frame_with_admission_facts(
12329 15,
12330 "target",
12331 unique_project_root("admission-facts"),
12332 Some(subc_control::ConsumerIdentity {
12333 module_id: "other".to_string(),
12334 launch_nonce: "other-nonce".to_string(),
12335 }),
12336 Some(json!({"x": 1})),
12337 ),
12338 )
12339 .await
12340 .unwrap();
12341 assert_eq!(
12342 parse_error(&different_reserved[0])["code"],
12343 "admission_facts_not_permitted"
12344 );
12345
12346 let other_target = handler
12347 .handle_control_frame(
12348 &route_ctx(ConnectionId::new(74)).0,
12349 route_open_frame_with_admission_facts(
12350 12,
12351 "other",
12352 unique_project_root("admission-facts"),
12353 Some(subc_control::ConsumerIdentity {
12354 module_id: "fed".to_string(),
12355 launch_nonce: "fed-nonce".to_string(),
12356 }),
12357 Some(json!({"x": 1})),
12358 ),
12359 )
12360 .await
12361 .unwrap();
12362 assert_eq!(
12363 parse_error(&other_target[0])["code"],
12364 "admission_facts_target_not_allowed"
12365 );
12366
12367 let nonexistent = handler
12368 .handle_control_frame(
12369 &route_ctx(ConnectionId::new(75)).0,
12370 route_open_frame_with_admission_facts(
12371 13,
12372 "missing",
12373 unique_project_root("admission-facts"),
12374 None,
12375 Some(json!({"x": 1})),
12376 ),
12377 )
12378 .await
12379 .unwrap();
12380 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12381
12382 let described = handler
12383 .handle_control_frame(
12384 &route_ctx(ConnectionId::new(76)).0,
12385 Frame::build(
12386 FrameType::Request,
12387 control_flags(),
12388 0,
12389 0,
12390 14,
12391 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12392 )
12393 .unwrap(),
12394 )
12395 .await
12396 .unwrap();
12397 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12398 serde_json::from_slice(&described[0].body).unwrap()
12399 else {
12400 panic!("expected server.describe response")
12401 };
12402 assert!(capabilities
12403 .iter()
12404 .any(|cap| cap == "admission_facts_relay_v1"));
12405 }
12406
12407 #[tokio::test]
12408 async fn admission_facts_without_configured_carrier_are_rejected() {
12409 let registry = Arc::new(Registry::default());
12410 let forwarding = Arc::new(ForwardingTable::default());
12411 let handler = ControlHandler::with_forwarding(registry, forwarding);
12412 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12413 hello_via_sink(
12414 &handler,
12415 &target_ctx,
12416 &mut target_rx,
12417 hello_frame("target", PROTOCOL_VERSION, 1),
12418 )
12419 .await;
12420
12421 let responses = handler
12422 .handle_control_frame(
12423 &route_ctx(ConnectionId::new(79)).0,
12424 route_open_frame_with_admission_facts(
12425 16,
12426 "target",
12427 unique_project_root("admission-facts"),
12428 None,
12429 Some(json!({"x": 1})),
12430 ),
12431 )
12432 .await
12433 .unwrap();
12434 assert_eq!(
12435 parse_error(&responses[0])["code"],
12436 "admission_facts_not_permitted"
12437 );
12438 }
12439
12440 #[tokio::test]
12441 async fn route_open_relays_consumer_capabilities_verbatim() {
12442 let registry = Arc::new(Registry::default());
12443 let forwarding = Arc::new(ForwardingTable::default());
12444 let handler =
12445 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12446 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12447 hello_via_sink(
12448 &handler,
12449 &module_ctx,
12450 &mut module_rx,
12451 hello_frame("aft", PROTOCOL_VERSION, 7),
12452 )
12453 .await;
12454
12455 let expected = vec!["elicitation".to_string(), "roots".to_string()];
12456 let expected_for_request = expected.clone();
12457 let project_root = unique_project_root("consumer-capabilities-present");
12458 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12459 let route_handler = handler.clone();
12460 let route_task = tokio::spawn(async move {
12461 route_handler
12462 .handle_control_frame(
12463 &client_ctx,
12464 route_open_frame_with_consumer_capabilities(
12465 401,
12466 "aft",
12467 project_root,
12468 Some(expected_for_request),
12469 ),
12470 )
12471 .await
12472 .unwrap()
12473 });
12474 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12475 .await
12476 .unwrap()
12477 .unwrap();
12478 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12479 let ModuleControlRequest::RouteBind {
12480 consumer_capabilities,
12481 ..
12482 } = bind
12483 else {
12484 panic!("expected route.bind request, got {bind:?}");
12485 };
12486 assert_eq!(consumer_capabilities, Some(expected.clone()));
12487
12488 handler
12489 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12490 .await
12491 .unwrap();
12492 let route_response = route_task.await.unwrap();
12493 assert!(route_response.is_empty());
12494 let published = client_rx.recv().await.unwrap();
12495 assert!(matches!(
12496 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12497 ClientControlResponse::RouteOpen { .. }
12498 ));
12499 }
12500
12501 #[tokio::test]
12502 async fn route_open_without_consumer_capabilities_relays_none() {
12503 let registry = Arc::new(Registry::default());
12504 let forwarding = Arc::new(ForwardingTable::default());
12505 let handler =
12506 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12507 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12508 hello_via_sink(
12509 &handler,
12510 &module_ctx,
12511 &mut module_rx,
12512 hello_frame("aft", PROTOCOL_VERSION, 7),
12513 )
12514 .await;
12515
12516 let project_root = unique_project_root("consumer-capabilities-absent");
12517 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12518 let route_handler = handler.clone();
12519 let route_task = tokio::spawn(async move {
12520 route_handler
12521 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12522 .await
12523 .unwrap()
12524 });
12525 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12526 .await
12527 .unwrap()
12528 .unwrap();
12529 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12530 let ModuleControlRequest::RouteBind {
12531 consumer_capabilities,
12532 ..
12533 } = bind
12534 else {
12535 panic!("expected route.bind request, got {bind:?}");
12536 };
12537 assert_eq!(consumer_capabilities, None);
12538
12539 handler
12540 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12541 .await
12542 .unwrap();
12543 let route_response = route_task.await.unwrap();
12544 assert!(route_response.is_empty());
12545 let published = client_rx.recv().await.unwrap();
12546 assert!(matches!(
12547 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12548 ClientControlResponse::RouteOpen { .. }
12549 ));
12550 }
12551
12552 async fn bind_role_versions_for(
12556 sent: Option<BTreeMap<String, String>>,
12557 connection: u64,
12558 ) -> Option<BTreeMap<String, String>> {
12559 let registry = Arc::new(Registry::default());
12560 let forwarding = Arc::new(ForwardingTable::default());
12561 let handler =
12562 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12563 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12564 hello_via_sink(
12565 &handler,
12566 &module_ctx,
12567 &mut module_rx,
12568 hello_frame("aft", PROTOCOL_VERSION, 7),
12569 )
12570 .await;
12571 let project_root = unique_project_root("role-versions");
12572 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12573 let route_handler = handler.clone();
12574 let route_task = tokio::spawn(async move {
12575 route_handler
12576 .handle_control_frame(
12577 &client_ctx,
12578 route_open_frame_with_role_versions(403, "aft", project_root, sent),
12579 )
12580 .await
12581 .unwrap()
12582 });
12583 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12584 .await
12585 .expect("a well-formed route.open reaches the module as a bind")
12586 .unwrap();
12587 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12588 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12589 panic!("expected route.bind request, got {bind:?}");
12590 };
12591 handler
12592 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12593 .await
12594 .unwrap();
12595 assert!(route_task.await.unwrap().is_empty());
12596 let published = client_rx.recv().await.unwrap();
12597 assert!(matches!(
12598 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12599 ClientControlResponse::RouteOpen { .. }
12600 ));
12601 role_versions
12602 }
12603
12604 #[tokio::test]
12605 async fn route_open_relays_role_versions_verbatim() {
12606 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12607 assert_eq!(
12608 bind_role_versions_for(Some(sent.clone()), 141).await,
12609 Some(sent)
12610 );
12611 }
12612
12613 #[tokio::test]
12616 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12617 assert_eq!(bind_role_versions_for(None, 143).await, None);
12618 assert_eq!(
12619 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12620 None
12621 );
12622 }
12623
12624 #[tokio::test]
12625 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12626 let registry = Arc::new(Registry::default());
12627 let forwarding = Arc::new(ForwardingTable::default());
12628 let handler =
12629 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12630 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12631 hello_via_sink(
12632 &handler,
12633 &module_ctx,
12634 &mut module_rx,
12635 hello_frame("aft", PROTOCOL_VERSION, 7),
12636 )
12637 .await;
12638
12639 let nine: BTreeMap<String, String> = (0..9)
12640 .map(|index| (format!("role-{index}"), "v1".to_string()))
12641 .collect();
12642 for (label, malformed) in [
12643 (
12644 "invalid role name",
12645 role_versions(&[("Tool_Provider", "v1")]),
12646 ),
12647 ("invalid version", role_versions(&[("tool-provider", "v0")])),
12648 ("nine entries", nine),
12649 ] {
12650 let responses = tokio::time::timeout(
12653 Duration::from_secs(1),
12654 handler.handle_control_frame(
12655 &route_ctx(ConnectionId::new(148)).0,
12656 route_open_frame_with_role_versions(
12657 404,
12658 "aft",
12659 unique_project_root("role-versions-malformed"),
12660 Some(malformed),
12661 ),
12662 ),
12663 )
12664 .await
12665 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12666 .unwrap();
12667 assert_eq!(responses.len(), 1, "{label}");
12668 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12669 let error = parse_error(&responses[0]);
12670 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12671 assert_eq!(
12672 error["detail"]["field"], "role_versions",
12673 "{label}: {error}"
12674 );
12675 assert!(
12676 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12677 "{label}: a malformed declaration is terminal"
12678 );
12679 assert!(
12680 module_rx.try_recv().is_err(),
12681 "{label}: the module must never see a bind"
12682 );
12683 }
12684 }
12685
12686 #[tokio::test]
12690 async fn route_role_versions_capability_is_advertised() {
12691 let handler = ControlHandler::new(Arc::new(Registry::default()));
12692 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12693 let ack = hello_via_sink(
12694 &handler,
12695 &ctx,
12696 &mut rx,
12697 hello_frame("m", PROTOCOL_VERSION, 1),
12698 )
12699 .await;
12700 let ack = parse_ack(&ack);
12701 assert!(
12702 ack.subc_capabilities
12703 .iter()
12704 .any(|c| c == "route-role-versions/v1"),
12705 "{:?}",
12706 ack.subc_capabilities
12707 );
12708
12709 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12710 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12711 let reply = handler
12712 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
12713 .await
12714 .unwrap()
12715 .pop()
12716 .unwrap();
12717 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12718 serde_json::from_slice(&reply.body).unwrap()
12719 else {
12720 panic!("not a server.describe reply");
12721 };
12722 assert!(
12723 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12724 "{capabilities:?}"
12725 );
12726 }
12727
12728 #[tokio::test]
12732 async fn catalog_events_capability_is_advertised() {
12733 let handler = ControlHandler::new(Arc::new(Registry::default()));
12734 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12735 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12736 let reply = handler
12737 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12738 .await
12739 .unwrap()
12740 .pop()
12741 .unwrap();
12742 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12743 serde_json::from_slice(&reply.body).unwrap()
12744 else {
12745 panic!("not a server.describe reply");
12746 };
12747 assert!(
12748 capabilities
12749 .iter()
12750 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12751 "{capabilities:?}"
12752 );
12753 }
12754
12755 #[tokio::test]
12756 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12757 let handler = ControlHandler::new(Arc::new(Registry::default()));
12758 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12759 let ack = hello_via_sink(
12760 &handler,
12761 &module,
12762 &mut module_rx,
12763 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12764 )
12765 .await;
12766 let ack = parse_ack(&ack);
12767 assert!(
12768 ack.subc_capabilities
12769 .iter()
12770 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12771 "{:?}",
12772 ack.subc_capabilities
12773 );
12774 assert!(ack
12775 .subc_capabilities
12776 .iter()
12777 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12778
12779 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12780 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12781 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12782 let reply = handler
12783 .handle_control_frame(&client, frame)
12784 .await
12785 .unwrap()
12786 .pop()
12787 .unwrap();
12788 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12789 serde_json::from_slice(&reply.body).unwrap()
12790 else {
12791 panic!("not a server.describe reply");
12792 };
12793 assert!(
12794 capabilities
12795 .iter()
12796 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12797 "{capabilities:?}"
12798 );
12799 assert!(capabilities
12800 .iter()
12801 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12802 }
12803
12804 #[tokio::test]
12805 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12806 let handler = ControlHandler::new(Arc::new(Registry::default()));
12807 let (module, mut rx) = route_ctx(ConnectionId::new(156));
12808 let ack = hello_via_sink(
12809 &handler,
12810 &module,
12811 &mut rx,
12812 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12813 )
12814 .await;
12815 assert!(parse_ack(&ack)
12816 .subc_capabilities
12817 .iter()
12818 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12819 assert!(parse_ack(&ack)
12820 .subc_capabilities
12821 .iter()
12822 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12823 let (client, _rx) = route_ctx(ConnectionId::new(157));
12824 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12825 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12826 let reply = handler
12827 .handle_control_frame(&client, frame)
12828 .await
12829 .unwrap()
12830 .pop()
12831 .unwrap();
12832 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12833 serde_json::from_slice(&reply.body).unwrap()
12834 else {
12835 panic!("server.describe reply")
12836 };
12837 assert!(capabilities
12838 .iter()
12839 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12840 assert!(capabilities
12841 .iter()
12842 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12843 }
12844
12845 #[tokio::test]
12846 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12847 let handler = ControlHandler::new(Arc::new(Registry::default()));
12848 let (module, mut rx) = route_ctx(ConnectionId::new(158));
12849 let ack = hello_via_sink(
12850 &handler,
12851 &module,
12852 &mut rx,
12853 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12854 )
12855 .await;
12856 assert!(parse_ack(&ack)
12857 .subc_capabilities
12858 .iter()
12859 .any(|cap| cap == "scope-workdir/v1"));
12860 let (client, _rx) = route_ctx(ConnectionId::new(159));
12861 let frame = Frame::build(
12862 FrameType::Request,
12863 control_flags(),
12864 0,
12865 0,
12866 5,
12867 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12868 )
12869 .unwrap();
12870 let reply = handler
12871 .handle_control_frame(&client, frame)
12872 .await
12873 .unwrap()
12874 .remove(0);
12875 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12876 serde_json::from_slice(&reply.body).unwrap()
12877 else {
12878 panic!("server.describe reply")
12879 };
12880 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
12881 }
12882
12883 #[tokio::test]
12884 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
12885 let handler = ControlHandler::new(Arc::new(Registry::default()));
12886 let (module, mut rx) = route_ctx(ConnectionId::new(160));
12887 let ack = hello_via_sink(
12888 &handler,
12889 &module,
12890 &mut rx,
12891 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
12892 )
12893 .await;
12894 assert!(parse_ack(&ack)
12895 .subc_capabilities
12896 .iter()
12897 .any(|cap| cap == "worker-runs/v1"));
12898 let (client, _rx) = route_ctx(ConnectionId::new(161));
12899 let frame = Frame::build(
12900 FrameType::Request,
12901 control_flags(),
12902 0,
12903 0,
12904 5,
12905 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12906 )
12907 .unwrap();
12908 let reply = handler
12909 .handle_control_frame(&client, frame)
12910 .await
12911 .unwrap()
12912 .remove(0);
12913 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12914 serde_json::from_slice(&reply.body).unwrap()
12915 else {
12916 panic!("server.describe reply")
12917 };
12918 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
12919 }
12920
12921 #[tokio::test]
12922 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
12923 let handler = ControlHandler::new(Arc::new(Registry::default()));
12924 let (module, mut rx) = route_ctx(ConnectionId::new(154));
12925 let ack = hello_via_sink(
12926 &handler,
12927 &module,
12928 &mut rx,
12929 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
12930 )
12931 .await;
12932 assert!(parse_ack(&ack)
12933 .subc_capabilities
12934 .iter()
12935 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12936 let (client, _rx) = route_ctx(ConnectionId::new(155));
12937 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12938 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12939 let reply = handler
12940 .handle_control_frame(&client, frame)
12941 .await
12942 .unwrap()
12943 .pop()
12944 .unwrap();
12945 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12946 serde_json::from_slice(&reply.body).unwrap()
12947 else {
12948 panic!("not a server.describe reply")
12949 };
12950 assert!(capabilities
12951 .iter()
12952 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
12953 }
12954
12955 #[tokio::test]
12956 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
12957 let registry = Arc::new(Registry::default());
12958 let forwarding = Arc::new(ForwardingTable::default());
12959 let handler =
12960 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12961 .with_health_probe_timeout(Duration::from_secs(5));
12962 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
12963 hello_via_sink(
12964 &handler,
12965 &module_ctx,
12966 &mut module_rx,
12967 non_routable_hello_frame_with_control_ops(
12968 "mcp",
12969 300,
12970 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12971 ),
12972 )
12973 .await;
12974 assert!(registry
12975 .get_module("mcp")
12976 .unwrap()
12977 .unwrap()
12978 .manifest
12979 .provides
12980 .is_empty());
12981
12982 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
12983 let route_response = handler
12984 .handle_control_frame(
12985 &route_client_ctx,
12986 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
12987 )
12988 .await
12989 .unwrap();
12990 assert_eq!(route_response[0].header.ty, FrameType::Error);
12991 assert_eq!(
12992 parse_error(&route_response[0])["code"],
12993 "target_unavailable"
12994 );
12995 assert!(parse_error(&route_response[0])["message"]
12996 .as_str()
12997 .unwrap()
12998 .contains("does not provide the requested target"));
12999 assert!(module_rx.try_recv().is_err());
13000
13001 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
13002 let health_handler = handler.clone();
13003 let health_task = tokio::spawn(async move {
13004 health_handler
13005 .handle_control_frame(
13006 &health_client_ctx,
13007 supervisor_health_probe_frame(302, "mcp"),
13008 )
13009 .await
13010 .unwrap()
13011 });
13012 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13013 .await
13014 .unwrap()
13015 .unwrap();
13016 assert_eq!(
13017 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13018 ModuleControlRequest::HealthCheck {}
13019 );
13020 handler
13021 .handle_control_frame(
13022 &module_ctx,
13023 health_response(health_frame.header.corr, HealthStatus::Ok),
13024 )
13025 .await
13026 .unwrap();
13027 let health_response = health_task.await.unwrap();
13028 assert_eq!(health_response[0].header.ty, FrameType::Response);
13029 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13030 ClientControlResponse::SupervisorHealthProbe {
13031 module_id, status, ..
13032 } => {
13033 assert_eq!(module_id, "mcp");
13034 assert_eq!(status, HealthStatus::Ok);
13035 }
13036 other => panic!("unexpected health response: {other:?}"),
13037 }
13038
13039 forwarding
13044 .cleanup_connection(module_ctx.connection_id)
13045 .unwrap();
13046 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
13047 let cleanup_response = tokio::time::timeout(
13048 Duration::from_millis(200),
13049 handler.handle_control_frame(
13050 &cleanup_probe_ctx,
13051 supervisor_health_probe_frame(303, "mcp"),
13052 ),
13053 )
13054 .await
13055 .expect("probe should fail immediately when the control lane is gone")
13056 .unwrap();
13057 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
13058 assert_eq!(
13059 parse_error(&cleanup_response[0])["code"],
13060 "target_unavailable"
13061 );
13062 assert!(parse_error(&cleanup_response[0])["message"]
13063 .as_str()
13064 .unwrap()
13065 .contains("no module connection"));
13066
13067 handler
13068 .cleanup_connection(module_ctx.connection_id)
13069 .unwrap();
13070 }
13071
13072 #[tokio::test]
13073 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
13074 let registry = Arc::new(Registry::default());
13075 let supervisor_handle = SupervisorHandle::new();
13076 let supervisor =
13077 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13078 .with_handle(supervisor_handle.clone())
13079 .with_connection_file_path(
13080 std::env::temp_dir()
13081 .join(format!("subc-route-open-warming-{}", std::process::id())),
13082 );
13083 let module = supervisor
13084 .supervise_configured(
13085 ModuleSpec {
13086 module_id: "warming".to_string(),
13087 program: fake_aft_stub_path(),
13088 args: Vec::new(),
13089 env: Vec::new(),
13090 reserved: false,
13091 reserved_prefixes: Vec::new(),
13092 protocol: ModuleProtocol::Subc,
13093 overlap: Default::default(),
13094 },
13095 true,
13096 )
13097 .unwrap();
13098 assert_eq!(module.state().unwrap(), ModuleState::Running);
13099
13100 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13101 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13102 let response = handler
13103 .handle_control_frame(
13104 &ctx,
13105 route_open_frame(304, "warming", unique_project_root("warming")),
13106 )
13107 .await
13108 .unwrap();
13109 module.stop().await.unwrap();
13110
13111 assert_eq!(response[0].header.ty, FrameType::Error);
13112 let error = parse_error(&response[0]);
13113 assert_eq!(error["code"], "module_warming");
13114 assert!(error["message"]
13115 .as_str()
13116 .unwrap()
13117 .contains("state=running, enabled=true, live=false"));
13118 }
13119
13120 #[test]
13121 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13122 let handler = ControlHandler::new(Arc::new(Registry::default()));
13123 let capture = EventCapture::default();
13124 let _subscriber =
13125 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13126 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13127 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
13128 let pending = (0..limit).collect::<Vec<_>>();
13129 let response = handler
13130 .route_open_capacity_refusal(
13131 &ctx,
13132 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
13133 "busy",
13134 pending.len(),
13135 limit,
13136 )
13137 .unwrap();
13138 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13139 let event = capture
13140 .events()
13141 .into_iter()
13142 .find(|event| {
13143 event.target == "control"
13144 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
13145 })
13146 .expect("connection admission refusal event");
13147 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13148 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13149 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13152 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13153 }
13154
13155 #[test]
13156 fn route_open_target_cap_logs_admission_reason_and_capacity() {
13157 let handler = ControlHandler::new(Arc::new(Registry::default()));
13158 let capture = EventCapture::default();
13159 let _subscriber =
13160 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13161 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
13162 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
13163 let guards = (0..limit)
13164 .map(|_| {
13165 handler
13166 .route_bind_concurrency
13167 .try_admit("busy", limit)
13168 .unwrap()
13169 })
13170 .collect::<Vec<_>>();
13171 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
13172 Err(in_flight) => in_flight,
13173 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
13174 };
13175 let response = handler
13176 .route_open_target_capacity_refusal(
13177 &ctx,
13178 &RouteOpenCaller::unknown(),
13179 &route_open_frame(397, "busy", unique_project_root("target-cap")),
13180 "busy",
13181 in_flight,
13182 )
13183 .unwrap();
13184 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13185 let event = capture
13186 .events()
13187 .into_iter()
13188 .find(|event| {
13189 event.target == "control"
13190 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
13191 })
13192 .expect("target admission refusal event");
13193 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13194 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13195 drop(guards);
13196 }
13197
13198 #[cfg(unix)]
13199 #[tokio::test(start_paused = true)]
13200 async fn supervisor_list_shows_a_failed_modules_next_retry() {
13201 let registry = Arc::new(Registry::default());
13202 let handle = SupervisorHandle::new();
13203 let supervisor =
13204 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
13205 .with_handle(handle.clone());
13206 let module = supervisor
13207 .spawn(crate::ModuleSpec {
13208 module_id: "retry-list".into(),
13209 program: PathBuf::from("/bin/sh"),
13210 args: vec!["-c".into(), "exit 17".into()],
13211 env: vec![],
13212 reserved: false,
13213 reserved_prefixes: vec![],
13214 protocol: ModuleProtocol::None,
13215 overlap: Default::default(),
13216 })
13217 .unwrap();
13218 let deadline = std::time::Instant::now() + Duration::from_secs(10);
13219 while module.state().unwrap() != crate::ModuleState::Failed {
13220 assert!(
13221 std::time::Instant::now() < deadline,
13222 "waiting for budget failure"
13223 );
13224 tokio::task::yield_now().await;
13225 }
13226 let handler = ControlHandler::new(registry).with_supervisor(handle);
13227 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13228 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13229 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13230 assert_eq!(body["modules"][0]["state"], "failed");
13231 let retry = body["modules"][0]["next_retry_at_ms"]
13232 .as_u64()
13233 .expect("scheduled retry on the wire");
13234 let now = std::time::SystemTime::now()
13235 .duration_since(std::time::UNIX_EPOCH)
13236 .unwrap()
13237 .as_millis() as u64;
13238 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
13239 module.set_enabled(false).await.unwrap();
13240 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13241 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13242 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13243 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
13244 }
13245
13246 #[tokio::test]
13247 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
13248 let registry = Arc::new(Registry::default());
13249 let handle = SupervisorHandle::new();
13250 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
13251 .with_handle(handle.clone());
13252 supervisor
13253 .supervise_configured(
13254 crate::ModuleSpec {
13255 module_id: "drift".into(),
13256 program: PathBuf::from("disabled-module"),
13257 args: Vec::new(),
13258 env: Vec::new(),
13259 reserved: false,
13260 reserved_prefixes: Vec::new(),
13261 protocol: ModuleProtocol::Subc,
13262 overlap: Default::default(),
13263 },
13264 false,
13265 )
13266 .unwrap();
13267 let handler = ControlHandler::new(registry).with_supervisor(handle);
13268 for configured in [None, Some(false), Some(true)] {
13269 handler.capability_evaluator.configure(
13270 configured.map(|enabled| ("drift".to_string(), enabled)),
13271 BTreeMap::new(),
13272 );
13273 let frame =
13274 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13275 let replies = handler
13276 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
13277 .await
13278 .unwrap();
13279 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13280 assert!(body.get("configured_enabled").is_none());
13281 assert_eq!(body["op"], "supervisor.ack");
13282 let frame =
13283 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13284 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13285 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13286 assert_eq!(
13287 body["modules"][0].get("configured_enabled"),
13288 configured.map(Value::Bool).as_ref()
13289 );
13290 }
13291 }
13292
13293 #[tokio::test(flavor = "current_thread")]
13294 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
13295 let handler = ControlHandler::new(Arc::new(Registry::default()));
13296 let capture = EventCapture::default();
13297 let _subscriber =
13298 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13299 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
13300 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13301
13302 let _ = handler
13303 .handle_client_control_request(
13304 &ctx,
13305 frame,
13306 ClientControlRequest::SupervisorSetEnabled {
13307 module_id: "broca".to_string(),
13308 enabled: false,
13309 },
13310 &mut None,
13311 )
13312 .await
13313 .unwrap();
13314
13315 let events = capture
13316 .events()
13317 .into_iter()
13318 .filter(|event| {
13319 event.target == "control"
13320 && event.fields.get("message").map(String::as_str)
13321 == Some("supervisor request received")
13322 })
13323 .collect::<Vec<_>>();
13324 assert_eq!(events.len(), 1, "one supervisor request log line");
13325 let event = &events[0];
13326 assert_eq!(event.level, tracing::Level::INFO);
13327 assert_eq!(
13328 event.fields.get("op"),
13329 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
13330 );
13331 assert_eq!(
13332 event.fields.get("module_id"),
13333 Some(&"\"broca\"".to_string())
13334 );
13335 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
13336 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
13337 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
13338 }
13339
13340 #[tokio::test(flavor = "current_thread")]
13341 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
13342 let registry = Arc::new(Registry::default());
13343 let module_connection = ConnectionId::new(303);
13344 registry
13345 .register_with_control_ops(
13346 manifest("aft", PROTOCOL_VERSION),
13347 PROTOCOL_VERSION,
13348 module_connection,
13349 module_baseline_control_ops(),
13350 )
13351 .unwrap();
13352 let handler = ControlHandler::new(registry);
13353 let capture = EventCapture::default();
13354 let _subscriber =
13355 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13356 let (ctx, _rx) = route_ctx(module_connection);
13357 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13358
13359 let _ = handler
13360 .handle_client_control_request(
13361 &ctx,
13362 frame,
13363 ClientControlRequest::SupervisorSetEnabled {
13364 module_id: "broca".to_string(),
13365 enabled: false,
13366 },
13367 &mut None,
13368 )
13369 .await
13370 .unwrap();
13371
13372 let callers = capture
13373 .events()
13374 .into_iter()
13375 .filter(|event| {
13376 event.fields.get("message").map(String::as_str)
13377 == Some("supervisor request received")
13378 })
13379 .map(|event| event.fields.get("caller").cloned())
13380 .collect::<Vec<_>>();
13381 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
13382 }
13383
13384 #[tokio::test(flavor = "current_thread")]
13385 async fn supervisor_list_does_not_log_request_received() {
13386 let handler = ControlHandler::new(Arc::new(Registry::default()));
13387 let capture = EventCapture::default();
13388 let _subscriber =
13389 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13390 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
13391 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13392
13393 handler
13394 .handle_client_control_request(
13395 &ctx,
13396 frame,
13397 ClientControlRequest::SupervisorList {},
13398 &mut None,
13399 )
13400 .await
13401 .unwrap();
13402
13403 assert!(capture.events().into_iter().all(|event| {
13404 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13405 }));
13406 }
13407
13408 #[tokio::test(flavor = "current_thread")]
13409 async fn supervisor_rescan_preview_does_not_log_request_received() {
13410 let handler = ControlHandler::new(Arc::new(Registry::default()));
13411 let capture = EventCapture::default();
13412 let _subscriber =
13413 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13414 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
13415 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13416
13417 handler
13418 .handle_client_control_request(
13419 &ctx,
13420 frame,
13421 ClientControlRequest::SupervisorRescan { preview: true },
13422 &mut None,
13423 )
13424 .await
13425 .unwrap();
13426
13427 assert!(capture.events().into_iter().all(|event| {
13428 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13429 }));
13430 }
13431
13432 struct ApproveOperator;
13433 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
13434 fn prompt(
13435 &self,
13436 _: &str,
13437 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
13438 ) -> crate::operator_confirm::ProviderResult {
13439 crate::operator_confirm::ProviderResult::Approved
13440 }
13441 }
13442
13443 #[tokio::test]
13444 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
13445 assert_eq!(
13446 ControlHandler::default().privileged_operator_prompt,
13447 PrivilegedOperatorPrompt::Log
13448 );
13449 for (initial, changed, running) in [
13450 ("log", "enforce", PrivilegedOperatorPrompt::Log),
13451 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
13452 ] {
13453 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
13454 let path = dir.join("subc.jsonc");
13455 std::fs::write(
13456 &path,
13457 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
13458 )
13459 .unwrap();
13460 let config = crate::daemon_config::load(&path).unwrap().unwrap();
13461 let registry = Arc::new(Registry::default());
13462 let handle = SupervisorHandle::new();
13463 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13464 .with_handle(handle.clone());
13465 let handler = ControlHandler::new(registry)
13466 .with_supervisor(handle)
13467 .with_privileged_operator_prompt(config.privileged_operator_prompt)
13468 .with_supervisor_rescan(supervisor, &path, config.port);
13469 handler.forwarding.operator_confirms().configure(
13470 Arc::new(ApproveOperator),
13471 crate::operator_confirm::OperatorLimits::default(),
13472 );
13473 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13474
13475 for (disk_mode, needs_restart) in [
13476 (Some(initial), false),
13477 (Some(changed), true),
13478 (Some(changed), true),
13479 (None, running == PrivilegedOperatorPrompt::Enforce),
13480 (Some(initial), false),
13481 ] {
13482 let mut doc = json!({"version":1});
13483 if let Some(mode) = disk_mode {
13484 doc["privileged_operator_prompt"] = json!(mode);
13485 }
13486 std::fs::write(&path, doc.to_string()).unwrap();
13487 for preview in [true, false] {
13488 let request = ClientControlRequest::SupervisorRescan { preview };
13489 let frame = Frame::build(
13490 FrameType::Request,
13491 control_flags(),
13492 0,
13493 0,
13494 1,
13495 serde_json::to_vec(&request).unwrap(),
13496 )
13497 .unwrap();
13498 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13499 assert_eq!(frames.len(), 1);
13500 let ClientControlResponse::SupervisorRescan { result } =
13501 serde_json::from_slice(&frames[0].body).unwrap()
13502 else {
13503 panic!("expected a rescan response");
13504 };
13505 let expected = if needs_restart {
13506 vec!["privileged_operator_prompt"]
13507 } else {
13508 Vec::new()
13509 };
13510 assert_eq!(result.restart_required, expected);
13511 assert_eq!(handler.privileged_operator_prompt, running);
13512 assert_eq!(
13513 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13514 running
13515 );
13516 }
13517 }
13518 }
13519 }
13520
13521 #[tokio::test]
13522 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13523 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13524 let path = dir.join("subc.jsonc");
13525 let registry = Arc::new(Registry::default());
13526 let handle = SupervisorHandle::new();
13527 let supervisor =
13528 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13529 let handler = ControlHandler::new(registry)
13530 .with_supervisor(handle)
13531 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13532 .with_supervisor_rescan(supervisor, &path, None);
13533 handler.forwarding.operator_confirms().configure(
13534 Arc::new(ApproveOperator),
13535 crate::operator_confirm::OperatorLimits::default(),
13536 );
13537 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13538 for preview in [true, false] {
13539 std::fs::write(
13540 &path,
13541 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13542 )
13543 .unwrap();
13544 let request = ClientControlRequest::SupervisorRescan { preview };
13545 let frame = Frame::build(
13546 FrameType::Request,
13547 control_flags(),
13548 0,
13549 0,
13550 1,
13551 serde_json::to_vec(&request).unwrap(),
13552 )
13553 .unwrap();
13554 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13555 assert_eq!(frames.len(), 1);
13556 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13557 assert_eq!(
13558 handler.privileged_operator_prompt,
13559 PrivilegedOperatorPrompt::Enforce
13560 );
13561 }
13562 }
13563
13564 #[tokio::test]
13569 async fn route_open_refusal_names_the_check_that_refused() {
13570 let handler = ControlHandler::new(Arc::new(Registry::default()));
13571 let capture = EventCapture::default();
13572 let _subscriber =
13573 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13574 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13575 let response = handler
13576 .handle_control_frame(
13577 &ctx,
13578 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13579 )
13580 .await
13581 .unwrap();
13582
13583 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13584 let event = capture
13585 .events()
13586 .into_iter()
13587 .find(|event| {
13588 event.target == "control"
13589 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13590 })
13591 .expect("route.open refusal event");
13592 assert_eq!(
13593 event.fields.get("reason"),
13594 Some(&"\"not_registered\"".to_string())
13595 );
13596 }
13597
13598 #[tokio::test]
13599 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13600 let registry = Arc::new(Registry::default());
13601 let supervisor_handle = SupervisorHandle::new();
13602 let supervisor =
13603 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13604 .with_handle(supervisor_handle.clone())
13605 .with_connection_file_path(std::env::temp_dir().join(format!(
13606 "subc-route-open-refusal-info-{}",
13607 std::process::id()
13608 )));
13609 let module = supervisor
13610 .supervise_configured(
13611 ModuleSpec {
13612 module_id: "warming".to_string(),
13613 program: fake_aft_stub_path(),
13614 args: Vec::new(),
13615 env: Vec::new(),
13616 reserved: false,
13617 reserved_prefixes: Vec::new(),
13618 protocol: ModuleProtocol::Subc,
13619 overlap: Default::default(),
13620 },
13621 true,
13622 )
13623 .unwrap();
13624 assert_eq!(module.state().unwrap(), ModuleState::Running);
13625
13626 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13627 assert!(handler
13628 .counters()
13629 .snapshot()
13630 .get("route_open_refused_by_code")
13631 .is_none());
13632 let capture = EventCapture::default();
13633 let _subscriber =
13634 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13635 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13636 let response = handler
13637 .handle_control_frame(
13638 &ctx,
13639 route_open_frame(394, "warming", unique_project_root("refusal-info")),
13640 )
13641 .await
13642 .unwrap();
13643 module.stop().await.unwrap();
13644
13645 assert_eq!(parse_error(&response[0])["code"], "module_warming");
13646 let event = capture
13647 .events()
13648 .into_iter()
13649 .find(|event| {
13650 event.target == "control"
13651 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13652 })
13653 .expect("route.open refusal event");
13654 assert_eq!(
13655 event.fields.get("module_id"),
13656 Some(&"\"warming\"".to_string())
13657 );
13658 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13659 assert_eq!(
13660 event.fields.get("reason"),
13661 Some(&"\"supervised_not_registered\"".to_string())
13662 );
13663 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13664 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13665 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13666 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13667 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13668 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13669 assert_eq!(
13670 handler.counters().snapshot()["route_open_refused_by_code"],
13671 json!({ "module_warming": 1 })
13672 );
13673 }
13674
13675 const OUTAGE_START: &str = "route.open refusing module: not serving";
13676 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13677
13678 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13679 capture
13680 .events()
13681 .into_iter()
13682 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13683 .collect()
13684 }
13685
13686 fn supervise_stub(
13687 registry: &Arc<Registry>,
13688 module_id: &str,
13689 enabled: bool,
13690 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13691 let supervisor_handle = SupervisorHandle::new();
13692 let supervisor =
13693 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
13694 .with_handle(supervisor_handle.clone())
13695 .with_connection_file_path(std::env::temp_dir().join(format!(
13696 "subc-route-outage-{module_id}-{}",
13697 std::process::id()
13698 )));
13699 let module = supervisor
13700 .supervise_configured(
13701 ModuleSpec {
13702 module_id: module_id.to_string(),
13703 program: fake_aft_stub_path(),
13704 args: Vec::new(),
13705 env: Vec::new(),
13706 reserved: false,
13707 reserved_prefixes: Vec::new(),
13708 protocol: ModuleProtocol::Subc,
13709 overlap: Default::default(),
13710 },
13711 enabled,
13712 )
13713 .unwrap();
13714 (supervisor_handle, module)
13715 }
13716
13717 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
13718 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
13719 module_id: module_id.to_string(),
13720 drain_timeout_ms: Some(50),
13721 })
13722 .unwrap();
13723 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
13724 }
13725
13726 #[test]
13730 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
13731 let registry = Arc::new(Registry::default());
13732 let forwarding = Arc::new(ForwardingTable::default());
13733 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13734 let second = ControlHandler::with_forwarding(registry, forwarding);
13735 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
13736 }
13737
13738 #[tokio::test(flavor = "current_thread")]
13742 async fn route_open_unknown_module_refusals_add_no_outage_state() {
13743 let handler = ControlHandler::new(Arc::new(Registry::default()));
13744 let capture = EventCapture::default();
13745 let _subscriber =
13746 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13747 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
13748 for corr in 0..8 {
13749 let response = handler
13750 .handle_control_frame(
13751 &ctx,
13752 route_open_frame(
13753 380 + corr,
13754 &format!("nobody-{corr}"),
13755 unique_project_root("outage-unknown"),
13756 ),
13757 )
13758 .await
13759 .unwrap();
13760 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13761 }
13762
13763 assert_eq!(handler.route_outages.tracked_module_count(), 0);
13764 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
13765 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
13766 }
13767
13768 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13773 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
13774 let registry = Arc::new(Registry::default());
13775 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
13776 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13777 let capture = EventCapture::default();
13778 let _subscriber =
13779 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13780 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
13781 handler.route_outages.record_accepted("outage-restart");
13784
13785 let response = handler
13786 .handle_control_frame(
13787 &ctx,
13788 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
13789 )
13790 .await
13791 .unwrap();
13792 assert_eq!(response[0].header.ty, FrameType::Error);
13793 let starts = outage_lines(&capture, OUTAGE_START);
13794 assert_eq!(starts.len(), 1, "{starts:?}");
13795 assert_eq!(starts[0].level, tracing::Level::WARN);
13796 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
13797 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
13798 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
13799 handler.route_outages.record_accepted("outage-restart");
13800 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
13801
13802 let restart = handler
13803 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
13804 .await
13805 .unwrap();
13806 assert_eq!(
13807 restart[0].header.ty,
13808 FrameType::Response,
13809 "{:?}",
13810 parse_error(&restart[0])
13811 );
13812 handler
13813 .handle_control_frame(
13814 &ctx,
13815 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13816 )
13817 .await
13818 .unwrap();
13819 module.stop().await.unwrap();
13820
13821 let starts = outage_lines(&capture, OUTAGE_START);
13822 assert_eq!(starts.len(), 2, "{starts:?}");
13823 assert_eq!(starts[1].level, tracing::Level::INFO);
13824 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13825 }
13826
13827 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13831 async fn failed_operator_restart_leaves_no_operator_mark() {
13832 let registry = Arc::new(Registry::default());
13833 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
13834 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13835 let capture = EventCapture::default();
13836 let _subscriber =
13837 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13838 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13839 handler.route_outages.record_accepted("outage-disabled");
13840
13841 let restart = handler
13842 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13843 .await
13844 .unwrap();
13845 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13846 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13847
13848 handler
13849 .handle_control_frame(
13850 &ctx,
13851 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13852 )
13853 .await
13854 .unwrap();
13855 let starts = outage_lines(&capture, OUTAGE_START);
13856 assert_eq!(starts.len(), 1, "{starts:?}");
13857 assert_eq!(starts[0].level, tracing::Level::WARN);
13858 }
13859
13860 #[tokio::test(flavor = "current_thread")]
13861 async fn route_open_unknown_module_escapes_target_module_id() {
13862 let handler = ControlHandler::new(Arc::new(Registry::default()));
13863 let capture = EventCapture::default();
13864 let _subscriber =
13865 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13866 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
13867 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13868 let response = handler
13869 .handle_control_frame(
13870 &ctx,
13871 route_open_frame(
13872 395,
13873 hostile_module_id,
13874 unique_project_root("hostile-target-module-id"),
13875 ),
13876 )
13877 .await
13878 .unwrap();
13879
13880 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13881 let event = capture
13882 .events()
13883 .into_iter()
13884 .find(|event| {
13885 event.target == "control"
13886 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13887 })
13888 .expect("route.open unknown-module refusal event");
13889 let logged = event.fields.get("module_id").expect("module_id field");
13890 assert!(!logged.bytes().any(|byte| byte < 0x20));
13891 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
13892 }
13893
13894 #[tokio::test(flavor = "current_thread")]
13897 async fn route_open_refusal_carries_daemon_origin() {
13898 let handler = ControlHandler::new(Arc::new(Registry::default()));
13899 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13900 let response = handler
13901 .handle_control_frame(
13902 &ctx,
13903 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
13904 )
13905 .await
13906 .unwrap();
13907
13908 assert_eq!(response.len(), 1);
13909 assert_eq!(response[0].header.ty, FrameType::Error);
13910 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13911 assert!(
13912 response[0].header.flags.is_daemon_origin(),
13913 "route.open refusal must carry daemon origin: {:?}",
13914 response[0].header.flags
13915 );
13916 }
13917
13918 #[tokio::test(flavor = "current_thread")]
13922 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
13923 let registry = Arc::new(Registry::default());
13924 let forwarding = Arc::new(ForwardingTable::default());
13925 let handler =
13926 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13927 let module_connection = ConnectionId::new(95);
13928 let (module_ctx, mut module_rx) = route_ctx(module_connection);
13929 hello_via_sink(
13930 &handler,
13931 &module_ctx,
13932 &mut module_rx,
13933 hello_frame("aft", PROTOCOL_VERSION, 395),
13934 )
13935 .await;
13936 forwarding
13937 .begin_module_drain("aft", RouteCloseReason::Restart)
13938 .unwrap()
13939 .expect("the module endpoint drains");
13940
13941 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
13942 let response = handler
13943 .handle_control_frame(
13944 &client_ctx,
13945 route_open_frame(396, "aft", unique_project_root("drain-origin")),
13946 )
13947 .await
13948 .unwrap();
13949
13950 assert_eq!(response.len(), 1);
13951 assert_eq!(response[0].header.ty, FrameType::Error);
13952 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
13953 assert!(
13954 response[0].header.flags.is_daemon_origin(),
13955 "drain module_reloading must carry daemon origin: {:?}",
13956 response[0].header.flags
13957 );
13958 }
13959
13960 #[tokio::test(flavor = "current_thread")]
13964 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
13965 let registry = Arc::new(Registry::default());
13966 let forwarding = Arc::new(ForwardingTable::default());
13967 let handler =
13968 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13969 let module_connection = ConnectionId::new(95);
13970 let (module_ctx, mut module_rx) = route_ctx(module_connection);
13971 hello_via_sink(
13972 &handler,
13973 &module_ctx,
13974 &mut module_rx,
13975 hello_frame("aft", PROTOCOL_VERSION, 395),
13976 )
13977 .await;
13978
13979 let client_connection = ConnectionId::new(96);
13980 let (client_ctx, _client_rx) = route_ctx(client_connection);
13981 let (route_task, bind) = relay_route_open(
13982 &handler,
13983 client_connection,
13984 &client_ctx.egress,
13985 &mut module_rx,
13986 396,
13987 "aft",
13988 "spoofed-reloading",
13989 )
13990 .await;
13991 let rejection = Frame::build(
13994 FrameType::Error,
13995 crate::router::daemon_error_flags(),
13996 0,
13997 0,
13998 bind.header.corr,
13999 serde_json::to_vec(&ErrorBody::new(
14000 "module_reloading",
14001 "module refused route.bind",
14002 ))
14003 .unwrap(),
14004 )
14005 .unwrap();
14006 handler
14007 .handle_control_frame(&module_ctx, rejection)
14008 .await
14009 .unwrap();
14010
14011 let response = route_task.await.unwrap();
14012 assert_eq!(response.len(), 1);
14013 assert_eq!(response[0].header.ty, FrameType::Error);
14014 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14015 assert!(
14016 !response[0].header.flags.is_daemon_origin(),
14017 "a module's own rejection must not carry daemon origin: {:?}",
14018 response[0].header.flags
14019 );
14020 }
14021
14022 #[tokio::test(flavor = "current_thread")]
14023 async fn route_open_module_rejection_uses_daemon_counter_key() {
14024 let registry = Arc::new(Registry::default());
14025 let forwarding = Arc::new(ForwardingTable::default());
14026 let handler =
14027 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14028 let module_connection = ConnectionId::new(95);
14029 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14030 hello_via_sink(
14031 &handler,
14032 &module_ctx,
14033 &mut module_rx,
14034 hello_frame("aft", PROTOCOL_VERSION, 395),
14035 )
14036 .await;
14037
14038 let client_connection = ConnectionId::new(96);
14039 let (client_ctx, _client_rx) = route_ctx(client_connection);
14040 let capture = EventCapture::default();
14041 let _subscriber =
14042 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14043 let (route_task, bind) = relay_route_open(
14044 &handler,
14045 client_connection,
14046 &client_ctx.egress,
14047 &mut module_rx,
14048 396,
14049 "aft",
14050 "hostile-module-code",
14051 )
14052 .await;
14053 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
14054 let rejection = Frame::build(
14055 FrameType::Error,
14056 control_flags(),
14057 0,
14058 0,
14059 bind.header.corr,
14060 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
14061 )
14062 .unwrap();
14063 handler
14064 .handle_control_frame(&module_ctx, rejection)
14065 .await
14066 .unwrap();
14067
14068 let response = route_task.await.unwrap();
14069 assert_eq!(parse_error(&response[0])["code"], hostile_code);
14070 let counters = handler.counters().snapshot();
14071 assert_eq!(
14072 counters["route_open_refused_by_code"],
14073 json!({ "module_rejected": 1 })
14074 );
14075 assert!(counters["route_open_refused_by_code"]
14076 .get(hostile_code)
14077 .is_none());
14078
14079 let event = capture
14080 .events()
14081 .into_iter()
14082 .find(|event| {
14083 event.target == "control"
14084 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
14085 })
14086 .expect("route.open module-rejection refusal event");
14087 let logged = event.fields.get("module_code").expect("module_code field");
14088 assert!(!logged.bytes().any(|byte| byte < 0x20));
14089 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14090 }
14091
14092 #[tokio::test]
14093 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
14094 let registry = Arc::new(Registry::default());
14095 let supervisor_handle = SupervisorHandle::new();
14096 let missing_program = std::env::temp_dir().join(format!(
14097 "subc-route-open-missing-program-{}",
14098 std::process::id()
14099 ));
14100 let supervisor =
14101 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
14102 .with_handle(supervisor_handle.clone());
14103 let module = supervisor
14104 .supervise_configured(
14105 ModuleSpec {
14106 module_id: "failed".to_string(),
14107 program: missing_program,
14108 args: Vec::new(),
14109 env: Vec::new(),
14110 reserved: false,
14111 reserved_prefixes: Vec::new(),
14112 protocol: ModuleProtocol::Subc,
14113 overlap: Default::default(),
14114 },
14115 true,
14116 )
14117 .unwrap();
14118 assert_eq!(module.state().unwrap(), ModuleState::Failed);
14119
14120 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14121 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
14122 let response = handler
14123 .handle_control_frame(
14124 &ctx,
14125 route_open_frame(305, "failed", unique_project_root("failed")),
14126 )
14127 .await
14128 .unwrap();
14129
14130 assert_eq!(response[0].header.ty, FrameType::Error);
14131 let error = parse_error(&response[0]);
14132 assert_eq!(error["code"], "target_unavailable");
14133 assert!(error["message"]
14134 .as_str()
14135 .unwrap()
14136 .contains("state=failed, enabled=true, live=false"));
14137 }
14138
14139 #[tokio::test]
14140 async fn route_open_role_mismatch_remains_target_unavailable() {
14141 let registry = Arc::new(Registry::default());
14142 let handler = ControlHandler::new(Arc::clone(®istry));
14143 handler
14144 .handle_control(
14145 ConnectionId::new(41),
14146 non_routable_hello_frame_with_control_ops("health-only", 306, None),
14147 )
14148 .unwrap();
14149
14150 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
14151 let response = handler
14152 .handle_control_frame(
14153 &ctx,
14154 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
14155 )
14156 .await
14157 .unwrap();
14158
14159 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14160 assert!(parse_error(&response[0])["message"]
14161 .as_str()
14162 .unwrap()
14163 .contains("does not provide the requested target"));
14164 }
14165
14166 #[tokio::test]
14167 async fn route_open_inactive_registration_remains_target_unavailable() {
14168 let registry = Arc::new(Registry::default());
14169 let handler = ControlHandler::new(Arc::clone(®istry));
14170 handler
14171 .handle_control(
14172 ConnectionId::new(43),
14173 hello_frame("inactive", PROTOCOL_VERSION, 308),
14174 )
14175 .unwrap();
14176 assert!(registry
14177 .set_module_state_for_test("inactive", ChannelState::Closed)
14178 .unwrap());
14179
14180 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
14181 let response = handler
14182 .handle_control_frame(
14183 &ctx,
14184 route_open_frame(309, "inactive", unique_project_root("inactive")),
14185 )
14186 .await
14187 .unwrap();
14188
14189 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14190 assert!(parse_error(&response[0])["message"]
14191 .as_str()
14192 .unwrap()
14193 .contains("is not active"));
14194 }
14195
14196 #[tokio::test]
14197 async fn late_health_reply_is_recorded_through_the_module_response_path() {
14198 let registry = Arc::new(Registry::default());
14199 let forwarding = Arc::new(ForwardingTable::default());
14200 let supervisor_handle = SupervisorHandle::new();
14201 let supervisor =
14202 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
14203 .with_forwarding(Arc::clone(&forwarding))
14204 .with_handle(supervisor_handle.clone());
14205 let module = supervisor
14206 .supervise_configured(
14207 crate::ModuleSpec {
14208 module_id: "late-health-response".to_string(),
14209 program: PathBuf::from("disabled-module"),
14210 args: Vec::new(),
14211 env: Vec::new(),
14212 reserved: false,
14213 reserved_prefixes: Vec::new(),
14214 protocol: ModuleProtocol::Subc,
14215 overlap: Default::default(),
14216 },
14217 false,
14218 )
14219 .unwrap();
14220 let handler =
14221 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14222 .with_supervisor(supervisor_handle);
14223 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
14224 handler
14225 .handle_control_frame(
14226 &module_ctx,
14227 hello_frame_with_control_ops(
14228 "late-health-response",
14229 PROTOCOL_VERSION,
14230 7,
14231 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14232 ),
14233 )
14234 .await
14235 .unwrap();
14236 let probe_started_at = Instant::now() - Duration::from_millis(80);
14237 let pending = forwarding
14238 .begin_health_probe_rpc_for(
14239 "late-health-response",
14240 MODULE_CONTROL_OP_HEALTH_CHECK,
14241 probe_started_at,
14242 Instant::now() - Duration::from_millis(1),
14243 )
14244 .unwrap();
14245 assert!(forwarding
14246 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
14247 .unwrap());
14248
14249 let responses = handler
14250 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
14251 .await
14252 .unwrap();
14253
14254 assert!(responses.is_empty());
14255 let health = module.status().unwrap().health;
14256 assert_eq!(health.late_answer_count, 1);
14257 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
14258 }
14259
14260 #[tokio::test]
14261 async fn health_probe_timeout_and_module_death_are_typed() {
14262 let registry = Arc::new(Registry::default());
14263 let forwarding = Arc::new(ForwardingTable::default());
14264 let handler =
14265 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14266 .with_health_probe_timeout(Duration::from_millis(50));
14267 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14268 hello_via_sink(
14269 &handler,
14270 &module_ctx,
14271 &mut module_rx,
14272 hello_frame_with_control_ops(
14273 "aft",
14274 PROTOCOL_VERSION,
14275 7,
14276 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14277 ),
14278 )
14279 .await;
14280
14281 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14282 let responses = handler
14283 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
14284 .await
14285 .unwrap();
14286 assert_eq!(responses[0].header.ty, FrameType::Error);
14287 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
14288 let _ = module_rx.try_recv();
14289
14290 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
14291 let health_handler = handler.clone();
14292 let death_task = tokio::spawn(async move {
14293 health_handler
14294 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
14295 .await
14296 .unwrap()
14297 });
14298 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
14299 .await
14300 .unwrap()
14301 .unwrap();
14302 handler
14303 .cleanup_connection(module_ctx.connection_id)
14304 .unwrap();
14305 let responses = death_task.await.unwrap();
14306 assert_eq!(responses[0].header.ty, FrameType::Error);
14307 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
14308 }
14309
14310 #[test]
14311 fn hello_requires_exact_protocol_version() {
14312 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
14313 let registry = Arc::new(Registry::default());
14314 let handler = ControlHandler::new(Arc::clone(®istry));
14315 let responses = handler
14316 .handle_control(
14317 ConnectionId::new(connection),
14318 hello_frame("aft", offered, 9),
14319 )
14320 .unwrap();
14321
14322 assert_eq!(responses.len(), 1);
14323 assert_eq!(responses[0].header.ty, FrameType::Error);
14324 let error = parse_error(&responses[0]);
14325 assert_eq!(error["code"], "version_unsupported");
14326 assert!(registry.get_module("aft").unwrap().is_none());
14327 assert_eq!(registry.active_registration_count().unwrap(), 0);
14328 }
14329 }
14330
14331 #[test]
14332 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
14333 let registry = Arc::new(Registry::default());
14334 let forwarding = Arc::new(ForwardingTable::default());
14335 let handler =
14336 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14337 let module_connection = ConnectionId::new(301);
14338 let registration = registry
14339 .register_with_control_ops(
14340 manifest("aft-push", PROTOCOL_VERSION),
14341 PROTOCOL_VERSION,
14342 module_connection,
14343 module_baseline_control_ops(),
14344 )
14345 .unwrap();
14346 let (module_tx, _module_rx) = mpsc::channel(8);
14347 let endpoint = forwarding
14348 .register_module_connection(
14349 module_connection,
14350 "aft-push".to_string(),
14351 PROTOCOL_VERSION,
14352 manifest_concurrency(®istration.manifest),
14353 FrameSink::new(module_tx),
14354 )
14355 .unwrap();
14356
14357 let unknown = Frame::build(
14359 FrameType::Push,
14360 control_flags(),
14361 0,
14362 0,
14363 5,
14364 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
14365 )
14366 .unwrap();
14367 let out = handler.handle_status_update(endpoint, unknown).unwrap();
14368 assert!(
14369 out.is_empty(),
14370 "unknown push op must be ignored, got {out:?}"
14371 );
14372
14373 let malformed = Frame::build(
14375 FrameType::Push,
14376 control_flags(),
14377 0,
14378 0,
14379 6,
14380 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
14381 )
14382 .unwrap();
14383 let out = handler.handle_status_update(endpoint, malformed).unwrap();
14384 assert_eq!(out.len(), 1);
14385 assert_eq!(out[0].header.ty, FrameType::Error);
14386 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
14387 }
14388
14389 #[test]
14390 fn hello_rejected_when_connection_already_owns_client_routes() {
14391 let registry = Arc::new(Registry::default());
14392 let forwarding = Arc::new(ForwardingTable::default());
14393 let handler =
14394 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14395 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
14397 let client_connection = ConnectionId::new(202);
14398
14399 let responses = handler
14402 .handle_control(
14403 client_connection,
14404 hello_frame("aft-second", PROTOCOL_VERSION, 9),
14405 )
14406 .unwrap();
14407 assert_eq!(responses[0].header.ty, FrameType::Error);
14408 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
14409 assert!(registry.get_module("aft-second").unwrap().is_none());
14410 }
14411
14412 #[tokio::test]
14413 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
14414 let registry = Arc::new(Registry::default());
14415 let forwarding = Arc::new(ForwardingTable::default());
14416 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
14417 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
14418 hello_via_sink(
14419 &handler,
14420 &module_ctx,
14421 &mut module_rx,
14422 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
14423 )
14424 .await;
14425 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
14426 let pending = forwarding
14427 .begin_route_bind_relay_for_test(
14428 client_ctx.connection_id,
14429 client_ctx.egress.clone(),
14430 2,
14431 "alpha",
14432 )
14433 .unwrap();
14434 forwarding
14435 .complete_pending_relay(
14436 module_ctx.connection_id,
14437 pending.corr,
14438 RouteBindRelayOutcome::Accepted,
14439 )
14440 .unwrap();
14441 client_rx.try_recv().unwrap();
14442 for module_id in ["beta", "alpha"] {
14443 let replies = handler
14444 .handle_control_frame(
14445 &module_ctx,
14446 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
14447 )
14448 .await
14449 .unwrap();
14450 assert_eq!(replies.len(), 1, "second HELLO must be refused");
14451 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
14452 }
14453 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
14454 assert!(registry.get_module("beta").unwrap().is_none());
14455 assert!(matches!(
14456 forwarding
14457 .lookup_data_route(
14458 client_ctx.connection_id,
14459 pending.client_channel,
14460 pending.client_epoch,
14461 )
14462 .unwrap(),
14463 DataRoute::Client(DataRouteState::Bound(_))
14464 ));
14465 assert!(handler
14466 .hello_launch_nonces
14467 .lock()
14468 .unwrap()
14469 .presented(module_ctx.connection_id, Some("alpha-nonce")));
14470 assert!(module_rx.try_recv().is_err());
14471 }
14472
14473 #[test]
14474 fn reserved_module_hello_requires_matching_launch_nonce() {
14475 let registry = Arc::new(Registry::default());
14476 let supervisor = SupervisorHandle::new();
14477 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
14480 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14481
14482 let no_nonce = handler
14484 .handle_control(
14485 ConnectionId::new(1),
14486 hello_frame("vault", PROTOCOL_VERSION, 1),
14487 )
14488 .unwrap();
14489 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
14490 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
14491 assert!(registry.get_module("vault").unwrap().is_none());
14492
14493 let wrong = handler
14495 .handle_control(
14496 ConnectionId::new(2),
14497 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
14498 )
14499 .unwrap();
14500 assert_eq!(wrong[0].header.ty, FrameType::Error);
14501 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
14502 assert!(registry.get_module("vault").unwrap().is_none());
14503
14504 let ok = handler
14506 .handle_control(
14507 ConnectionId::new(3),
14508 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
14509 )
14510 .unwrap();
14511 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
14512 assert!(registry.get_module("vault").unwrap().is_some());
14513 }
14514
14515 #[test]
14516 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
14517 let registry = Arc::new(Registry::default());
14518 let supervisor = SupervisorHandle::new();
14519 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14520 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14521 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14522
14523 let squat = handler
14524 .handle_control(
14525 ConnectionId::new(1),
14526 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
14527 )
14528 .unwrap();
14529 assert_eq!(squat[0].header.ty, FrameType::Error);
14530 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
14531 assert!(parse_error(&squat[0])["message"]
14532 .as_str()
14533 .unwrap()
14534 .contains("fed:"));
14535
14536 let accepted_peer = handler
14537 .handle_control(
14538 ConnectionId::new(2),
14539 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
14540 )
14541 .unwrap();
14542 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
14543
14544 let accepted_short = handler
14545 .handle_control(
14546 ConnectionId::new(3),
14547 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
14548 )
14549 .unwrap();
14550 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
14551
14552 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
14553 let response = handler
14554 .handle_control(
14555 ConnectionId::new(conn),
14556 hello_frame(module_id, PROTOCOL_VERSION, conn),
14557 )
14558 .unwrap();
14559 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
14560 }
14561 }
14562
14563 #[test]
14564 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
14565 let registry = Arc::new(Registry::default());
14566 let supervisor = SupervisorHandle::new();
14567 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14568 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14569 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
14570 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14571
14572 let owner_nonce = handler
14573 .handle_control(
14574 ConnectionId::new(1),
14575 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
14576 )
14577 .unwrap();
14578 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
14579 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
14580 assert!(registry.get_module("fed:special").unwrap().is_none());
14581
14582 let exact_nonce = handler
14583 .handle_control(
14584 ConnectionId::new(2),
14585 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
14586 )
14587 .unwrap();
14588 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14589 assert!(registry.get_module("fed:special").unwrap().is_some());
14590 }
14591
14592 #[test]
14593 fn non_reserved_module_ignores_launch_nonce() {
14594 let registry = Arc::new(Registry::default());
14595 let handler = ControlHandler::new(Arc::clone(®istry));
14598 let no_nonce = handler
14599 .handle_control(
14600 ConnectionId::new(1),
14601 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14602 )
14603 .unwrap();
14604 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14605 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14606
14607 let echoed_nonce = handler
14608 .handle_control(
14609 ConnectionId::new(2),
14610 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14611 )
14612 .unwrap();
14613 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14614 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14615 }
14616
14617 #[test]
14618 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14619 let handler = ControlHandler::default();
14620 let conn = ConnectionId::new(1);
14621 let malformed = Frame::build(
14622 FrameType::Hello,
14623 control_flags(),
14624 0,
14625 0,
14626 3,
14627 b"{not json".to_vec(),
14628 )
14629 .unwrap();
14630
14631 let error = handler.handle_control(conn, malformed).unwrap();
14632 assert_eq!(error[0].header.ty, FrameType::Error);
14633 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
14634
14635 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14636 let pong = handler.handle_control(conn, ping).unwrap();
14637 assert_eq!(pong[0].header.ty, FrameType::Pong);
14638 assert_eq!(pong[0].header.corr, 4);
14639 }
14640
14641 #[test]
14642 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
14643 let registry = Arc::new(Registry::default());
14644 let handler = ControlHandler::new(Arc::clone(®istry));
14645
14646 handler
14647 .handle_control(
14648 ConnectionId::new(1),
14649 hello_frame("aft", PROTOCOL_VERSION, 1),
14650 )
14651 .unwrap();
14652 let duplicate = handler
14653 .handle_control(
14654 ConnectionId::new(2),
14655 hello_frame("aft", PROTOCOL_VERSION, 2),
14656 )
14657 .unwrap();
14658
14659 assert_eq!(duplicate[0].header.ty, FrameType::Error);
14660 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14661 let registration = registry.get_module("aft").unwrap().unwrap();
14662 assert_eq!(registration.connection_id, ConnectionId::new(1));
14663 }
14664
14665 #[test]
14666 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
14667 let registry = Arc::new(Registry::default());
14668 let forwarding = Arc::new(ForwardingTable::default());
14669 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
14670 let handler =
14671 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14672 .with_process_liveness(process_liveness);
14673 let (ctx, route_channel, route_epoch) =
14674 bind_liveness_route(®istry, &forwarding, "aft-dead");
14675 let responses = handler
14676 .handle_route_poll(
14677 &ctx,
14678 route_poll_frame(41, PollKind::Liveness, route_channel),
14679 route_channel,
14680 route_epoch,
14681 PollKind::Liveness,
14682 )
14683 .unwrap();
14684
14685 assert_eq!(responses.len(), 1);
14686 assert_eq!(responses[0].header.ty, FrameType::Response);
14687 assert_route_poll_liveness(&responses[0], false);
14688 }
14689
14690 #[test]
14691 fn liveness_poll_without_process_source_uses_bound_route() {
14692 let registry = Arc::new(Registry::default());
14693 let forwarding = Arc::new(ForwardingTable::default());
14694 let handler =
14695 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14696 let (ctx, route_channel, route_epoch) =
14697 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
14698 let responses = handler
14699 .handle_route_poll(
14700 &ctx,
14701 route_poll_frame(42, PollKind::Liveness, route_channel),
14702 route_channel,
14703 route_epoch,
14704 PollKind::Liveness,
14705 )
14706 .unwrap();
14707
14708 assert_route_poll_liveness(&responses[0], true);
14709 }
14710
14711 #[test]
14712 fn liveness_poll_untracked_process_source_uses_bound_route() {
14713 let registry = Arc::new(Registry::default());
14714 let forwarding = Arc::new(ForwardingTable::default());
14715 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
14716 let handler =
14717 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14718 .with_process_liveness(process_liveness);
14719 let (ctx, route_channel, route_epoch) =
14720 bind_liveness_route(®istry, &forwarding, "aft-untracked");
14721 let responses = handler
14722 .handle_route_poll(
14723 &ctx,
14724 route_poll_frame(43, PollKind::Liveness, route_channel),
14725 route_channel,
14726 route_epoch,
14727 PollKind::Liveness,
14728 )
14729 .unwrap();
14730
14731 assert_route_poll_liveness(&responses[0], true);
14732 }
14733
14734 #[tokio::test]
14735 async fn unknown_op_returns_unknown_control_op() {
14736 let handler = ControlHandler::default();
14737 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
14738 let request = Frame::build(
14739 FrameType::Request,
14740 control_flags(),
14741 0,
14742 0,
14743 55,
14744 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
14745 )
14746 .unwrap();
14747
14748 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14749
14750 assert_eq!(response.len(), 1);
14751 assert_eq!(response[0].header.ty, FrameType::Error);
14752 assert_eq!(response[0].header.corr, 55);
14753 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
14754 }
14755
14756 #[tokio::test]
14757 async fn supervisor_provenance_rejects_unknown_exact_module() {
14758 let handler = ControlHandler::default();
14759 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
14760 let request = Frame::build(
14761 FrameType::Request,
14762 control_flags(),
14763 0,
14764 0,
14765 57,
14766 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
14767 )
14768 .unwrap();
14769
14770 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14771
14772 assert_eq!(response.len(), 1);
14773 assert_eq!(response[0].header.ty, FrameType::Error);
14774 assert_eq!(response[0].header.corr, 57);
14775 let error = parse_error(&response[0]);
14776 assert_eq!(error["code"], "unknown_module");
14777 assert_eq!(error["message"], "module_id 'missing' is not supervised");
14778 }
14779
14780 #[test]
14781 fn provenance_probe_override_keeps_handler_tests_deterministic() {
14782 let expected = subc_control::RunningImageAgreement::Unavailable {
14783 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14784 };
14785 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
14786 assert_eq!(handler.provenance_probe_override, Some(expected));
14787 }
14788
14789 #[test]
14790 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
14791 let verdict = reload_verdict(
14792 std::path::Path::new("/bin/new"),
14793 Some(std::path::Path::new("/bin/old")),
14794 subc_control::RunningImageAgreement::Unavailable {
14795 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14796 },
14797 );
14798 assert!(matches!(
14799 verdict.path,
14800 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
14801 if configured == std::path::Path::new("/bin/new")
14802 && spawned_from == std::path::Path::new("/bin/old")
14803 ));
14804 }
14805
14806 #[test]
14807 fn launch_spec_without_running_digest_is_unavailable_not_match() {
14808 use subc_control::{
14809 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
14810 };
14811 let configured = crate::supervise::ModuleSpec {
14812 module_id: "legacy".into(),
14813 program: "/module".into(),
14814 args: Vec::new(),
14815 env: Vec::new(),
14816 reserved: false,
14817 reserved_prefixes: Vec::new(),
14818 protocol: subc_control::ModuleProtocol::Subc,
14819 overlap: Default::default(),
14820 };
14821 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
14822 assert_eq!(
14823 launch_spec_agreement(&configured, false, Some(&digest)),
14824 Agreement::Unavailable {
14825 reason: Reason::NotRunning
14826 }
14827 );
14828 assert_eq!(
14829 launch_spec_agreement(&configured, true, None),
14830 Agreement::Unavailable {
14831 reason: Reason::SpawnedSpecUnavailable
14832 }
14833 );
14834 assert_eq!(
14835 launch_spec_agreement(&configured, true, Some(&digest)),
14836 Agreement::Match
14837 );
14838 }
14839
14840 #[cfg(windows)]
14841 #[test]
14842 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
14843 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
14844 let configured = dir.join("module.exe");
14845 std::fs::write(&configured, b"executable path fixture").unwrap();
14846 let spawned = std::fs::canonicalize(&configured).unwrap();
14847 assert_ne!(
14848 configured, spawned,
14849 "fixture must exercise different path spellings"
14850 );
14851 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
14852 let lower_case = configured.to_string_lossy().to_lowercase();
14853 for spelling in [
14854 &configured,
14855 Path::new(&forward_slashes),
14856 Path::new(&lower_case),
14857 ] {
14858 let verdict = reload_verdict(
14859 spelling,
14860 Some(&spawned),
14861 subc_control::RunningImageAgreement::Unavailable {
14862 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14863 },
14864 );
14865 assert_eq!(
14866 verdict.path,
14867 ReloadPathAgreement::Match,
14868 "configured {spelling:?}, spawned {spawned:?}"
14869 );
14870 }
14871 }
14872
14873 #[cfg(windows)]
14874 #[test]
14875 fn windows_reload_path_keeps_different_existing_program_pending() {
14876 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
14877 let old = dir.join("old.exe");
14878 let new = dir.join("new.exe");
14879 std::fs::write(&old, b"same executable content").unwrap();
14880 std::fs::copy(&old, &new).unwrap();
14881 let spawned = std::fs::canonicalize(&old).unwrap();
14882 let image = subc_control::RunningImageAgreement::Unavailable {
14883 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14884 };
14885 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
14886 assert!(
14887 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
14888 if configured == new && spawned_from == spawned)
14889 );
14890 let missing = dir.join("missing.exe");
14891 assert!(matches!(
14892 reload_verdict(&missing, Some(&spawned), image).path,
14893 ReloadPathAgreement::Mismatch { .. }
14894 ));
14895 }
14896
14897 #[test]
14898 fn reload_verdict_detects_replaced_image_at_same_path() {
14899 let image = subc_control::RunningImageAgreement::Mismatch {
14900 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
14901 digest: "old".into(),
14902 },
14903 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
14904 digest: "new".into(),
14905 },
14906 };
14907 let verdict = reload_verdict(
14908 std::path::Path::new("/bin/same"),
14909 Some(std::path::Path::new("/bin/same")),
14910 image.clone(),
14911 );
14912 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14913 assert_eq!(verdict.image, image);
14914 }
14915
14916 #[test]
14917 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
14918 let image = subc_control::RunningImageAgreement::Unavailable {
14919 reason: subc_control::RunningImageUnavailableReason::NotRunning,
14920 };
14921 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
14922 assert_eq!(
14923 verdict.path,
14924 subc_control::ReloadPathAgreement::Unavailable {
14925 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
14926 }
14927 );
14928 assert_eq!(verdict.image, image);
14929
14930 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
14931 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
14932 };
14933 let verdict = reload_verdict(
14934 std::path::Path::new("/bin/same"),
14935 Some(std::path::Path::new("/bin/same")),
14936 unconfirmed.clone(),
14937 );
14938 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14939 assert_eq!(verdict.image, unconfirmed);
14940 }
14941
14942 #[test]
14943 fn reload_verdict_preserves_each_image_unavailability_reason() {
14944 use subc_control::RunningImageUnavailableReason as Reason;
14945
14946 for reason in [
14947 Reason::NotRunning,
14948 Reason::UnsupportedPlatform,
14949 Reason::RunningExecutableUnreadable,
14950 Reason::SpawnedPathUnreadable,
14951 Reason::HashFailed,
14952 Reason::ProcessIdentityUnconfirmed,
14953 Reason::Unknown("future_probe_reason".to_string()),
14954 ] {
14955 let image = subc_control::RunningImageAgreement::Unavailable {
14956 reason: reason.clone(),
14957 };
14958 let verdict = reload_verdict(
14959 std::path::Path::new("/bin/same"),
14960 Some(std::path::Path::new("/bin/same")),
14961 image.clone(),
14962 );
14963 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
14964 assert_eq!(verdict.image, image, "{reason:?}");
14965 }
14966 }
14967
14968 #[tokio::test]
14969 async fn malformed_control_bodies_return_invalid_control_body() {
14970 let handler = ControlHandler::default();
14971 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
14972
14973 for (corr, body) in [
14974 (56, br#"{"route_channel":1}"#.as_slice()),
14975 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
14976 (
14977 58,
14978 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
14979 ),
14980 ] {
14981 let request = Frame::build(
14982 FrameType::Request,
14983 control_flags(),
14984 0,
14985 0,
14986 corr,
14987 body.to_vec(),
14988 )
14989 .unwrap();
14990 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14991
14992 assert_eq!(response.len(), 1);
14993 assert_eq!(response[0].header.ty, FrameType::Error);
14994 assert_eq!(response[0].header.corr, corr);
14995 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
14996 }
14997 }
14998
14999 #[tokio::test]
15000 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
15001 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15002 let registry = Arc::new(Registry::default());
15003 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
15004 let router = Router::with_control_handler(Arc::clone(&control));
15005 let connection = router.begin_connection();
15006 let (ctx, mut rx) = route_ctx(connection.id());
15007
15008 router
15009 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
15010 .await
15011 .unwrap();
15012 let response = rx.recv().await.unwrap();
15013 let ack = parse_ack(&response);
15014 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15015 let channel = 1;
15016
15017 let goodbye =
15018 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
15019 router.route_for_connection(&ctx, goodbye).await.unwrap();
15020 assert!(rx.try_recv().is_err());
15021 assert!(registry.get_module("aft").unwrap().is_none());
15022
15023 router
15024 .route_for_connection(&ctx, channel_request(channel, 13))
15025 .await
15026 .unwrap();
15027 let error_frame = rx.recv().await.unwrap();
15028 assert_eq!(error_frame.header.ty, FrameType::Error);
15029 assert_eq!(error_frame.header.channel, channel);
15030 let captured = crate::router::test_log::captured_logs(&logs);
15031 assert_eq!(
15032 captured
15033 .lines()
15034 .filter(|line| {
15035 line.contains("module_id=aft")
15036 && line.contains("reason=explicit_goodbye")
15037 && line.contains("module registration ended")
15038 })
15039 .count(),
15040 1,
15041 "unexpected GOODBYE registry log: {captured}"
15042 );
15043 }
15044
15045 #[tokio::test]
15046 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
15047 let registry = Arc::new(Registry::default());
15048 let handler = ControlHandler::new(registry).with_capability_config(
15049 [("prov".to_string(), true), ("cons".to_string(), true)],
15050 BTreeMap::new(),
15051 );
15052 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
15053 register_capability_manifest(
15054 &handler,
15055 &provider_ctx,
15056 &mut provider_rx,
15057 capability_manifest("prov", &["thing/v1"], &[]),
15058 1,
15059 )
15060 .await;
15061 let mut consumer = capability_manifest("cons", &[], &[]);
15062 consumer.capabilities.as_mut().unwrap().requires.push(
15063 subc_protocol::manifest::CapabilityRequirement {
15064 capability: "thing/v1".to_string(),
15065 need: subc_protocol::manifest::CapabilityNeed::Required,
15066 },
15067 );
15068 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
15069 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
15070 assert_eq!(
15071 handler.capability_evaluator.verdict("cons", "thing/v1"),
15072 Some(CapabilityVerdict::Provided)
15073 );
15074 let (mut client_rx, _) = open_route_for_capability_test(
15075 &handler,
15076 &provider_ctx,
15077 &mut provider_rx,
15078 703,
15079 3,
15080 "prov",
15081 None,
15082 )
15083 .await;
15084 let goodbye =
15085 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15086 handler
15087 .handle_control_frame(&provider_ctx, goodbye)
15088 .await
15089 .unwrap();
15090 assert_eq!(
15091 handler.capability_evaluator.verdict("cons", "thing/v1"),
15092 Some(CapabilityVerdict::NeverProvided)
15093 );
15094 let closed = client_rx
15095 .try_recv()
15096 .expect("GOODBYE pushes route.closed before route GOODBYE");
15097 assert!(
15098 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
15099 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
15100 );
15101 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
15102 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
15103 }
15104
15105 struct RouteClosedFixture {
15106 handler: ControlHandler,
15107 #[cfg(unix)]
15108 supervisor: Supervisor,
15109 module_ctx: RouteCtx,
15110 #[cfg(unix)]
15111 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15112 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15113 route: Arc<crate::forwarding::RouteBinding>,
15114 }
15115
15116 async fn route_closed_fixture() -> RouteClosedFixture {
15117 let registry = Arc::new(Registry::default());
15118 let handler = ControlHandler::new(Arc::clone(®istry));
15119 #[cfg(unix)]
15120 let supervisor = Supervisor::new(registry, RestartPolicy::default())
15121 .with_forwarding(Arc::clone(&handler.forwarding));
15122 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
15123 register_capability_manifest(
15124 &handler,
15125 &module_ctx,
15126 &mut module_rx,
15127 capability_manifest("prov", &[], &[]),
15128 1,
15129 )
15130 .await;
15131 let (client_rx, bind) = open_route_for_capability_test(
15132 &handler,
15133 &module_ctx,
15134 &mut module_rx,
15135 702,
15136 2,
15137 "prov",
15138 None,
15139 )
15140 .await;
15141 let ModuleControlRequest::RouteBind {
15142 route_channel: channel,
15143 epoch: route_epoch,
15144 ..
15145 } = bind
15146 else {
15147 panic!("expected route.bind");
15148 };
15149 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
15150 handler
15151 .forwarding
15152 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
15153 .unwrap()
15154 else {
15155 panic!("the acknowledged route must be bound");
15156 };
15157 RouteClosedFixture {
15158 handler,
15159 #[cfg(unix)]
15160 supervisor,
15161 module_ctx,
15162 #[cfg(unix)]
15163 module_rx,
15164 client_rx,
15165 route,
15166 }
15167 }
15168
15169 impl RouteClosedFixture {
15170 #[cfg(unix)]
15171 async fn cut(&mut self) {
15172 let notices = async {
15173 let mut draining = self.module_rx.recv().await.unwrap();
15174 assert!(matches!(
15175 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
15176 &draining.body
15177 )
15178 .unwrap(),
15179 subc_protocol::session::ModuleControlCommand::Draining {
15180 reason: RouteCloseReason::Restart,
15181 ..
15182 }
15183 ));
15184 draining.flushed.take().unwrap().send(()).unwrap();
15185 let mut closing = self.client_rx.recv().await.unwrap();
15186 assert!(matches!(
15187 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
15188 ClientControlPush::RouteClosing {
15189 reason: RouteCloseReason::Restart,
15190 ..
15191 }
15192 ));
15193 closing.flushed.take().unwrap().send(()).unwrap();
15194 };
15195 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
15196 result.unwrap();
15197 }
15198
15199 fn disconnect(&mut self) -> ClientControlPush {
15200 self.handler
15201 .cleanup_connection(self.module_ctx.connection_id)
15202 .unwrap();
15203 let closed = self.client_rx.try_recv().unwrap();
15204 assert_eq!(closed.header.ty, FrameType::Push);
15205 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
15206 assert!(matches!(
15207 &push,
15208 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
15209 if channels == &[self.route.client_channel]
15210 ));
15211 assert_eq!(
15212 self.client_rx.try_recv().unwrap().header.ty,
15213 FrameType::Goodbye
15214 );
15215 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
15216 push
15217 }
15218 }
15219
15220 #[cfg(unix)]
15221 #[tokio::test(start_paused = true)]
15222 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
15223 let mut fixture = route_closed_fixture().await;
15224 fixture.cut().await;
15225 assert!(matches!(
15226 fixture.disconnect(),
15227 ClientControlPush::RouteClosed {
15228 reason: RouteCloseReason::Restart,
15229 drained: true,
15230 ..
15231 }
15232 ));
15233 }
15234
15235 #[cfg(unix)]
15236 #[tokio::test(start_paused = true)]
15237 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
15238 let mut fixture = route_closed_fixture().await;
15239 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
15240 let started = tokio::time::Instant::now();
15241 fixture.cut().await;
15242 assert!(started.elapsed() >= Duration::from_secs(2));
15243 assert_eq!(fixture.route.flow.in_flight(), 1);
15244 assert!(matches!(
15245 fixture.disconnect(),
15246 ClientControlPush::RouteClosed {
15247 reason: RouteCloseReason::Restart,
15248 drained: false,
15249 ..
15250 }
15251 ));
15252 }
15253
15254 #[tokio::test]
15255 async fn module_connection_crash_reports_crash_and_not_drained() {
15256 let mut fixture = route_closed_fixture().await;
15257 assert!(matches!(
15261 fixture.disconnect(),
15262 ClientControlPush::RouteClosed {
15263 reason: RouteCloseReason::Crash,
15264 drained: false,
15265 ..
15266 }
15267 ));
15268 }
15269
15270 #[tokio::test]
15271 async fn dropping_router_connection_releases_registration() {
15272 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15273 let registry = Arc::new(Registry::default());
15274 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
15275 let router = Router::with_control_handler(Arc::clone(&control));
15276 let connection = router.begin_connection();
15277 let connection_id = connection.id();
15278 let (ctx, mut rx) = route_ctx(connection_id);
15279
15280 router
15281 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
15282 .await
15283 .unwrap();
15284 let response = rx.recv().await.unwrap();
15285 let ack = parse_ack(&response);
15286 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15287 assert!(registry.get_module("aft").unwrap().is_some());
15288
15289 drop(connection);
15290
15291 assert!(registry.get_module("aft").unwrap().is_none());
15292 assert_eq!(registry.active_registration_count().unwrap(), 0);
15293
15294 control
15295 .cleanup_connection(ConnectionId::new(u64::MAX))
15296 .unwrap();
15297 let captured = crate::router::test_log::captured_logs(&logs);
15298 println!("captured registry lifecycle logs:\n{captured}");
15299 let events: Vec<_> = captured
15300 .lines()
15301 .filter(|line| {
15302 line.contains("module registered module_id=aft ")
15303 || line.contains("module registration ended")
15304 })
15305 .collect();
15306 assert_eq!(
15307 events.len(),
15308 2,
15309 "unexpected registry lifecycle logs: {captured}"
15310 );
15311 assert!(events[0].contains("module registered module_id=aft "));
15312 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
15313 assert!(events[1].contains(&format!(
15314 "module_id=aft connection_id={}",
15315 connection_id.get()
15316 )));
15317 assert!(events[1].contains("reason=connection_closed"));
15318 assert!(events[1].contains("module registration ended"));
15319 assert_eq!(
15320 captured
15321 .lines()
15322 .filter(|line| line.contains("module registered module_id=aft "))
15323 .count(),
15324 1,
15325 "legacy registration admission line must appear once: {captured}"
15326 );
15327 }
15328
15329 #[tokio::test]
15330 async fn hello_registration_keeps_legacy_module_registered_line_once() {
15331 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15332 let registry = Arc::new(Registry::default());
15333 let control = ControlHandler::new(Arc::clone(®istry));
15334 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
15335 hello_via_sink(
15336 &control,
15337 &ctx,
15338 &mut rx,
15339 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
15340 )
15341 .await;
15342
15343 let captured = crate::router::test_log::captured_logs(&logs);
15344 let admissions: Vec<_> = captured
15345 .lines()
15346 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
15347 .collect();
15348 assert_eq!(
15349 admissions.len(),
15350 1,
15351 "legacy admission line must remain exactly once: {captured}"
15352 );
15353 assert!(admissions[0].contains("routable_provider=true"));
15354 assert!(admissions[0].contains("connection_id=777"));
15355 }
15356
15357 fn capability_manifest(
15358 module_id: &str,
15359 provides: &[&str],
15360 must_never_reach: &[&str],
15361 ) -> ModuleManifest {
15362 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
15363 manifest.capabilities = Some(CapabilityDeclarations {
15364 provides: provides
15365 .iter()
15366 .map(|capability| (*capability).to_string())
15367 .collect(),
15368 requires: Vec::new(),
15369 must_never_reach: must_never_reach
15370 .iter()
15371 .map(|capability| (*capability).to_string())
15372 .collect(),
15373 });
15374 manifest
15375 }
15376
15377 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
15378 Frame::build(
15379 FrameType::Hello,
15380 control_flags(),
15381 0,
15382 0,
15383 corr,
15384 serde_json::to_vec(&ModuleHelloBody {
15385 protocol_ver: manifest.protocol_ver,
15386 manifest,
15387 control_ops: None,
15388 launch_nonce: None,
15389 })
15390 .expect("capability test HELLO serializes"),
15391 )
15392 .expect("capability test HELLO frame builds")
15393 }
15394
15395 fn catalog_update_with_capabilities_frame(
15396 corr: u64,
15397 capabilities: CapabilityDeclarations,
15398 ) -> Frame {
15399 Frame::build(
15400 FrameType::Request,
15401 control_flags(),
15402 0,
15403 0,
15404 corr,
15405 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15406 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
15407 capabilities: Some(capabilities),
15408 ready: None,
15409 })
15410 .expect("capability catalog.update serializes"),
15411 )
15412 .expect("capability catalog.update frame builds")
15413 }
15414
15415 async fn register_capability_manifest(
15416 handler: &ControlHandler,
15417 ctx: &RouteCtx,
15418 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15419 manifest: ModuleManifest,
15420 corr: u64,
15421 ) {
15422 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
15423 }
15424
15425 async fn open_route_for_capability_test(
15426 handler: &ControlHandler,
15427 target_ctx: &RouteCtx,
15428 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15429 client_connection_id: u64,
15430 corr: u64,
15431 target_module_id: &str,
15432 consumer_identity: Option<ConsumerIdentity>,
15433 ) -> (
15434 mpsc::Receiver<crate::router::OutboundFrame>,
15435 ModuleControlRequest,
15436 ) {
15437 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
15438 let route_handler = handler.clone();
15439 let target_module_id = target_module_id.to_string();
15440 let route_task = tokio::spawn(async move {
15441 route_handler
15442 .handle_control_frame(
15443 &client_ctx,
15444 route_open_frame_with_admission_facts(
15445 corr,
15446 &target_module_id,
15447 unique_project_root("admission-facts"),
15448 consumer_identity,
15449 None,
15450 ),
15451 )
15452 .await
15453 .expect("capability test route.open succeeds")
15454 });
15455 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
15456 .await
15457 .expect("capability test route.open must reach route.bind")
15458 .expect("target control receiver stays open");
15459 let bind_request: ModuleControlRequest =
15460 serde_json::from_slice(&bind.body).expect("route.bind decodes");
15461 handler
15462 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
15463 .await
15464 .expect("capability test route.bind ACK succeeds");
15465 assert!(route_task.await.expect("route.open task joins").is_empty());
15466 let opened = client_rx
15467 .recv()
15468 .await
15469 .expect("successful route.open publishes a response");
15470 assert!(matches!(
15471 serde_json::from_slice::<ClientControlResponse>(&opened.body),
15472 Ok(ClientControlResponse::RouteOpen { .. })
15473 ));
15474 (client_rx, bind_request)
15475 }
15476
15477 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
15478 assert_eq!(frame.header.ty, FrameType::Push);
15479 assert_eq!(frame.header.channel, 0);
15480 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
15481 .expect("route.closed control push decodes");
15482 let ClientControlPush::RouteClosed { channels, .. } = &push else {
15483 panic!("expected route.closed");
15484 };
15485 assert_eq!(channels.len(), 1, "exactly one violating route closed");
15486 let closed_channel = channels[0];
15487 let channels = channels.clone();
15488 assert_eq!(
15489 push,
15490 ClientControlPush::RouteClosed {
15491 module_id: target_module_id.to_string(),
15492 channels,
15493 reason: RouteCloseReason::CapabilityDenied,
15494 drained: false,
15495 abandoned: 0,
15496 excluded_subscriptions: 0,
15497 terminal: Some(false),
15498 }
15499 );
15500 closed_channel
15501 }
15502
15503 #[tokio::test]
15504 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
15505 let registry = Arc::new(Registry::default());
15506 let forwarding = Arc::new(ForwardingTable::default());
15507 let supervisor = SupervisorHandle::new();
15508 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15509 let handler =
15510 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15511 .with_supervisor(supervisor);
15512 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
15513 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
15514 register_capability_manifest(
15515 &handler,
15516 &target_ctx,
15517 &mut target_rx,
15518 capability_manifest("target", &["credentials-provider/v1"], &[]),
15519 1,
15520 )
15521 .await;
15522 register_capability_manifest(
15523 &handler,
15524 &opener_ctx,
15525 &mut opener_rx,
15526 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15527 2,
15528 )
15529 .await;
15530
15531 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
15532 let replies = handler
15533 .handle_control_frame(
15534 &client_ctx,
15535 route_open_frame_with_admission_facts(
15536 3,
15537 "target",
15538 unique_project_root("admission-facts"),
15539 Some(ConsumerIdentity {
15540 module_id: "opener".to_string(),
15541 launch_nonce: "opener-nonce".to_string(),
15542 }),
15543 None,
15544 ),
15545 )
15546 .await
15547 .expect("denied route.open returns a typed frame");
15548 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15549 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15550 assert!(
15551 target_rx.try_recv().is_err(),
15552 "forbidden route.open must not relay route.bind"
15553 );
15554 }
15555
15556 #[tokio::test]
15557 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
15558 let registry = Arc::new(Registry::default());
15559 let forwarding = Arc::new(ForwardingTable::default());
15560 let supervisor = SupervisorHandle::new();
15561 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15562 let handler =
15563 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15564 .with_supervisor(supervisor);
15565 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
15566 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
15567 register_capability_manifest(
15568 &handler,
15569 &target_ctx,
15570 &mut target_rx,
15571 capability_manifest("target", &["credentials-provider/v1"], &[]),
15572 1,
15573 )
15574 .await;
15575 register_capability_manifest(
15576 &handler,
15577 &old_opener_ctx,
15578 &mut old_opener_rx,
15579 capability_manifest("opener", &[], &[]),
15580 2,
15581 )
15582 .await;
15583 let (mut client_rx, _) = open_route_for_capability_test(
15584 &handler,
15585 &target_ctx,
15586 &mut target_rx,
15587 712,
15588 3,
15589 "target",
15590 Some(ConsumerIdentity {
15591 module_id: "opener".to_string(),
15592 launch_nonce: "opener-nonce".to_string(),
15593 }),
15594 )
15595 .await;
15596 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15597
15598 handler
15599 .cleanup_connection(old_opener_ctx.connection_id)
15600 .expect("old opener registration cleans up");
15601 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15602 register_capability_manifest(
15603 &handler,
15604 &new_opener_ctx,
15605 &mut new_opener_rx,
15606 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15607 4,
15608 )
15609 .await;
15610
15611 let closed_channel = assert_capability_denied_push(
15612 client_rx
15613 .try_recv()
15614 .expect("HELLO deny addition must emit route.closed")
15615 .frame,
15616 "target",
15617 );
15618 let goodbye = client_rx
15619 .try_recv()
15620 .expect("HELLO deny addition must send the client a route GOODBYE")
15621 .frame;
15622 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15623 assert_eq!(goodbye.header.channel, closed_channel);
15624 assert_ne!(goodbye.header.channel, 0);
15625 assert_ne!(goodbye.header.epoch, 0);
15626 assert!(client_rx.try_recv().is_err());
15627 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15628 assert!(matches!(
15629 target_rx.try_recv(),
15630 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15631 ));
15632 }
15633
15634 #[tokio::test]
15635 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
15636 let registry = Arc::new(Registry::default());
15637 let forwarding = Arc::new(ForwardingTable::default());
15638 let supervisor = SupervisorHandle::new();
15639 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15640 let handler =
15641 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15642 .with_supervisor(supervisor);
15643 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
15644 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
15645 register_capability_manifest(
15646 &handler,
15647 &target_ctx,
15648 &mut target_rx,
15649 capability_manifest("target", &[], &[]),
15650 1,
15651 )
15652 .await;
15653 register_capability_manifest(
15654 &handler,
15655 &opener_ctx,
15656 &mut opener_rx,
15657 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15658 2,
15659 )
15660 .await;
15661 let (mut client_rx, _) = open_route_for_capability_test(
15662 &handler,
15663 &target_ctx,
15664 &mut target_rx,
15665 722,
15666 3,
15667 "target",
15668 Some(ConsumerIdentity {
15669 module_id: "opener".to_string(),
15670 launch_nonce: "opener-nonce".to_string(),
15671 }),
15672 )
15673 .await;
15674 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15675
15676 let replies = handler
15677 .handle_control_frame(
15678 &target_ctx,
15679 catalog_update_with_capabilities_frame(
15680 4,
15681 CapabilityDeclarations {
15682 provides: vec!["credentials-provider/v1".to_string()],
15683 requires: Vec::new(),
15684 must_never_reach: Vec::new(),
15685 },
15686 ),
15687 )
15688 .await
15689 .expect("claim catalog.update succeeds");
15690 assert!(matches!(
15691 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
15692 Ok(ModuleControlResponseToModule::CatalogUpdate {})
15693 ));
15694 let closed_channel = assert_capability_denied_push(
15695 client_rx
15696 .try_recv()
15697 .expect("claim addition must emit route.closed")
15698 .frame,
15699 "target",
15700 );
15701 let goodbye = client_rx
15702 .try_recv()
15703 .expect("claim addition must send the client a route GOODBYE")
15704 .frame;
15705 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15706 assert_eq!(goodbye.header.channel, closed_channel);
15707 assert_ne!(goodbye.header.channel, 0);
15708 assert_ne!(goodbye.header.epoch, 0);
15709 assert!(client_rx.try_recv().is_err());
15710 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15711 assert!(matches!(
15712 target_rx.try_recv(),
15713 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15714 ));
15715 }
15716
15717 #[tokio::test]
15718 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
15719 let registry = Arc::new(Registry::default());
15720 let forwarding = Arc::new(ForwardingTable::default());
15721 let supervisor = SupervisorHandle::new();
15722 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15723 let handler =
15724 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15725 .with_supervisor(supervisor);
15726 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
15727 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
15728 register_capability_manifest(
15729 &handler,
15730 &target_ctx,
15731 &mut target_rx,
15732 capability_manifest("target", &["credentials-provider/v1"], &[]),
15733 1,
15734 )
15735 .await;
15736 register_capability_manifest(
15737 &handler,
15738 &opener_ctx,
15739 &mut opener_rx,
15740 capability_manifest("opener", &[], &[]),
15741 2,
15742 )
15743 .await;
15744 let (mut client_rx, _) = open_route_for_capability_test(
15745 &handler,
15746 &target_ctx,
15747 &mut target_rx,
15748 732,
15749 3,
15750 "target",
15751 Some(ConsumerIdentity {
15752 module_id: "opener".to_string(),
15753 launch_nonce: "opener-nonce".to_string(),
15754 }),
15755 )
15756 .await;
15757 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15758
15759 handler
15760 .handle_control_frame(
15761 &target_ctx,
15762 catalog_update_with_capabilities_frame(
15763 4,
15764 CapabilityDeclarations {
15765 provides: Vec::new(),
15766 requires: Vec::new(),
15767 must_never_reach: Vec::new(),
15768 },
15769 ),
15770 )
15771 .await
15772 .expect("claim removal catalog.update succeeds");
15773 assert_eq!(
15774 forwarding.active_binding_count().unwrap(),
15775 1,
15776 "removing an attested target claim must leave the route census unchanged"
15777 );
15778 assert!(
15779 client_rx.try_recv().is_err(),
15780 "claim removal must not emit route.closed or a client route GOODBYE"
15781 );
15782 assert!(
15783 target_rx.try_recv().is_err(),
15784 "claim removal must not send the target a route GOODBYE"
15785 );
15786 }
15787
15788 #[tokio::test]
15791 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
15792 let registry = Arc::new(Registry::default());
15793 let forwarding = Arc::new(ForwardingTable::default());
15794 let supervisor = SupervisorHandle::new();
15795 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15796 let handler =
15797 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15798 .with_supervisor(supervisor);
15799 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
15800 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
15801 register_capability_manifest(
15802 &handler,
15803 &target_ctx,
15804 &mut target_rx,
15805 capability_manifest("target", &["credentials-provider/v1"], &[]),
15806 1,
15807 )
15808 .await;
15809 register_capability_manifest(
15810 &handler,
15811 &opener_ctx,
15812 &mut opener_rx,
15813 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15814 2,
15815 )
15816 .await;
15817
15818 let (_client_rx, bind) = open_route_for_capability_test(
15819 &handler,
15820 &target_ctx,
15821 &mut target_rx,
15822 742,
15823 3,
15824 "target",
15825 None,
15826 )
15827 .await;
15828 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
15829 panic!("direct scope-honesty route must bind");
15830 };
15831 assert_eq!(principal, Some(Principal::Direct));
15832 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15833 }
15834
15835 #[tokio::test]
15838 async fn must_never_reach_self_route_is_capability_forbidden() {
15839 let registry = Arc::new(Registry::default());
15840 let forwarding = Arc::new(ForwardingTable::default());
15841 let supervisor = SupervisorHandle::new();
15842 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
15843 let handler =
15844 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15845 .with_supervisor(supervisor);
15846 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
15847 register_capability_manifest(
15848 &handler,
15849 &self_ctx,
15850 &mut self_rx,
15851 capability_manifest(
15852 "self-provider",
15853 &["credentials-provider/v1"],
15854 &["credentials-provider/v1"],
15855 ),
15856 1,
15857 )
15858 .await;
15859
15860 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
15861 let replies = handler
15862 .handle_control_frame(
15863 &client_ctx,
15864 route_open_frame_with_admission_facts(
15865 2,
15866 "self-provider",
15867 unique_project_root("admission-facts"),
15868 Some(ConsumerIdentity {
15869 module_id: "self-provider".to_string(),
15870 launch_nonce: "self-nonce".to_string(),
15871 }),
15872 None,
15873 ),
15874 )
15875 .await
15876 .expect("self-route refusal returns a typed frame");
15877 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15878 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15879 assert!(
15880 self_rx.try_recv().is_err(),
15881 "self denial must not relay route.bind"
15882 );
15883 }
15884
15885 #[test]
15886 fn unsupported_channel_zero_frame_returns_error() {
15887 let handler = ControlHandler::default();
15888 let request = Frame::build(
15889 FrameType::Request,
15890 control_flags(),
15891 0,
15892 0,
15893 21,
15894 b"opaque".to_vec(),
15895 )
15896 .unwrap();
15897
15898 let response = handler
15899 .handle_control(ConnectionId::new(1), request)
15900 .unwrap();
15901
15902 assert_eq!(response[0].header.ty, FrameType::Error);
15903 assert_eq!(
15904 parse_error(&response[0])["code"],
15905 "unsupported_control_frame"
15906 );
15907 }
15908
15909 mod swap {
15914 use super::*;
15915
15916 const INCUMBENT: ConnectionId = ConnectionId::new(30);
15917 const CANDIDATE: ConnectionId = ConnectionId::new(40);
15918
15919 struct Swap {
15920 registry: Arc<Registry>,
15921 forwarding: Arc<ForwardingTable>,
15922 handler: ControlHandler,
15923 incumbent_ctx: RouteCtx,
15924 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15925 candidate_ctx: RouteCtx,
15926 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15927 }
15928
15929 async fn swap_with_incumbent() -> Swap {
15930 let registry = Arc::new(Registry::default());
15931 let forwarding = Arc::new(ForwardingTable::default());
15932 let handler =
15933 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15934 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
15935 hello_via_sink(
15936 &handler,
15937 &incumbent_ctx,
15938 &mut incumbent_rx,
15939 hello_frame("aft", PROTOCOL_VERSION, 7),
15940 )
15941 .await;
15942 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
15943 Swap {
15944 registry,
15945 forwarding,
15946 handler,
15947 incumbent_ctx,
15948 incumbent_rx,
15949 candidate_ctx,
15950 candidate_rx,
15951 }
15952 }
15953
15954 fn register_candidate(swap: &Swap, ready: Option<bool>) {
15955 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
15956 candidate_manifest.ready = ready;
15957 let registration = swap
15958 .registry
15959 .register_candidate_with_control_ops(
15960 candidate_manifest,
15961 PROTOCOL_VERSION,
15962 CANDIDATE,
15963 module_baseline_control_ops(),
15964 )
15965 .unwrap();
15966 swap.forwarding
15967 .register_candidate_module_connection(
15968 CANDIDATE,
15969 "aft".to_string(),
15970 PROTOCOL_VERSION,
15971 manifest_concurrency(®istration.manifest),
15972 swap.candidate_ctx.egress.clone(),
15973 )
15974 .unwrap();
15975 }
15976
15977 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
15978 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
15979 swap.registry.promote_candidate("aft").unwrap().unwrap();
15980 cutover.incumbent.unwrap()
15981 }
15982
15983 #[tokio::test]
15992 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
15993 let mut swap = swap_with_incumbent().await;
15994 let hold = Arc::new(std::sync::Barrier::new(2));
15995 let handler = swap
15996 .handler
15997 .clone()
15998 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
15999 handler
16000 .supervisor
16001 .open_swap("aft", "candidate-nonce".to_string());
16002 let hello = {
16003 let handler = handler.clone();
16004 let candidate_ctx = swap.candidate_ctx.clone();
16005 std::thread::spawn(move || {
16006 handler
16007 .handle_hello(
16008 candidate_ctx.connection_id,
16009 Some(candidate_ctx.egress.clone()),
16010 hello_frame_with_nonce(
16011 "aft",
16012 PROTOCOL_VERSION,
16013 8,
16014 Some("candidate-nonce"),
16015 ),
16016 )
16017 .unwrap()
16018 })
16019 };
16020
16021 hold.wait();
16022 assert!(swap
16025 .registry
16026 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
16027 .unwrap()
16028 .is_some());
16029 assert!(swap
16030 .registry
16031 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16032 .unwrap()
16033 .is_none());
16034 assert!(swap
16035 .forwarding
16036 .module_endpoint_for_connection(CANDIDATE)
16037 .unwrap()
16038 .is_none());
16039
16040 hold.wait();
16041 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
16042 assert_eq!(
16043 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
16044 FrameType::HelloAck
16045 );
16046 let published = swap
16047 .registry
16048 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16049 .unwrap()
16050 .expect("candidate is published once its HELLO completes");
16051 assert_eq!(published.connection_id, CANDIDATE);
16052 assert!(swap
16053 .forwarding
16054 .module_endpoint_for_connection(CANDIDATE)
16055 .unwrap()
16056 .is_some());
16057 }
16058
16059 fn keyed_total(counters: &Value, key: &str) -> u64 {
16060 counters[key]
16061 .as_object()
16062 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
16063 .unwrap_or(0)
16064 }
16065
16066 #[tokio::test]
16067 async fn replacement_logs_old_end_and_new_admission_once() {
16068 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16069 let mut swap = swap_with_incumbent().await;
16070 swap.handler
16071 .supervisor
16072 .open_swap("aft", "candidate-nonce".to_string());
16073 hello_via_sink(
16074 &swap.handler,
16075 &swap.candidate_ctx,
16076 &mut swap.candidate_rx,
16077 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
16078 )
16079 .await;
16080 cutover(&swap);
16081 swap.handler.cleanup_connection(INCUMBENT).unwrap();
16082
16083 let captured = crate::router::test_log::captured_logs(&logs);
16084 println!("captured replacement registry logs:\n{captured}");
16085 let new_admissions: Vec<_> = captured
16086 .lines()
16087 .filter(|line| {
16088 line.contains("connection_id=40")
16089 && line.contains("swap candidate registered; not routable until cutover")
16090 })
16091 .collect();
16092 assert_eq!(
16093 new_admissions.len(),
16094 1,
16095 "unexpected admission logs: {captured}"
16096 );
16097 assert!(new_admissions[0].contains("module_id=aft"));
16098 assert!(new_admissions[0].contains("connection_id=40"));
16099 assert!(
16100 new_admissions[0].contains("swap candidate registered; not routable until cutover")
16101 );
16102 assert!(new_admissions[0].contains("ready=true"));
16103
16104 let old_admissions: Vec<_> = captured
16105 .lines()
16106 .filter(|line| {
16107 line.contains("module registered module_id=aft ")
16108 && line.contains("connection_id=30")
16109 })
16110 .collect();
16111 assert_eq!(
16112 old_admissions.len(),
16113 1,
16114 "unexpected incumbent admission: {captured}"
16115 );
16116
16117 let old_ends: Vec<_> = captured
16118 .lines()
16119 .filter(|line| {
16120 line.contains("connection_id=30") && line.contains("module registration ended")
16121 })
16122 .collect();
16123 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
16124 assert!(old_ends[0].contains("module_id=aft"));
16125 assert!(old_ends[0].contains("connection_id=30"));
16126 assert!(old_ends[0].contains("reason=replaced"));
16127 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
16128 assert_eq!(
16129 captured
16130 .lines()
16131 .filter(|line| line.contains("module registration promoted"))
16132 .count(),
16133 1,
16134 "unexpected promotion logs: {captured}"
16135 );
16136 }
16137
16138 #[tokio::test]
16144 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
16145 let mut swap = swap_with_incumbent().await;
16146 let handler = swap.handler.clone();
16147
16148 let cotenant = ConnectionId::new(31);
16150 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
16151 let (cotenant_task, cotenant_bind) = relay_route_open(
16152 &handler,
16153 cotenant,
16154 &cotenant_ctx.egress,
16155 &mut swap.incumbent_rx,
16156 100,
16157 "aft",
16158 "swap-cotenant",
16159 )
16160 .await;
16161 handler
16162 .handle_control_frame(
16163 &swap.incumbent_ctx,
16164 route_bind_ack(cotenant_bind.header.corr),
16165 )
16166 .await
16167 .unwrap();
16168 assert!(cotenant_task.await.unwrap().is_empty());
16169 let (cotenant_channel, cotenant_epoch) =
16170 published_route(&cotenant_rx.recv().await.unwrap());
16171
16172 let caller = ConnectionId::new(32);
16174 let (caller_ctx, mut caller_rx) = route_ctx(caller);
16175 let (caller_task, caller_bind) = relay_route_open(
16176 &handler,
16177 caller,
16178 &caller_ctx.egress,
16179 &mut swap.incumbent_rx,
16180 101,
16181 "aft",
16182 "swap-caller",
16183 )
16184 .await;
16185 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
16186
16187 register_candidate(&swap, None);
16188 cutover(&swap);
16189
16190 let ack = handler
16192 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
16193 .await;
16194 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
16195 if module_loop_error.is_some() {
16196 handler.cleanup_connection(INCUMBENT).unwrap();
16199 }
16200
16201 assert!(
16203 cotenant_rx.try_recv().is_err(),
16204 "the co-tenant route on the incumbent was torn down by one late ack: \
16205 {module_loop_error:?}"
16206 );
16207 assert!(matches!(
16208 swap.forwarding
16209 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
16210 .unwrap(),
16211 DataRoute::Client(DataRouteState::Bound(_))
16212 ));
16213 assert_eq!(module_loop_error, None);
16214 assert!(swap
16215 .registry
16216 .get_module_by_connection(INCUMBENT)
16217 .unwrap()
16218 .is_some());
16219
16220 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
16222 .await
16223 .expect("the incumbent is told to drop the abandoned binding")
16224 .unwrap()
16225 .frame;
16226 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16227 assert_eq!(goodbye.header.channel, abandoned_channel);
16228 assert_eq!(goodbye.header.epoch, abandoned_epoch);
16229 assert!(swap.incumbent_rx.try_recv().is_err());
16230
16231 let response = caller_task.await.unwrap();
16233 assert_eq!(response.len(), 1);
16234 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
16235 assert!(caller_rx.try_recv().is_err());
16236
16237 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
16241 let counters = handler.counters().snapshot();
16242 assert_eq!(
16243 keyed_total(&counters, "route_open_accepted_by_principal"),
16244 1
16245 );
16246 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
16247 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
16248 }
16249
16250 #[tokio::test]
16254 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
16255 let mut swap = swap_with_incumbent().await;
16256 register_candidate(&swap, None);
16257 let incumbent = cutover(&swap);
16258 swap.forwarding
16259 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
16260 .unwrap()
16261 .expect("the incumbent is still registered");
16262
16263 let client = ConnectionId::new(33);
16264 let (client_ctx, mut client_rx) = route_ctx(client);
16265 let route_handler = swap.handler.clone();
16266 let open_ctx = RouteCtx {
16267 connection_id: client,
16268 egress: client_ctx.egress.clone(),
16269 };
16270 let mut route_task = tokio::spawn(async move {
16271 route_handler
16272 .handle_control_frame(
16273 &open_ctx,
16274 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
16275 )
16276 .await
16277 .unwrap()
16278 });
16279 let bind = tokio::select! {
16280 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
16281 response = &mut route_task => {
16282 let response = response.unwrap();
16283 panic!(
16284 "post-cutover route.open was refused instead of relayed to the candidate: {}",
16285 parse_error(&response[0])["code"]
16286 );
16287 }
16288 };
16289 swap.handler
16290 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
16291 .await
16292 .unwrap();
16293 assert!(route_task.await.unwrap().is_empty());
16294 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
16295 match swap
16296 .forwarding
16297 .lookup_data_route(client, channel, epoch)
16298 .unwrap()
16299 {
16300 DataRoute::Client(DataRouteState::Bound(route)) => {
16301 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
16302 }
16303 other => panic!("expected a bound route on the candidate, got {other:?}"),
16304 }
16305 assert!(swap.incumbent_rx.try_recv().is_err());
16306 }
16307
16308 #[tokio::test]
16313 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
16314 let swap = swap_with_incumbent().await;
16315 register_candidate(&swap, Some(false));
16316 let update = Frame::build(
16317 FrameType::Request,
16318 control_flags(),
16319 0,
16320 0,
16321 55,
16322 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16323 provides: manifest("aft", PROTOCOL_VERSION).provides,
16324 capabilities: None,
16325 ready: Some(true),
16326 })
16327 .unwrap(),
16328 )
16329 .unwrap();
16330
16331 let replies = swap
16332 .handler
16333 .handle_control_frame(&swap.candidate_ctx, update)
16334 .await
16335 .unwrap();
16336
16337 assert_eq!(replies.len(), 1);
16338 assert_eq!(
16339 replies[0].header.ty,
16340 FrameType::Response,
16341 "candidate catalog.update was refused: {:?}",
16342 serde_json::from_slice::<Value>(&replies[0].body).ok()
16343 );
16344 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
16345 assert_eq!(
16346 swap.registry
16347 .get_module("aft")
16348 .unwrap()
16349 .unwrap()
16350 .connection_id,
16351 INCUMBENT
16352 );
16353 }
16354 }
16355
16356 mod swap_admission {
16360 use super::*;
16361
16362 const INCUMBENT_NONCE: &str = "incumbent-nonce";
16363 const CANDIDATE_NONCE: &str = "candidate-nonce";
16364
16365 fn handler_with_incumbent(
16366 module_id: &str,
16367 reserved: bool,
16368 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
16369 let registry = Arc::new(Registry::default());
16370 let supervisor = SupervisorHandle::new();
16371 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
16372 if reserved {
16373 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
16374 }
16375 let handler =
16376 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
16377 let incumbent = handler
16378 .handle_control(
16379 ConnectionId::new(1),
16380 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
16381 )
16382 .unwrap();
16383 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
16384 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
16385 (registry, supervisor, handler)
16386 }
16387
16388 #[test]
16394 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
16395 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
16396
16397 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
16398 let replies = handler
16399 .handle_control(
16400 ConnectionId::new(connection),
16401 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
16402 )
16403 .unwrap();
16404 assert_eq!(replies[0].header.ty, FrameType::Error);
16405 assert_eq!(
16406 parse_error(&replies[0])["code"],
16407 "swap_token_invalid",
16408 "nonce {nonce:?}"
16409 );
16410 }
16411 assert!(registry.get_candidate("aft").unwrap().is_none());
16412 assert_eq!(
16413 registry.get_module("aft").unwrap().unwrap().connection_id,
16414 ConnectionId::new(1)
16415 );
16416
16417 let admitted = handler
16420 .handle_control(
16421 ConnectionId::new(4),
16422 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
16423 )
16424 .unwrap();
16425 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
16426 assert_eq!(
16427 registry
16428 .get_candidate("aft")
16429 .unwrap()
16430 .unwrap()
16431 .connection_id,
16432 ConnectionId::new(4)
16433 );
16434 assert_eq!(
16435 registry.get_module("aft").unwrap().unwrap().connection_id,
16436 ConnectionId::new(1),
16437 "the candidate must not take the active slot"
16438 );
16439 let replayed = handler
16440 .handle_control(
16441 ConnectionId::new(5),
16442 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
16443 )
16444 .unwrap();
16445 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
16446
16447 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
16451 let squatter = handler
16452 .handle_control(
16453 ConnectionId::new(6),
16454 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
16455 )
16456 .unwrap();
16457 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
16458 assert!(
16459 registry.get_module("aft").unwrap().is_none(),
16460 "a squatter took the active slot of an id being swapped"
16461 );
16462 }
16463
16464 #[test]
16469 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
16470 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
16471
16472 let replies = handler
16473 .handle_control(
16474 ConnectionId::new(2),
16475 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16476 )
16477 .unwrap();
16478
16479 assert_eq!(
16480 replies[0].header.ty,
16481 FrameType::HelloAck,
16482 "reserved candidate refused: {:?}",
16483 serde_json::from_slice::<Value>(&replies[0].body).ok()
16484 );
16485 assert_eq!(
16486 registry
16487 .get_candidate("vault")
16488 .unwrap()
16489 .unwrap()
16490 .connection_id,
16491 ConnectionId::new(2)
16492 );
16493 }
16494
16495 #[test]
16498 fn without_an_open_swap_the_ordinary_gates_decide() {
16499 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
16500 supervisor.close_swap("vault");
16501
16502 let candidate = handler
16503 .handle_control(
16504 ConnectionId::new(2),
16505 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16506 )
16507 .unwrap();
16508 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
16509 let duplicate = handler
16510 .handle_control(
16511 ConnectionId::new(3),
16512 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
16513 )
16514 .unwrap();
16515 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
16516 assert!(registry.get_candidate("vault").unwrap().is_none());
16517 }
16518 }
16519
16520 mod scopes {
16524 mod idle_tests {
16525 use super::*;
16526 include!("scope_idle_tests.rs");
16527 }
16528 mod route_open_timings {
16529 use super::*;
16530 include!("route_open_timing_tests.rs");
16531 }
16532 mod route_open_refusal_log {
16533 use super::*;
16534 include!("route_open_refusal_log_tests.rs");
16535 }
16536 mod activity {
16537 include!("activity_scope_tests.rs");
16538 }
16539 mod delta {
16540 include!("scope_handler_delta_tests.rs");
16541 }
16542 mod call_modes {
16543 include!("scope_call_mode_tests.rs");
16544 }
16545 #[tokio::test]
16546 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
16547 let rig = rig().await;
16548 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
16549 "s".into(),
16550 1,
16551 "tool".into(),
16552 ))
16553 .unwrap();
16554 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
16555 let (direct, _rx) = wide_ctx(999);
16556 for (ctx, expected) in [
16557 (&rig.owner, FrameType::Response),
16558 (&direct, FrameType::Error),
16559 ] {
16560 let frame =
16561 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
16562 .unwrap();
16563 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
16564 assert_eq!(replies[0].header.ty, expected);
16565 if expected == FrameType::Response {
16566 let reply: subc_control::ScopeServedByReply =
16567 serde_json::from_slice(&replies[0].body).unwrap();
16568 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
16569 } else {
16570 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
16571 }
16572 }
16573 let reply = call(
16574 &rig.handler,
16575 &rig.owner,
16576 &ModuleControlRequestFromModule::ScopeDescribe {
16577 owner: Principal::Reserved {
16578 module_id: OWNER.into(),
16579 },
16580 scope_ref: "s".into(),
16581 },
16582 )
16583 .await;
16584 assert!(matches!(
16585 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
16586 ModuleControlResponseToModule::ScopeDescribe { .. }
16587 ));
16588 }
16589
16590 use subc_protocol::scope::{
16591 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
16592 ScopeStatus,
16593 };
16594
16595 use super::*;
16596
16597 const OWNER: &str = "prefrontal-core";
16598
16599 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
16600 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
16601 }
16602
16603 async fn call(
16604 handler: &ControlHandler,
16605 ctx: &RouteCtx,
16606 request: &ModuleControlRequestFromModule,
16607 ) -> Frame {
16608 let body = serde_json::to_vec(request).unwrap();
16609 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16610 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16611 assert_eq!(replies.len(), 1, "{replies:?}");
16612 replies.pop().unwrap()
16613 }
16614
16615 async fn sync(
16616 handler: &ControlHandler,
16617 ctx: &RouteCtx,
16618 generation: u64,
16619 scopes: Vec<ScopeRecord>,
16620 ) -> Result<ModuleControlResponseToModule, String> {
16621 let reply = call(
16622 handler,
16623 ctx,
16624 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16625 )
16626 .await;
16627 match reply.header.ty {
16628 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16629 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16630 }
16631 }
16632
16633 async fn describe(
16634 handler: &ControlHandler,
16635 ctx: &RouteCtx,
16636 owner: &str,
16637 scope_ref: &str,
16638 ) -> ModuleControlResponseToModule {
16639 let reply = call(
16640 handler,
16641 ctx,
16642 &ModuleControlRequestFromModule::ScopeDescribe {
16643 owner: Principal::Reserved {
16644 module_id: owner.to_string(),
16645 },
16646 scope_ref: scope_ref.to_string(),
16647 },
16648 )
16649 .await;
16650 assert_eq!(
16651 reply.header.ty,
16652 FrameType::Response,
16653 "{:?}",
16654 parse_error(&reply)
16655 );
16656 serde_json::from_slice(&reply.body).unwrap()
16657 }
16658
16659 #[tokio::test]
16660 async fn scope_describe_never_echoes_bind_proof_or_hash() {
16661 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16662 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16663 let mut rig = rig().await;
16664 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16665 .await;
16666 let _bound = rig
16667 .bound(
16668 Some(AFT),
16669 PLEXUS,
16670 Some(
16671 rig.selector("s", Some(1))
16672 .with_bind_proof(Some(PROOF.into())),
16673 ),
16674 )
16675 .await;
16676 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
16677 let output = serde_json::to_string(&reply).unwrap();
16678 assert!(output.contains("\"status\":\"live\""));
16679 assert!(!output.contains(PROOF));
16680 assert!(!output.contains(HASH));
16681 }
16682
16683 async fn module(
16685 handler: &ControlHandler,
16686 connection: u64,
16687 module_id: &str,
16688 nonce: Option<&str>,
16689 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16690 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
16691 hello_via_sink(
16692 handler,
16693 &ctx,
16694 &mut rx,
16695 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
16696 )
16697 .await;
16698 (ctx, rx)
16699 }
16700
16701 #[tokio::test]
16704 async fn a_client_connection_cannot_sync_or_describe() {
16705 let handler = ControlHandler::new(Arc::new(Registry::default()));
16706 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
16707 for request in [
16708 ModuleControlRequestFromModule::ScopeSync {
16709 generation: 1,
16710 scopes: vec![head("s", 1)],
16711 },
16712 ModuleControlRequestFromModule::ScopeDescribe {
16713 owner: Principal::Direct,
16714 scope_ref: "s".to_string(),
16715 },
16716 ] {
16717 let reply = call(&handler, &ctx, &request).await;
16718 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
16719 }
16720 assert!(
16721 !handler
16722 .scopes
16723 .read()
16724 .unwrap()
16725 .describe(
16726 &Principal::Reserved {
16727 module_id: OWNER.to_string()
16728 },
16729 "s"
16730 )
16731 .owner_synced
16732 );
16733 }
16734
16735 #[tokio::test]
16738 async fn a_module_without_a_supervised_launch_cannot_sync() {
16739 let handler = ControlHandler::new(Arc::new(Registry::default()));
16740 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
16741 assert_eq!(
16742 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
16743 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16744 );
16745 }
16746
16747 #[tokio::test]
16748 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
16749 let supervisor = SupervisorHandle::new();
16750 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16751 let handler = ControlHandler::new(Arc::new(Registry::default()))
16752 .with_supervisor(supervisor.clone());
16753 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16754 sync(&handler, &incumbent, 1, vec![head("s", 1)])
16755 .await
16756 .expect("the current launch syncs");
16757
16758 supervisor.open_swap(OWNER, "n2".to_string());
16761 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
16762 assert_eq!(
16763 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16764 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16765 );
16766 sync(&handler, &incumbent, 2, vec![head("s", 1)])
16767 .await
16768 .expect("the serving owner syncs during the swap");
16769
16770 supervisor.close_swap(OWNER);
16773 assert_eq!(
16774 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16775 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16776 );
16777 sync(&handler, &incumbent, 3, vec![head("s", 1)])
16778 .await
16779 .expect("the serving owner syncs after the rollback");
16780 handler.cleanup_connection(candidate.connection_id).unwrap();
16781
16782 supervisor.open_swap(OWNER, "n3".to_string());
16787 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
16788 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
16789 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
16790 .await
16791 .expect("the promoted launch takes authority");
16792 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16793 panic!("unexpected reply {reply:?}");
16794 };
16795 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
16796 assert_eq!(
16797 sync(&handler, &incumbent, 4, Vec::new()).await,
16798 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16799 );
16800 }
16801
16802 #[tokio::test]
16805 async fn closing_the_authority_connection_frees_sync_authority() {
16806 let supervisor = SupervisorHandle::new();
16807 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16808 let handler = ControlHandler::new(Arc::new(Registry::default()))
16809 .with_supervisor(supervisor.clone());
16810 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16811 sync(&handler, &first, 10, vec![head("s", 1)])
16812 .await
16813 .unwrap();
16814 handler.cleanup_connection(first.connection_id).unwrap();
16815
16816 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
16817 sync(&handler, &second, 1, vec![head("s", 1)])
16818 .await
16819 .expect("the next connection takes the released authority");
16820 }
16821
16822 #[tokio::test]
16823 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
16824 let supervisor = SupervisorHandle::new();
16825 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16826 let handler =
16827 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
16828 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
16829 sync(&handler, &first, 10, vec![head("s", 1)])
16830 .await
16831 .unwrap();
16832 handler
16833 .handle_control_frame(
16834 &first,
16835 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
16836 )
16837 .await
16838 .unwrap();
16839 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
16840 sync(&handler, &second, 1, vec![head("s", 1)])
16841 .await
16842 .expect("GOODBYE releases authority even if the old socket remains open");
16843 }
16844
16845 #[tokio::test]
16846 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
16847 let registry = Arc::new(Registry::default());
16848 let supervisor_handle = SupervisorHandle::new();
16849 let supervisor =
16850 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16851 .with_handle(supervisor_handle.clone())
16852 .with_daemon_incarnation("incarnation-7".to_string());
16853 supervisor
16856 .supervise_configured(
16857 ModuleSpec {
16858 module_id: OWNER.to_string(),
16859 program: PathBuf::from("/nonexistent/prefrontal-core"),
16860 args: Vec::new(),
16861 env: Vec::new(),
16862 reserved: false,
16863 reserved_prefixes: Vec::new(),
16864 protocol: ModuleProtocol::Subc,
16865 overlap: Default::default(),
16866 },
16867 false,
16868 )
16869 .unwrap();
16870 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
16871 let handler =
16872 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
16873 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
16874
16875 let ModuleControlResponseToModule::ScopeDescribe {
16877 status,
16878 daemon_incarnation,
16879 owner_synced,
16880 owner_configured,
16881 scope,
16882 ..
16883 } = describe(&handler, &reader, OWNER, "s").await
16884 else {
16885 panic!("not a describe reply");
16886 };
16887 assert_eq!(status, ScopeStatus::NotLive);
16888 assert_eq!(daemon_incarnation, "incarnation-7");
16889 assert!(!owner_synced);
16890 assert!(owner_configured);
16891 assert!(scope.is_none());
16892
16893 let ModuleControlResponseToModule::ScopeDescribe {
16896 status,
16897 owner_configured,
16898 ..
16899 } = describe(&handler, &reader, "ghost", "s").await
16900 else {
16901 panic!("not a describe reply");
16902 };
16903 assert_eq!(status, ScopeStatus::NotLive);
16904 assert!(!owner_configured);
16905
16906 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
16908 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
16909 let ModuleControlResponseToModule::ScopeDescribe {
16910 status,
16911 scope_epoch,
16912 owner_synced,
16913 scope,
16914 ..
16915 } = describe(&handler, &reader, OWNER, "s").await
16916 else {
16917 panic!("not a describe reply");
16918 };
16919 assert_eq!(status, ScopeStatus::Live);
16920 assert_eq!(scope_epoch, Some(4));
16921 assert!(owner_synced);
16922 let stamp = scope.expect("a live scope carries its stamp");
16923 assert!(
16924 stamp.owner_authorized,
16925 "prefrontal-core is the default authority"
16926 );
16927 assert_eq!(stamp.kind, ScopeKind::Head);
16928 }
16929
16930 #[tokio::test]
16931 async fn scope_authority_owners_decides_owner_authorized() {
16932 let supervisor = SupervisorHandle::new();
16933 supervisor.set_spawn_nonce("broca", "b1".to_string());
16934 let handler = ControlHandler::new(Arc::new(Registry::default()))
16935 .with_supervisor(supervisor)
16936 .with_scope_authority_owners(vec!["broca".to_string()]);
16937 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
16938 let mut gated = head("s", 1);
16939 gated.attributes.agent_id = Some("agent".to_string());
16940 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
16941 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
16942 describe(&handler, &broca, "broca", "s").await
16943 else {
16944 panic!("not a describe reply");
16945 };
16946 assert!(scope.unwrap().owner_authorized);
16947 }
16948
16949 #[tokio::test]
16953 async fn scope_ops_and_the_scopes_capability_are_advertised() {
16954 let handler = ControlHandler::new(Arc::new(Registry::default()));
16955 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
16956 let ack = hello_via_sink(
16957 &handler,
16958 &ctx,
16959 &mut rx,
16960 hello_frame("m", PROTOCOL_VERSION, 1),
16961 )
16962 .await;
16963 let ack = parse_ack(&ack);
16964 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
16965 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
16966 }
16967 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
16968
16969 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
16970 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
16971 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
16972 let reply = handler
16973 .handle_control_frame(&client, frame)
16974 .await
16975 .unwrap()
16976 .pop()
16977 .unwrap();
16978 let ClientControlResponse::ServerDescribe { capabilities, .. } =
16979 serde_json::from_slice(&reply.body).unwrap()
16980 else {
16981 panic!("not a server.describe reply");
16982 };
16983 assert!(
16984 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
16985 "{capabilities:?}"
16986 );
16987 }
16988
16989 const PLEXUS: &str = "plexus";
16992 const OTHER: &str = "other";
16993 const AFT: &str = "aft";
16994 const BROCA: &str = "broca";
16995 const MAGIC: &str = "magic-context";
16996
16997 fn nonce(module_id: &str) -> String {
16998 format!("nonce-{module_id}")
16999 }
17000
17001 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17002 let (tx, rx) = mpsc::channel(64);
17003 (
17004 RouteCtx {
17005 connection_id: ConnectionId::new(connection),
17006 egress: FrameSink::new(tx),
17007 },
17008 rx,
17009 )
17010 }
17011
17012 struct Rig {
17016 handler: ControlHandler,
17017 forwarding: Arc<ForwardingTable>,
17018 owner: RouteCtx,
17019 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17020 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
17021 generation: u64,
17022 next_connection: u64,
17023 _supervisor: Supervisor,
17024 }
17025
17026 async fn rig() -> Rig {
17027 rig_with_flow_support(true).await
17028 }
17029
17030 async fn rig_with_flow_support(flow_support: bool) -> Rig {
17031 rig_with_scope_capabilities(flow_support, false).await
17032 }
17033
17034 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
17035 let mut provides = Vec::new();
17036 if flow_support {
17037 provides.push("flow-scopes/v1");
17038 }
17039 if run_support {
17040 provides.push("agent-run-scopes/v1");
17041 }
17042 rig_with_target_capabilities(&provides).await
17043 }
17044
17045 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
17046 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
17047 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17048 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17051 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
17052 Frame::build(
17053 FrameType::Hello,
17054 control_flags(),
17055 0,
17056 0,
17057 connection,
17058 serde_json::to_vec(&body).unwrap(),
17059 )
17060 .unwrap()
17061 }
17062
17063 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
17064 let registry = Arc::new(Registry::default());
17065 let forwarding = Arc::new(ForwardingTable::default());
17066 let supervisor_handle = SupervisorHandle::new();
17067 let supervisor =
17068 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
17069 .with_handle(supervisor_handle.clone());
17070 supervisor
17071 .supervise_configured(
17072 ModuleSpec {
17073 module_id: OWNER.to_string(),
17074 program: PathBuf::from("/nonexistent/prefrontal-core"),
17075 args: Vec::new(),
17076 env: Vec::new(),
17077 reserved: false,
17078 reserved_prefixes: Vec::new(),
17079 protocol: ModuleProtocol::Subc,
17080 overlap: Default::default(),
17081 },
17082 false,
17083 )
17084 .unwrap();
17085 for module_id in [OWNER, AFT, BROCA, MAGIC] {
17086 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
17087 }
17088 let handler =
17089 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17090 .with_supervisor(supervisor_handle);
17091 let (owner, mut owner_rx) = wide_ctx(1);
17092 hello_via_sink(
17093 &handler,
17094 &owner,
17095 &mut owner_rx,
17096 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
17097 )
17098 .await;
17099 let mut modules = BTreeMap::new();
17100 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
17101 let (ctx, mut rx) = wide_ctx(connection);
17102 let hello = hello_with_capabilities(module_id, connection, provides);
17103 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
17104 modules.insert(module_id.to_string(), (ctx, rx));
17105 }
17106 Rig {
17107 handler,
17108 forwarding,
17109 owner,
17110 _owner_rx: owner_rx,
17111 modules,
17112 generation: 0,
17113 next_connection: 100,
17114 _supervisor: supervisor,
17115 }
17116 }
17117
17118 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
17119 ScopeCarrier::new(Principal::Reserved {
17120 module_id: module_id.to_string(),
17121 })
17122 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
17123 }
17124
17125 fn session(scope_epoch: u64) -> ScopeRecord {
17128 let mut record = head("s", scope_epoch);
17129 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
17130 record.attributes.agent_id = Some("agent-1".to_string());
17131 record.attributes.delegates = true;
17132 record
17133 }
17134
17135 impl Rig {
17136 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
17137 self.generation += 1;
17138 sync(&self.handler, &self.owner, self.generation, scopes)
17139 .await
17140 .expect("the owner's sync is accepted");
17141 }
17142
17143 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17144 ScopeSelector::new(
17145 Principal::Reserved {
17146 module_id: OWNER.to_string(),
17147 },
17148 scope_ref.to_string(),
17149 scope_epoch,
17150 )
17151 }
17152
17153 fn open_frame(
17154 &mut self,
17155 opener: Option<&str>,
17156 target: &str,
17157 scope: Option<ScopeSelector>,
17158 ) -> (
17159 RouteCtx,
17160 mpsc::Receiver<crate::router::OutboundFrame>,
17161 Frame,
17162 ) {
17163 self.next_connection += 1;
17164 let (ctx, rx) = wide_ctx(self.next_connection);
17165 let root = unique_project_root("scoped-open");
17166 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
17167 target: RouteTarget::ToolProvider {
17168 module_id: target.to_string(),
17169 },
17170 identity: BindIdentity::new(
17171 root.path().to_path_buf(),
17172 "unit".to_string(),
17173 "session".to_string(),
17174 ),
17175 consumer_identity: opener.map(|module_id| ConsumerIdentity {
17176 module_id: module_id.to_string(),
17177 launch_nonce: nonce(module_id),
17178 }),
17179 consumer_capabilities: None,
17180 role_versions: None,
17181 admission_facts: None,
17182 scope,
17183 })
17184 .unwrap();
17185 let frame = Frame::build(
17186 FrameType::Request,
17187 control_flags(),
17188 0,
17189 0,
17190 self.next_connection,
17191 body,
17192 )
17193 .unwrap();
17194 (ctx, rx, frame)
17195 }
17196
17197 async fn refused(
17199 &mut self,
17200 opener: Option<&str>,
17201 target: &str,
17202 scope: Option<ScopeSelector>,
17203 ) -> String {
17204 self.refusal_body(opener, target, scope).await["code"]
17205 .as_str()
17206 .unwrap()
17207 .to_string()
17208 }
17209
17210 async fn refusal_body(
17211 &mut self,
17212 opener: Option<&str>,
17213 target: &str,
17214 scope: Option<ScopeSelector>,
17215 ) -> Value {
17216 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
17217 let replies = tokio::time::timeout(
17218 Duration::from_secs(2),
17219 self.handler.handle_control_frame(&ctx, frame),
17220 )
17221 .await
17222 .expect("the open must be refused before waiting for a bind ack")
17223 .unwrap();
17224 assert_eq!(replies.len(), 1, "{replies:?}");
17225 assert_eq!(replies[0].header.ty, FrameType::Error);
17226 let (_, module_rx) = self.modules.get_mut(target).unwrap();
17227 assert!(
17228 module_rx.try_recv().is_err(),
17229 "a refused open relays nothing"
17230 );
17231 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
17232 parse_error(&replies[0])
17233 }
17234
17235 async fn relayed(
17237 &mut self,
17238 opener: Option<&str>,
17239 target: &str,
17240 scope: Option<ScopeSelector>,
17241 ) -> Relayed {
17242 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
17243 let handler = self.handler.clone();
17244 let task_ctx = ctx.clone();
17245 let task = tokio::spawn(async move {
17246 handler
17247 .handle_control_frame(&task_ctx, frame)
17248 .await
17249 .unwrap()
17250 });
17251 let (_, module_rx) = self.modules.get_mut(target).unwrap();
17252 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
17253 .await
17254 .expect("the target receives the relayed route.bind")
17255 .unwrap()
17256 .frame;
17257 Relayed {
17258 target: target.to_string(),
17259 client: ctx,
17260 client_rx: rx,
17261 task,
17262 bind,
17263 }
17264 }
17265
17266 async fn ack(&self, relayed: &Relayed) {
17267 let (module, _) = &self.modules[&relayed.target];
17268 self.handler
17269 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
17270 .await
17271 .unwrap();
17272 }
17273
17274 async fn bound(
17276 &mut self,
17277 opener: Option<&str>,
17278 target: &str,
17279 scope: Option<ScopeSelector>,
17280 ) -> Bound {
17281 let relayed = self.relayed(opener, target, scope).await;
17282 self.ack(&relayed).await;
17283 let Relayed {
17284 target,
17285 client,
17286 mut client_rx,
17287 task,
17288 bind,
17289 } = relayed;
17290 assert!(
17291 task.await.unwrap().is_empty(),
17292 "the open is answered by commit"
17293 );
17294 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
17295 Bound {
17296 target,
17297 client,
17298 client_rx,
17299 channel,
17300 epoch,
17301 bind,
17302 }
17303 }
17304
17305 fn live(&self, route: &Bound) -> bool {
17306 matches!(
17307 self.forwarding
17308 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17309 .unwrap(),
17310 DataRoute::Client(DataRouteState::Bound(_))
17311 )
17312 }
17313 }
17314
17315 struct Relayed {
17316 target: String,
17317 client: RouteCtx,
17318 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17319 task: tokio::task::JoinHandle<Vec<Frame>>,
17320 bind: Frame,
17321 }
17322
17323 struct Bound {
17324 target: String,
17325 client: RouteCtx,
17326 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17327 channel: u16,
17328 epoch: u32,
17329 bind: Frame,
17330 }
17331
17332 impl Bound {
17333 fn closed_reason(&mut self) -> RouteCloseReason {
17336 let mut reason = None;
17337 let mut goodbye = false;
17338 while let Ok(outbound) = self.client_rx.try_recv() {
17339 let frame = outbound.frame;
17340 match frame.header.ty {
17341 FrameType::Goodbye => {
17342 assert_eq!(
17343 (frame.header.channel, frame.header.epoch),
17344 (self.channel, self.epoch)
17345 );
17346 goodbye = true;
17347 }
17348 FrameType::Push => {
17349 let ClientControlPush::RouteClosed {
17350 reason: r,
17351 module_id,
17352 ..
17353 } = serde_json::from_slice(&frame.body).unwrap()
17354 else {
17355 panic!("unexpected push");
17356 };
17357 assert_eq!(module_id, self.target);
17358 reason = Some(r);
17359 }
17360 other => panic!("unexpected frame {other:?}"),
17361 }
17362 }
17363 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
17364 reason.expect("the client is told why the route closed")
17365 }
17366
17367 fn untouched(&mut self) -> bool {
17368 self.client_rx.try_recv().is_err()
17369 }
17370
17371 fn stamp(&self) -> Option<ScopeStamp> {
17372 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
17373 ModuleControlRequest::RouteBind { scope, .. } => scope,
17374 other => panic!("expected a route.bind, got {other:?}"),
17375 }
17376 }
17377 }
17378
17379 #[tokio::test]
17380 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
17381 ) {
17382 let mut rig = rig().await;
17383 let mut record = session(1);
17384 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
17385 record.child_owners = vec![Principal::Reserved {
17386 module_id: MAGIC.to_string(),
17387 }];
17388 rig.sync(vec![record]).await;
17389 let scope = || Some(rig_selector("s", Some(1)));
17390
17391 rig.bound(Some(OWNER), PLEXUS, scope()).await;
17394 rig.bound(Some(AFT), OTHER, scope()).await;
17395 rig.bound(Some(BROCA), PLEXUS, scope()).await;
17396
17397 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
17401 assert_eq!(
17402 rig.refused(opener, target, scope()).await,
17403 error_codes::SCOPE_NOT_CARRIER,
17404 "{opener:?} -> {target}"
17405 );
17406 }
17407 }
17408
17409 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17410 ScopeSelector::new(
17411 Principal::Reserved {
17412 module_id: OWNER.to_string(),
17413 },
17414 scope_ref.to_string(),
17415 scope_epoch,
17416 )
17417 }
17418
17419 #[tokio::test]
17420 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
17421 let mut rig = rig().await;
17422 rig.sync(vec![session(1)]).await;
17423 for opener in [OWNER, AFT] {
17424 assert_eq!(
17425 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
17426 .await,
17427 error_codes::SCOPE_EPOCH_REQUIRED,
17428 "{opener}"
17429 );
17430 }
17431 }
17432
17433 #[tokio::test]
17434 async fn admission_separates_not_synced_not_live_and_ended() {
17435 let mut rig = rig().await;
17436 let code = rig
17438 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17439 .await;
17440 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
17441 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
17442
17443 let ghost = ScopeSelector::new(
17445 Principal::Reserved {
17446 module_id: "ghost".to_string(),
17447 },
17448 "s".to_string(),
17449 Some(1),
17450 );
17451 assert_eq!(
17452 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
17453 error_codes::SCOPE_NOT_LIVE
17454 );
17455
17456 rig.sync(vec![session(2)]).await;
17457 assert_eq!(
17458 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
17459 .await,
17460 error_codes::SCOPE_NOT_LIVE
17461 );
17462 for epoch in [1, 3] {
17463 assert_eq!(
17464 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
17465 .await,
17466 error_codes::SCOPE_ENDED,
17467 "epoch {epoch}"
17468 );
17469 }
17470 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
17472 .await;
17473 }
17474
17475 #[tokio::test]
17476 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
17477 let mut rig = rig().await;
17478 rig.sync(vec![session(1)]).await;
17479 let route = rig
17480 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17481 .await;
17482 let stamp = route.stamp().expect("a scoped bind carries the stamp");
17483 assert_eq!(stamp.scope_ref, "s");
17484 assert_eq!(stamp.scope_epoch, 1);
17485 assert_eq!(stamp.kind, ScopeKind::Head);
17486 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
17487 assert!(stamp.attributes.delegates);
17488 assert!(stamp.owner_authorized);
17489 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17490 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
17491
17492 let (broca, mut broca_rx) = wide_ctx(50);
17495 hello_via_sink(
17496 &rig.handler,
17497 &broca,
17498 &mut broca_rx,
17499 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
17500 )
17501 .await;
17502 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
17503 .await
17504 .unwrap();
17505 let own = ScopeSelector::new(
17506 Principal::Reserved {
17507 module_id: BROCA.to_string(),
17508 },
17509 "b".to_string(),
17510 Some(1),
17511 );
17512 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
17513 assert!(!route.stamp().unwrap().owner_authorized);
17514 }
17515
17516 #[tokio::test]
17517 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
17518 let mut rig = rig().await;
17519 let mut record = head("s", 1);
17520 record.carriers = vec![carrier(AFT, None)];
17521 let flow_id = "Flow:run-7/step_2!~";
17522 record.attributes.flow_id = Some(flow_id.to_string());
17523 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
17524 .await
17525 .unwrap();
17526 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17527 panic!("not a sync reply");
17528 };
17529 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
17530 let route = rig
17531 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17532 .await;
17533 assert!(rig.live(&route), "the stamped bind committed");
17534 let stamp = route.stamp().expect("a flow scope carries a stamp");
17535 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
17536 assert_eq!(stamp.attributes.agent_id, None);
17537 assert!(!stamp.attributes.delegates);
17538 assert!(stamp.owner_authorized);
17539 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17540 assert_eq!(unscoped.stamp(), None);
17541 }
17542
17543 #[tokio::test]
17544 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
17545 let mut rig = rig_with_flow_support(false).await;
17546 let mut record = session(1);
17547 record.attributes.flow_id = Some("flow:7".to_string());
17548 rig.sync(vec![record]).await;
17549 for opener in [OWNER, AFT] {
17550 let body = rig
17551 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17552 .await;
17553 assert_eq!(body["code"], "target_flow_unsupported");
17554 let message = body["message"].as_str().unwrap();
17555 for required in [PLEXUS, "flow-scopes/v1"] {
17556 assert!(message.contains(required), "{message}");
17557 }
17558 }
17559 }
17560
17561 #[tokio::test]
17562 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
17563 let mut rig = rig_with_flow_support(true).await;
17564 let mut record = session(1);
17565 record.attributes.flow_id = Some("flow:7".to_string());
17566 rig.sync(vec![record]).await;
17567 for opener in [OWNER, AFT] {
17568 let route = rig
17569 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17570 .await;
17571 assert!(rig.live(&route));
17572 assert_eq!(
17573 route.stamp().unwrap().attributes.flow_id.as_deref(),
17574 Some("flow:7")
17575 );
17576 }
17577 }
17578
17579 #[tokio::test]
17580 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
17581 use std::future::Future;
17582
17583 let mut rig = rig_with_flow_support(true).await;
17584 let mut record = session(1);
17585 record.attributes.flow_id = Some("flow:7".to_string());
17586 rig.sync(vec![record]).await;
17587 let (client, mut client_rx, frame) =
17588 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
17589 for _ in 0..64 {
17592 client.egress.try_send(route_bind_ack(1)).unwrap();
17593 }
17594 let handler = rig.handler.clone();
17595 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17596 std::future::poll_fn(|cx| {
17597 assert!(open.as_mut().poll(cx).is_pending());
17598 std::task::Poll::Ready(())
17599 })
17600 .await;
17601 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17602
17603 let old_connection = rig.modules[PLEXUS].0.connection_id;
17604 rig.handler.cleanup_connection(old_connection).unwrap();
17605 let (replacement, mut replacement_rx) = wide_ctx(200);
17606 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17607 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17608 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17609 let hello = Frame::build(
17610 FrameType::Hello,
17611 control_flags(),
17612 0,
17613 0,
17614 200,
17615 serde_json::to_vec(&body).unwrap(),
17616 )
17617 .unwrap();
17618 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17619
17620 client_rx.try_recv().unwrap();
17621 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17622 .await
17623 .expect("the replacement is refused without waiting for a bind ack")
17624 .unwrap();
17625 assert_eq!(replies.len(), 1);
17626 let body = parse_error(&replies[0]);
17627 assert_eq!(body["code"], "target_flow_unsupported");
17628 for required in [PLEXUS, "flow-scopes/v1"] {
17629 assert!(body["message"].as_str().unwrap().contains(required));
17630 }
17631 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
17632 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17633 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17634 assert!(rig
17635 .handler
17636 .registry
17637 .get_module_by_connection(replacement.connection_id)
17638 .unwrap()
17639 .is_some());
17640 }
17641
17642 #[tokio::test]
17643 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
17644 use std::future::Future;
17645 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17646 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17647 const NEXT_HASH: &str =
17648 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17649 let mut rig = rig().await;
17650 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
17651 record.carriers.push(ScopeCarrier::new(Principal::Direct));
17652 rig.sync(vec![record.clone()]).await;
17653 let (client, mut rx, frame) = rig.open_frame(
17654 None,
17655 PLEXUS,
17656 Some(
17657 rig.selector("s", Some(1))
17658 .with_bind_proof(Some(PROOF.into())),
17659 ),
17660 );
17661 for _ in 0..64 {
17664 client.egress.try_send(route_bind_ack(1)).unwrap();
17665 }
17666 let handler = rig.handler.clone();
17667 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17668 std::future::poll_fn(|cx| {
17669 assert!(open.as_mut().poll(cx).is_pending());
17670 std::task::Poll::Ready(())
17671 })
17672 .await;
17673 record.bind_proof_sha256 = Some(NEXT_HASH.into());
17674 rig.sync(vec![record]).await;
17675 rx.try_recv().unwrap();
17676 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17677 .await
17678 .unwrap()
17679 .unwrap();
17680 assert_eq!(
17681 parse_error(&replies[0])["code"],
17682 error_codes::SCOPE_BIND_PROOF_MISMATCH
17683 );
17684 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
17685 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17686 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17687 }
17688
17689 #[tokio::test]
17690 async fn scope_bind_proof_reserved_openers_ignore_proof() {
17691 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17692 let mut rig = rig().await;
17693 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17694 .await;
17695 for opener in [OWNER, AFT] {
17696 for proof in [None, Some("malformed+proof=")] {
17697 let route = rig
17698 .bound(
17699 Some(opener),
17700 PLEXUS,
17701 Some(
17702 rig.selector("s", Some(1))
17703 .with_bind_proof(proof.map(str::to_string)),
17704 ),
17705 )
17706 .await;
17707 assert!(rig.live(&route));
17708 }
17709 }
17710 }
17711
17712 #[tokio::test]
17713 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
17714 use std::future::Future;
17715 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17716 const NEXT_HASH: &str =
17717 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17718 let mut rig = rig().await;
17719 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17720 .await;
17721 let (client, mut rx, frame) =
17722 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
17723 for _ in 0..64 {
17724 client.egress.try_send(route_bind_ack(1)).unwrap();
17725 }
17726 let handler = rig.handler.clone();
17727 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17728 std::future::poll_fn(|cx| {
17729 assert!(open.as_mut().poll(cx).is_pending());
17730 std::task::Poll::Ready(())
17731 })
17732 .await;
17733 rig.sync(vec![
17734 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
17735 ])
17736 .await;
17737 rx.try_recv().unwrap();
17738 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17739 let bind = std::future::poll_fn(|cx| {
17740 assert!(open.as_mut().poll(cx).is_pending());
17741 module_rx.poll_recv(cx)
17742 })
17743 .await
17744 .unwrap()
17745 .frame;
17746 handler
17747 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
17748 .await
17749 .unwrap();
17750 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17751 .await
17752 .unwrap()
17753 .unwrap();
17754 assert_eq!(replies.len(), 1);
17755 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
17758 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17759 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
17760 let route = rig
17761 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
17762 .await;
17763 assert!(rig.live(&route));
17764 }
17765
17766 #[tokio::test]
17767 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
17768 {
17769 let mut rig = rig_with_flow_support(false).await;
17770 rig.sync(vec![session(1)]).await;
17771 let route = rig
17772 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17773 .await;
17774 assert!(rig.live(&route));
17775 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
17776 let mut record = session(1);
17777 record.attributes.flow_id = Some("flow:7".to_string());
17778 rig.sync(vec![record]).await;
17779 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
17780 assert!(rig.live(&unscoped));
17781 assert_eq!(unscoped.stamp(), None);
17782 }
17783
17784 #[tokio::test]
17785 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
17786 let mut rig = rig().await;
17787 let mut record = session(1);
17788 record.attributes.flow_id = Some("flow:7".to_string());
17789 rig.sync(vec![record.clone()]).await;
17790 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17791 let mut owner_route = rig
17792 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17793 .await;
17794 let mut carrier_route = rig
17795 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
17796 .await;
17797 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17798 rig.sync(vec![record.clone()]).await;
17799 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17800 assert!(rig.live(&owner_route) && owner_route.untouched());
17801 assert!(rig.live(&carrier_route) && carrier_route.untouched());
17802
17803 record.attributes.flow_id = Some("flow:8".to_string());
17804 rig.sync(vec![record]).await;
17805 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
17806 let before = before.unwrap();
17807 assert_eq!(after.scope_epoch, before.scope_epoch);
17808 assert!(after.version > before.version);
17809 for route in [&mut owner_route, &mut carrier_route] {
17810 assert!(!rig.live(route));
17811 assert_eq!(
17812 route.closed_reason(),
17813 RouteCloseReason::ScopeDelegationChanged
17814 );
17815 }
17816 assert!(rig.live(&unscoped) && unscoped.untouched());
17817 for target in [PLEXUS, OTHER] {
17820 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
17821 let goodbye = module_rx
17822 .try_recv()
17823 .expect("the provider sees the drain")
17824 .frame;
17825 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17826 assert!(module_rx.try_recv().is_err());
17827 }
17828 let rebound = rig
17829 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17830 .await;
17831 assert_eq!(
17832 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
17833 Some("flow:8")
17834 );
17835 }
17836
17837 #[tokio::test]
17842 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
17843 let mut rig = rig().await;
17844 rig.sync(vec![session(1)]).await;
17845 let mut cotenant = rig
17846 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17847 .await;
17848
17849 let mut changed = session(1);
17850 changed.child_owners.push(Principal::Reserved {
17851 module_id: MAGIC.to_string(),
17852 });
17853 let mut ended = None;
17854 for (code, next) in [
17855 (error_codes::SCOPE_CHANGED, vec![changed]),
17856 (error_codes::SCOPE_ENDED, Vec::new()),
17857 ] {
17858 let relayed = rig
17859 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17860 .await;
17861 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
17862 ended = Some(next.is_empty());
17863 rig.sync(next).await;
17864 rig.ack(&relayed).await;
17865 let replies = relayed.task.await.unwrap();
17866 assert_eq!(replies.len(), 1, "{replies:?}");
17867 assert_eq!(parse_error(&replies[0])["code"], code);
17868 assert_eq!(
17869 subc_protocol::error_codes::is_retryable_route_open(code),
17870 code == error_codes::SCOPE_CHANGED
17871 );
17872 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17873 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17875 let mut goodbyes = Vec::new();
17878 while let Ok(outbound) = plexus_rx.try_recv() {
17879 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
17880 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
17881 }
17882 assert!(
17883 goodbyes.contains(&(bind_channel, bind_epoch)),
17884 "{goodbyes:?}"
17885 );
17886 assert!(rig
17887 .handler
17888 .registry
17889 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
17890 .unwrap()
17891 .is_some());
17892 }
17893 assert_eq!(ended, Some(true));
17894 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
17897 }
17898
17899 #[tokio::test]
17902 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
17903 struct Case {
17904 name: &'static str,
17905 change: fn(&mut ScopeRecord),
17906 closed: [Option<RouteCloseReason>; 4],
17909 }
17910 use RouteCloseReason::*;
17911 let cases = [
17912 Case {
17913 name: "a carrier entry removed",
17914 change: |r| {
17915 r.carriers.retain(|c| {
17916 c.principal
17917 != Principal::Reserved {
17918 module_id: AFT.to_string(),
17919 }
17920 })
17921 },
17922 closed: [None, Some(ScopeCarrierRemoved), None, None],
17923 },
17924 Case {
17925 name: "a target removed from a carrier",
17926 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
17927 closed: [None, None, None, Some(ScopeCarrierRemoved)],
17928 },
17929 Case {
17930 name: "a bare carrier narrowed to targets",
17931 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
17932 closed: [None, Some(ScopeCarrierRemoved), None, None],
17933 },
17934 Case {
17935 name: "delegates turned off",
17936 change: |r| r.attributes.delegates = false,
17937 closed: [Some(ScopeDelegationChanged); 4],
17938 },
17939 Case {
17940 name: "agent_id changed",
17941 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
17942 closed: [Some(ScopeDelegationChanged); 4],
17943 },
17944 Case {
17945 name: "a carrier added, child owners changed, the record re-sent",
17946 change: |r| {
17947 r.carriers.push(carrier(MAGIC, None));
17948 r.child_owners.push(Principal::Reserved {
17949 module_id: MAGIC.to_string(),
17950 });
17951 },
17952 closed: [None; 4],
17953 },
17954 Case {
17955 name: "a target added",
17956 change: |r| {
17957 r.carriers[1]
17958 .targets
17959 .as_mut()
17960 .unwrap()
17961 .push("third".to_string())
17962 },
17963 closed: [None; 4],
17964 },
17965 Case {
17966 name: "delegates turned on",
17967 change: |r| r.attributes.delegates = true,
17968 closed: [None; 4],
17969 },
17970 ];
17971 for case in cases {
17972 let mut rig = rig().await;
17973 rig.sync(vec![session(1)]).await;
17974 let scope = || Some(rig_selector("s", Some(1)));
17975 let mut routes = [
17976 rig.bound(Some(OWNER), PLEXUS, scope()).await,
17977 rig.bound(Some(AFT), PLEXUS, scope()).await,
17978 rig.bound(Some(BROCA), PLEXUS, scope()).await,
17979 rig.bound(Some(BROCA), OTHER, scope()).await,
17980 ];
17981 let mut record = session(1);
17982 (case.change)(&mut record);
17983 rig.sync(vec![record]).await;
17984 for (index, expected) in case.closed.iter().enumerate() {
17985 let route = &mut routes[index];
17986 match expected {
17987 Some(reason) => {
17988 assert!(!rig.live(route), "{}: route {index} still live", case.name);
17989 assert_eq!(
17990 route.closed_reason(),
17991 *reason,
17992 "{}: route {index}",
17993 case.name
17994 );
17995 }
17996 None => {
17997 assert!(rig.live(route), "{}: route {index} closed", case.name);
17998 assert!(
17999 route.untouched(),
18000 "{}: route {index} was told something",
18001 case.name
18002 );
18003 }
18004 }
18005 }
18006 }
18007 }
18008
18009 #[tokio::test]
18010 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
18011 for next in [Vec::new(), vec![session(2)]] {
18013 let mut rig = rig().await;
18014 rig.sync(vec![session(1)]).await;
18015 let mut route = rig
18016 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18017 .await;
18018 rig.sync(next).await;
18019 assert!(!rig.live(&route));
18020 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
18021 }
18022
18023 let mut rig = rig().await;
18026 let mut child = session(1);
18027 child.scope_ref = "child".to_string();
18028 child.kind = ScopeKind::Worker;
18029 child.parent = Some(ScopeParent::new(
18030 Principal::Reserved {
18031 module_id: OWNER.to_string(),
18032 },
18033 "s".to_string(),
18034 1,
18035 ));
18036 rig.sync(vec![session(1), child.clone()]).await;
18037 let mut child_route = rig
18038 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
18039 .await;
18040 assert_eq!(
18041 child_route.stamp().unwrap().parent_state,
18042 Some(ParentState::Linked)
18043 );
18044 rig.sync(vec![child]).await;
18045 assert!(!rig.live(&child_route));
18046 assert_eq!(
18047 child_route.closed_reason(),
18048 RouteCloseReason::ScopeParentEnded
18049 );
18050 }
18051
18052 #[tokio::test]
18053 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
18054 ) {
18055 let mut rig = rig().await;
18056 rig.sync(vec![session(1)]).await;
18057 let mut route = rig
18058 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18059 .await;
18060 let before = rig.forwarding.published_scope_tag(OWNER, "s");
18061 rig.sync(vec![session(1)]).await;
18062 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18063 assert!(rig.live(&route) && route.untouched());
18064
18065 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
18069 .forwarding
18070 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18071 .unwrap()
18072 else {
18073 panic!("the route is bound");
18074 };
18075 binding.flow.acquire_tagged(9, false).await.unwrap();
18076 let mut widened = session(1);
18077 widened.carriers.push(carrier(MAGIC, None));
18078 rig.sync(vec![widened]).await;
18079 assert!(rig.live(&route) && route.untouched());
18080 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18081 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
18082 assert_eq!(binding.flow.in_flight(), 1);
18085 binding
18086 .flow
18087 .acquire_tagged(10, false)
18088 .await
18089 .expect("the flow is still open");
18090 }
18091
18092 #[tokio::test]
18095 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
18096 let mut rig = rig().await;
18097 rig.sync(vec![session(1)]).await;
18098 let mut on_incumbent = rig
18099 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18100 .await;
18101
18102 let (candidate, _candidate_rx) = wide_ctx(9);
18105 let registration = rig
18106 .handler
18107 .registry
18108 .register_candidate_with_control_ops(
18109 manifest(PLEXUS, PROTOCOL_VERSION),
18110 PROTOCOL_VERSION,
18111 candidate.connection_id,
18112 module_baseline_control_ops(),
18113 )
18114 .unwrap();
18115 rig.forwarding
18116 .register_candidate_module_connection(
18117 candidate.connection_id,
18118 PLEXUS.to_string(),
18119 PROTOCOL_VERSION,
18120 manifest_concurrency(®istration.manifest),
18121 candidate.egress.clone(),
18122 )
18123 .unwrap();
18124 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
18125 rig.handler
18126 .registry
18127 .promote_candidate(PLEXUS)
18128 .unwrap()
18129 .unwrap();
18130 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
18131
18132 rig.sync(Vec::new()).await;
18133 assert!(!rig.live(&on_incumbent));
18134 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
18135 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18136 let goodbye = incumbent_rx
18137 .try_recv()
18138 .expect("the superseded endpoint is told")
18139 .frame;
18140 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18141 }
18142 }
18143}
18144
18145#[cfg(test)]
18146mod concurrency_default_exposure_tests {
18147 use super::*;
18148
18149 fn hello_body(role_json: &str) -> Vec<u8> {
18150 format!(
18151 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
18152 )
18153 .into_bytes()
18154 }
18155
18156 fn manifest_from(body: &[u8]) -> ModuleManifest {
18157 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
18158 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
18159 .expect("manifest parses")
18160 }
18161
18162 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
18163
18164 #[test]
18165 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
18166 let body = hello_body(&format!(
18167 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
18168 ));
18169 let manifest = manifest_from(&body);
18170 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18173 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
18174 }
18175
18176 #[test]
18177 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
18178 let body = hello_body(&format!(
18179 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
18180 ));
18181 let manifest = manifest_from(&body);
18182 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18183 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
18184 }
18185
18186 #[test]
18187 fn non_management_roles_are_never_reported() {
18188 let body = hello_body(
18189 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
18190 );
18191 let manifest = manifest_from(&body);
18192 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
18193 }
18194}