1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_confirm_operations,
25 validate_hello_event_declarations, validate_hello_flow_operations,
26 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
27 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
28 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
29 },
30 peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
31 scope::{
32 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
33 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
34 SCOPE_SYNC_OP,
35 },
36 session::{
37 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
38 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
39 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
40 ROLE_VERSIONS_FIELD,
41 },
42 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
43 Priority, RouteTarget, PROTOCOL_VERSION,
44};
45use tokio::time::{timeout_at, Instant};
46use tracing::{debug, info, instrument::WithSubscriber, warn};
47
48use crate::{
49 caller_identity::CallerIdentity,
50 capability_requirements::{
51 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
52 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
53 RequirementStatus, RuntimeModule,
54 },
55 child_roster::DaemonShutdownFlag,
56 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
57 forwarding::{
58 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
59 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
60 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
61 },
62 observability::{
63 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
64 },
65 privileged_prompt::{prompt_text, PrivilegedOperation},
66 provenance::{
67 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
68 },
69 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
70 router::{daemon_error_flags, RouteCtx, RouterError},
71 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
72 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
73 stderr_tail::{CaptureState, TailEntry},
74 supervise::{
75 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
76 SupervisorHandle, SwapHelloAdmission,
77 },
78 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
79};
80
81pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
87
88const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
89const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
90const CAP_PING_PONG: &str = "ping_pong_v1";
91const CAP_SESSION_ATTACH: &str = "session_attach_v1";
92const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
93
94const SUBC_CONTROL_OPS: &[&str] = &[
95 ops::DAEMON_SHUTDOWN_INTENT,
96 ops::ACTIVITY_RECORD,
97 ops::ACTIVITY_QUERY,
98 ops::ACTIVITY_SUBSCRIBE,
99 ops::SERVER_DESCRIBE,
100 ops::CATALOG_LIST,
101 ops::ROUTE_OPEN,
102 ops::ROUTE_POLL,
103 ops::ROUTE_CLOSING,
104 ops::ROUTE_CLOSED,
105 ops::SUPERVISOR_LIST,
106 ops::SUPERVISOR_RESTART,
107 ops::SUPERVISOR_SWAP,
108 ops::SUPERVISOR_RELOAD,
109 ops::SUPERVISOR_RESCAN,
110 ops::SUPERVISOR_RELEASE_RESERVED,
111 ops::SUPERVISOR_SET_ENABLED,
112 ops::SUPERVISOR_HEALTH_PROBE,
113 ops::SUPERVISOR_HEALTH,
114 ops::SUPERVISOR_STDERR_TAIL,
115 ops::SUPERVISOR_TERMINALS,
116 ops::SUPERVISOR_ROUTES,
117 ops::SUPERVISOR_PROVENANCE,
118 ops::SUPERVISOR_SPAWN_SNAPSHOT,
119 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
120];
121
122const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
129
130mod rescan_checked;
131use rescan_checked::{RescanMode, RescanPlan};
132
133const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
134 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
135 "supervisor.live_roots",
136 SCOPE_SYNC_OP,
137 SCOPE_APPLY_OP,
138 SCOPE_DESCRIBE_OP,
139 "operator.confirm",
140];
141
142const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
146
147const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
148
149pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
157
158pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
173
174pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
192
193const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
194const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
195
196fn reload_verdict(
197 configured: &Path,
198 spawned_from: Option<&Path>,
199 image: subc_control::RunningImageAgreement,
200) -> PendingReloadVerdict {
201 let path = match spawned_from {
202 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
203 ReloadPathAgreement::Match
204 }
205 Some(spawned_from) => ReloadPathAgreement::Mismatch {
206 configured: configured.to_path_buf(),
207 spawned_from: spawned_from.to_path_buf(),
208 },
209 None => ReloadPathAgreement::Unavailable {
210 reason: if matches!(
211 image,
212 subc_control::RunningImageAgreement::Unavailable {
213 reason: subc_control::RunningImageUnavailableReason::NotRunning
214 }
215 ) {
216 ReloadPathUnavailableReason::NotRunning
217 } else {
218 ReloadPathUnavailableReason::SpawnedPathUnavailable
219 },
220 },
221 };
222 PendingReloadVerdict {
223 path,
224 image,
225 spec: None,
226 }
227}
228
229fn launch_spec_agreement(
230 configured: &crate::supervise::ModuleSpec,
231 running: bool,
232 spawned: Option<&crate::supervise::LaunchSpecDigest>,
233) -> subc_control::LaunchSpecAgreement {
234 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
235 if !running {
236 return Agreement::Unavailable {
237 reason: Reason::NotRunning,
238 };
239 }
240 match spawned {
241 None => Agreement::Unavailable {
242 reason: Reason::SpawnedSpecUnavailable,
243 },
244 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
245 Agreement::Match
246 }
247 Some(_) => Agreement::Mismatch,
248 }
249}
250
251fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
252 if configured == spawned_from {
253 return true;
254 }
255 #[cfg(windows)]
256 {
257 match (
263 std::fs::canonicalize(configured),
264 std::fs::canonicalize(spawned_from),
265 ) {
266 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
267 _ => false,
268 }
269 }
270 #[cfg(not(windows))]
271 {
272 false
273 }
274}
275
276#[derive(Clone)]
277struct DaemonProvenanceFacts {
278 build: DaemonBuildProvenance,
279 pid: Option<u32>,
280 started_at_ms: Option<u64>,
281 start_clock: Option<crate::clock::StartClock>,
282 executable_path: Option<PathBuf>,
283 executable_identity: Option<SpawnedFileIdentity>,
284 process_start_time: Option<u64>,
285 probe: ExecutableIdentityProbe,
286}
287
288impl Default for DaemonProvenanceFacts {
289 fn default() -> Self {
290 Self {
291 build: DaemonBuildProvenance {
292 build_git_sha: None,
293 build_lock_digest: None,
294 },
295 pid: None,
296 started_at_ms: None,
297 start_clock: None,
298 executable_path: None,
299 executable_identity: None,
300 process_start_time: None,
301 probe: ExecutableIdentityProbe::default(),
302 }
303 }
304}
305
306#[derive(Debug, Clone)]
307struct SupervisorRescanContext {
308 supervisor: Supervisor,
309 config_path: PathBuf,
310 configured_port: Option<u16>,
311 storage_config: Option<crate::daemon_config::StorageConfig>,
312 admission_facts_carrier_module_id: Option<String>,
313 admission_facts_targets: Option<Vec<String>>,
314 scope_authority_owners: Vec<String>,
315 privileged_operator_prompt: PrivilegedOperatorPrompt,
316 generation: Arc<std::sync::atomic::AtomicU64>,
322}
323
324const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
345 "reloading",
346 "supervisor_not_live",
347 "registration_not_active",
348 "no_forwarding_connection",
349 "relay_send_failed",
350];
351
352#[derive(Clone)]
354pub struct ControlHandler {
355 registry: Arc<Registry>,
356 activity: Option<crate::activity::ActivityLedger>,
357 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
358 forwarding: Arc<ForwardingTable>,
359 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
360 supervisor: SupervisorHandle,
361 subc_capabilities: Arc<[String]>,
362 route_bind_relay_timeout: Duration,
366 route_bind_relay_timeouts: BTreeMap<String, Duration>,
370 route_bind_breakers: RouteBindBreakers,
373 route_bind_concurrency: RouteBindConcurrency,
376 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
381 route_bind_breaker_threshold: u32,
383 route_bind_breaker_cooldown: Duration,
385 health_probe_timeout: Duration,
386 storage_config: Option<crate::daemon_config::StorageConfig>,
389 machine_id: Option<crate::machine_id::MachineId>,
393 admission_facts_carrier_module_id: Option<String>,
394 admission_facts_targets: Option<Vec<String>>,
395 scopes: Arc<RwLock<ScopeTable>>,
399 scope_deadline_added: Arc<tokio::sync::Notify>,
403 scope_authority_owners: Vec<String>,
406 privileged_operator_prompt: PrivilegedOperatorPrompt,
408 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
411 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
412 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
413 local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
414 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
415 pub(crate) operator_apps:
416 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
417 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
422 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
428 #[cfg(test)]
430 pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
431 #[cfg(test)]
432 pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
433 #[allow(dead_code)]
437 #[cfg(windows)]
438 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
439 #[allow(dead_code)]
440 #[cfg(windows)]
441 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
442 #[allow(dead_code)]
446 #[cfg(windows)]
447 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
448 #[cfg(all(test, windows))]
451 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
452 #[cfg(test)]
453 caller_lookup: Option<TestCallerLookup>,
454 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
457 rescan: Option<SupervisorRescanContext>,
458 connected_clients: ConnectedClients,
459 counters: DaemonCounters,
460 local_endpoint_unavailable: Option<String>,
463 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
464 daemon_shutdown: DaemonShutdownFlag,
469 daemon_provenance: DaemonProvenanceFacts,
470 #[cfg(test)]
471 control_dispatch_delay: Option<Duration>,
472 #[cfg(test)]
473 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
474 #[cfg(test)]
478 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
479 #[cfg(test)]
485 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
486}
487
488impl fmt::Debug for ControlHandler {
489 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
490 f.debug_struct("ControlHandler")
491 .field("registry", &self.registry)
492 .field("forwarding", &self.forwarding)
493 .field("process_liveness", &self.process_liveness.is_some())
494 .field("supervisor", &self.supervisor)
495 .field("subc_capabilities", &self.subc_capabilities)
496 .finish()
497 }
498}
499
500struct RouteOpenRequest {
501 target: RouteTarget,
502 identity: BindIdentity,
503 consumer_identity: Option<ConsumerIdentity>,
504 consumer_capabilities: Option<Vec<String>>,
505 role_versions: Option<BTreeMap<String, String>>,
506 busy_refusals: Option<bool>,
507 admission_facts: Option<serde_json::Value>,
508 scope: Option<ScopeSelector>,
509}
510
511#[derive(Clone, Copy)]
512enum RouteOpenPhase {
513 Admission,
514 ProjectRoot,
515 EgressReservation,
516 BindSend,
517 BindAckWait,
518}
519
520struct RouteOpenTimings {
524 module_id: String,
525 principal: String,
526 admission: Duration,
527 project_root: Duration,
528 egress_reservation: Duration,
529 bind_send: Duration,
530 bind_ack_wait: Duration,
531 phase: Option<RouteOpenPhase>,
532 phase_started_at: Instant,
533}
534
535impl RouteOpenTimings {
536 fn new() -> Self {
537 Self {
538 module_id: String::new(),
539 principal: "unresolved".to_string(),
541 admission: Duration::ZERO,
542 project_root: Duration::ZERO,
543 egress_reservation: Duration::ZERO,
544 bind_send: Duration::ZERO,
545 bind_ack_wait: Duration::ZERO,
546 phase: Some(RouteOpenPhase::Admission),
547 phase_started_at: Instant::now(),
548 }
549 }
550
551 fn finish(&mut self) {
552 let elapsed = self.phase_started_at.elapsed();
553 match self.phase.take() {
554 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
555 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
556 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
557 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
558 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
559 None => {}
560 }
561 }
562
563 fn enter(&mut self, phase: RouteOpenPhase) {
564 self.finish();
565 self.phase = Some(phase);
566 self.phase_started_at = Instant::now();
567 }
568}
569
570struct RouteBindReservationGuard {
571 forwarding: Arc<ForwardingTable>,
572 endpoint: ModuleEndpointId,
573 relay_corr: u64,
574 armed: bool,
575}
576
577struct ModuleControlRpcGuard {
578 forwarding: Arc<ForwardingTable>,
579 endpoint: ModuleEndpointId,
580 corr: u64,
581 armed: bool,
582}
583
584impl ModuleControlRpcGuard {
585 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
586 Self {
587 forwarding,
588 endpoint,
589 corr,
590 armed: true,
591 }
592 }
593
594 fn disarm(&mut self) {
595 self.armed = false;
596 }
597}
598
599impl Drop for ModuleControlRpcGuard {
600 fn drop(&mut self) {
601 if self.armed {
602 let _ = self
603 .forwarding
604 .cancel_module_control_rpc(self.endpoint, self.corr);
605 }
606 }
607}
608
609impl RouteBindReservationGuard {
610 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
611 Self {
612 forwarding,
613 endpoint,
614 relay_corr,
615 armed: true,
616 }
617 }
618
619 fn release_and_disarm(&mut self) {
620 if !self.armed {
621 return;
622 }
623 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
624 self.endpoint,
625 self.relay_corr,
626 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
627 ) {
628 send_goodbye_target_best_effort(
629 &self.forwarding.counters(),
630 &target,
631 "canceled route.bind",
632 );
633 }
634 self.armed = false;
635 }
636
637 fn disarm(&mut self) {
638 self.armed = false;
639 }
640}
641
642impl Drop for RouteBindReservationGuard {
643 fn drop(&mut self) {
644 self.release_and_disarm();
645 }
646}
647
648#[derive(Debug, Clone, Default)]
678pub(crate) struct RouteBindBreakers {
679 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
680}
681
682#[derive(Debug, Clone, Default)]
683pub(crate) struct RouteBindConcurrency {
684 modules: Arc<Mutex<HashMap<String, usize>>>,
685}
686
687struct RouteBindConcurrencyGuard {
688 concurrency: RouteBindConcurrency,
689 module_id: String,
690}
691
692impl RouteBindConcurrency {
693 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
696 let mut modules = self
697 .modules
698 .lock()
699 .expect("route.bind concurrency mutex poisoned");
700 let in_flight = modules.entry(module_id.to_string()).or_default();
701 if *in_flight >= limit {
702 return Err(*in_flight);
703 }
704 *in_flight += 1;
705 Ok(RouteBindConcurrencyGuard {
706 concurrency: self.clone(),
707 module_id: module_id.to_string(),
708 })
709 }
710}
711
712impl Drop for RouteBindConcurrencyGuard {
713 fn drop(&mut self) {
714 let mut modules = self
715 .concurrency
716 .modules
717 .lock()
718 .expect("route.bind concurrency mutex poisoned");
719 let remove = {
720 let in_flight = modules
721 .get_mut(&self.module_id)
722 .expect("admitted route.bind has a concurrency entry");
723 *in_flight -= 1;
724 *in_flight == 0
725 };
726 if remove {
727 modules.remove(&self.module_id);
728 }
729 }
730}
731
732#[derive(Debug, Default)]
733struct ModuleBreakerState {
734 consecutive_timeouts: u32,
736 cooldown_until: Option<Instant>,
739 probe_in_flight: Option<Arc<()>>,
744}
745
746enum RouteBindAdmission<'a> {
748 Admitted {
749 guard: RouteBindBreakerGuard<'a>,
750 probe: bool,
753 },
754 Refused {
755 consecutive_timeouts: u32,
756 retry_in: Duration,
760 probe_in_flight: bool,
761 },
762}
763
764struct RouteBindBreakerGuard<'a> {
771 breakers: RouteBindBreakers,
772 module_id: &'a str,
773 probe_token: Option<Arc<()>>,
774 settled: bool,
775}
776
777impl RouteBindBreakerGuard<'_> {
778 fn record_accepted(&mut self) -> bool {
782 self.settled = true;
783 self.breakers.record_accepted(self.module_id)
784 }
785
786 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
789 self.settled = true;
790 self.breakers.record_timeout(
791 self.module_id,
792 self.probe_token.as_ref(),
793 threshold,
794 cooldown,
795 )
796 }
797
798 fn record_inconclusive(&mut self) {
807 self.settled = true;
808 self.breakers
809 .record_inconclusive(self.module_id, self.probe_token.as_ref());
810 }
811}
812
813impl Drop for RouteBindBreakerGuard<'_> {
814 fn drop(&mut self) {
815 if !self.settled {
816 self.breakers
817 .record_inconclusive(self.module_id, self.probe_token.as_ref());
818 }
819 }
820}
821
822struct BreakerOpened {
826 consecutive_timeouts: u32,
827 reopened_after_probe: bool,
830}
831
832impl RouteBindBreakers {
833 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
834 self.modules
835 .lock()
836 .expect("route.bind breaker mutex poisoned")
837 }
838
839 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
842 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
843 probe: probe_token.is_some(),
844 guard: RouteBindBreakerGuard {
845 breakers: self.clone(),
846 module_id,
847 probe_token,
848 settled: false,
849 },
850 };
851
852 let mut modules = self.lock();
853 let Some(state) = modules.get_mut(module_id) else {
854 return admitted(None);
855 };
856 let Some(cooldown_until) = state.cooldown_until else {
857 return admitted(None);
858 };
859 if state.probe_in_flight.is_some() {
860 return RouteBindAdmission::Refused {
861 consecutive_timeouts: state.consecutive_timeouts,
862 retry_in: Duration::ZERO,
863 probe_in_flight: true,
864 };
865 }
866 let now = Instant::now();
867 if now < cooldown_until {
868 return RouteBindAdmission::Refused {
869 consecutive_timeouts: state.consecutive_timeouts,
870 retry_in: cooldown_until - now,
871 probe_in_flight: false,
872 };
873 }
874 let token = Arc::new(());
875 state.probe_in_flight = Some(Arc::clone(&token));
876 admitted(Some(token))
877 }
878
879 fn record_accepted(&self, module_id: &str) -> bool {
880 self.lock()
881 .remove(module_id)
882 .is_some_and(|state| state.cooldown_until.is_some())
883 }
884
885 fn record_timeout(
886 &self,
887 module_id: &str,
888 probe_token: Option<&Arc<()>>,
889 threshold: u32,
890 cooldown: Duration,
891 ) -> Option<BreakerOpened> {
892 let mut modules = self.lock();
893 let state = modules.entry(module_id.to_string()).or_default();
894 let was_open = state.cooldown_until.is_some();
895 let was_probe = Self::owns_probe(state, probe_token);
896 if was_probe {
897 state.probe_in_flight = None;
898 }
899 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
900 if state.consecutive_timeouts < threshold {
901 return None;
902 }
903 state.cooldown_until = Some(Instant::now() + cooldown);
904 Some(BreakerOpened {
905 consecutive_timeouts: state.consecutive_timeouts,
906 reopened_after_probe: was_open && was_probe,
907 })
908 }
909
910 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
911 state
915 .probe_in_flight
916 .as_ref()
917 .zip(token)
918 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
919 }
920
921 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
922 if let Some(state) = self.lock().get_mut(module_id) {
923 if Self::owns_probe(state, probe_token) {
924 state.probe_in_flight = None;
925 }
926 }
927 }
928
929 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
944 self.lock()
945 .remove(module_id)
946 .map(|state| state.consecutive_timeouts)
947 .filter(|discarded| *discarded > 0)
948 }
949
950 fn open_snapshot(&self) -> Option<serde_json::Value> {
957 let now = Instant::now();
958 let modules = self.lock();
959 let open = modules
960 .iter()
961 .filter_map(|(module_id, state)| {
962 let cooldown_until = state.cooldown_until?;
963 Some((
964 module_id.clone(),
965 serde_json::json!({
966 "consecutive_timeouts": state.consecutive_timeouts,
967 "cooldown_remaining_ms":
968 cooldown_until.saturating_duration_since(now).as_millis() as u64,
969 "probe_in_flight": state.probe_in_flight.is_some(),
970 }),
971 ))
972 })
973 .collect::<serde_json::Map<String, serde_json::Value>>();
974 (!open.is_empty()).then_some(serde_json::Value::Object(open))
975 }
976}
977
978impl ControlHandler {
979 pub fn new(registry: Arc<Registry>) -> Self {
980 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
981 }
982
983 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
984 let counters = forwarding.counters();
985 let route_bind_breakers = forwarding.route_bind_breakers();
989 let route_bind_concurrency = forwarding.route_bind_concurrency();
990 let route_outages = forwarding.route_outages();
991 Self {
992 registry,
993 activity: None,
994 activity_streams: Arc::new(Mutex::new(HashMap::new())),
995 forwarding,
996 process_liveness: None,
997 supervisor: SupervisorHandle::new(),
998 subc_capabilities: Arc::from([
999 CAP_MANIFEST_REGISTRATION.to_string(),
1000 CAP_CHANNEL_LIFECYCLE.to_string(),
1001 CAP_PING_PONG.to_string(),
1002 CAP_SESSION_ATTACH.to_string(),
1003 CAP_ADMISSION_FACTS_RELAY.to_string(),
1004 CAP_SCOPES_V1.to_string(),
1005 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1006 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1007 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1008 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1009 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1010 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1011 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1012 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1013 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1014 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1015 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1016 ]),
1017 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1018 route_bind_relay_timeouts: BTreeMap::new(),
1019 route_bind_breakers,
1020 route_bind_concurrency,
1021 route_outages,
1022 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1023 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1024 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1025 storage_config: None,
1026 machine_id: None,
1027 admission_facts_carrier_module_id: None,
1028 admission_facts_targets: None,
1029 scopes: Arc::new(RwLock::new(ScopeTable::new(
1030 crate::daemon_config::default_scope_authority_owners(),
1031 ))),
1032 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1033 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1034 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1035 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1036 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1037 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1038 local_connections: Arc::new(Mutex::new(HashSet::new())),
1039 operator_app_policy: Default::default(),
1040 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1041 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1042 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1043 #[cfg(test)]
1044 app_open_test_platform: None,
1045 #[cfg(test)]
1046 app_open_test_launch: None,
1047 #[cfg(windows)]
1048 windows_verify: None,
1049 #[cfg(windows)]
1050 windows_flash: None,
1051 #[cfg(windows)]
1052 windows_track: None,
1053 #[cfg(all(test, windows))]
1054 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1055 #[cfg(test)]
1056 caller_lookup: None,
1057 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1058 rescan: None,
1059 connected_clients: ConnectedClients::new(),
1060 local_endpoint_unavailable: None,
1061 counters,
1062 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1063 daemon_shutdown: DaemonShutdownFlag::default(),
1064 daemon_provenance: DaemonProvenanceFacts::default(),
1065 #[cfg(test)]
1066 control_dispatch_delay: None,
1067 #[cfg(test)]
1068 provenance_probe_override: None,
1069 #[cfg(test)]
1070 route_bind_relay_sent_hold: None,
1071 #[cfg(test)]
1072 hello_registry_to_forwarding_hold: None,
1073 }
1074 }
1075
1076 pub fn with_storage_config(
1079 mut self,
1080 storage_config: Option<crate::daemon_config::StorageConfig>,
1081 ) -> Self {
1082 self.storage_config = storage_config;
1083 self
1084 }
1085
1086 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1089 self.activity = Some(crate::activity::ActivityLedger::open(
1090 run_dir.join("activity.jsonl"),
1091 incarnation,
1092 ));
1093 self.supervisor.set_activity(self.activity.clone());
1094 self
1095 }
1096
1097 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1100 self.machine_id = machine_id;
1101 self
1102 }
1103
1104 pub fn with_admission_facts_config(
1108 mut self,
1109 carrier_module_id: Option<String>,
1110 targets: Option<Vec<String>>,
1111 ) -> Self {
1112 self.admission_facts_carrier_module_id = carrier_module_id;
1113 self.admission_facts_targets = targets;
1114 self
1115 }
1116
1117 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1121 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1122 self.scope_authority_owners = owners;
1123 self
1124 }
1125
1126 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1128 self.privileged_operator_prompt = mode;
1129 self
1130 }
1131
1132 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1135 self.route_bind_relay_timeout = timeout;
1136 self
1137 }
1138
1139 pub fn with_route_bind_relay_timeouts(
1145 mut self,
1146 timeouts: impl IntoIterator<Item = (String, Duration)>,
1147 ) -> Self {
1148 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1149 self
1150 }
1151
1152 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1158 self.route_bind_relay_timeouts
1159 .get(module_id)
1160 .copied()
1161 .unwrap_or(self.route_bind_relay_timeout)
1162 }
1163
1164 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1172 self.route_bind_breaker_threshold = threshold.max(1);
1173 self.route_bind_breaker_cooldown = cooldown;
1174 self
1175 }
1176
1177 #[cfg(test)]
1178 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1179 self.health_probe_timeout = timeout;
1180 self
1181 }
1182
1183 #[cfg(test)]
1184 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1185 self.control_dispatch_delay = Some(delay);
1186 self
1187 }
1188
1189 #[cfg(test)]
1193 pub(crate) fn with_route_bind_relay_sent_hold(
1194 mut self,
1195 hold: Arc<tokio::sync::Barrier>,
1196 ) -> Self {
1197 self.route_bind_relay_sent_hold = Some(hold);
1198 self
1199 }
1200
1201 #[cfg(test)]
1206 pub(crate) fn with_hello_registry_to_forwarding_hold(
1207 mut self,
1208 hold: Arc<std::sync::Barrier>,
1209 ) -> Self {
1210 self.hello_registry_to_forwarding_hold = Some(hold);
1211 self
1212 }
1213
1214 #[cfg(test)]
1215 fn hold_between_registry_and_forwarding(&self) {
1216 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1217 hold.wait();
1218 hold.wait();
1219 }
1220 }
1221
1222 pub fn with_process_liveness(
1223 mut self,
1224 process_liveness: Arc<dyn ModuleProcessLiveness>,
1225 ) -> Self {
1226 self.process_liveness = Some(process_liveness);
1227 self
1228 }
1229
1230 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1231 self.supervisor = supervisor;
1232 self.supervisor.set_activity(self.activity.clone());
1233 self
1234 }
1235
1236 pub fn with_daemon_provenance(
1237 mut self,
1238 pid: u32,
1239 started_at_ms: u64,
1240 executable_path: Option<PathBuf>,
1241 build_git_sha: Option<String>,
1242 build_lock_digest: Option<String>,
1243 ) -> Self {
1244 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1245 let process_start_time = process_start_time(pid);
1246 self.daemon_provenance = DaemonProvenanceFacts {
1247 build: DaemonBuildProvenance {
1248 build_git_sha,
1249 build_lock_digest,
1250 },
1251 pid: Some(pid),
1252 started_at_ms: Some(started_at_ms),
1253 start_clock: None,
1254 executable_path,
1255 executable_identity,
1256 process_start_time,
1257 probe: ExecutableIdentityProbe::default(),
1258 };
1259 self
1260 }
1261
1262 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1263 self.daemon_provenance.start_clock = Some(clock);
1264 self
1265 }
1266
1267 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1269 self.scopes
1270 .write()
1271 .unwrap_or_else(|poisoned| poisoned.into_inner())
1272 .set_wall_clock(Arc::new(clock));
1273 self
1274 }
1275
1276 #[cfg(test)]
1277 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1278 self.provenance_probe_override = Some(result);
1279 self
1280 }
1281
1282 pub fn with_capability_config(
1286 self,
1287 modules: impl IntoIterator<Item = (String, bool)>,
1288 reserved_capabilities: BTreeMap<String, String>,
1289 ) -> Self {
1290 self.capability_evaluator
1291 .configure(modules, reserved_capabilities);
1292 self
1293 }
1294
1295 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1299 self.daemon_shutdown = flag;
1300 self
1301 }
1302
1303 pub fn with_supervisor_rescan(
1304 mut self,
1305 supervisor: Supervisor,
1306 config_path: impl Into<PathBuf>,
1307 configured_port: Option<u16>,
1308 ) -> Self {
1309 self.rescan = Some(SupervisorRescanContext {
1310 supervisor,
1311 config_path: config_path.into(),
1312 configured_port,
1313 storage_config: self.storage_config.clone(),
1314 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1315 admission_facts_targets: self.admission_facts_targets.clone(),
1316 scope_authority_owners: self.scope_authority_owners.clone(),
1317 privileged_operator_prompt: self.privileged_operator_prompt,
1318 generation: Arc::default(),
1319 });
1320 self
1321 }
1322
1323 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1324 self.connected_clients = connected_clients;
1325 self
1326 }
1327
1328 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1331 self.local_endpoint_unavailable = reason;
1332 self
1333 }
1334
1335 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1336 Arc::clone(&self.forwarding)
1337 }
1338
1339 pub(crate) fn counters(&self) -> DaemonCounters {
1340 self.counters.clone()
1341 }
1342
1343 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1346 tokio::spawn(async move {
1347 loop {
1348 self.capability_evaluator
1349 .wait_for_change_or_deadline()
1350 .await;
1351 self.refresh_capability_requirements();
1352 }
1353 });
1354 }
1355
1356 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1362 tokio::spawn(async move {
1363 loop {
1364 let has_deadlines = self
1365 .scopes
1366 .read()
1367 .unwrap_or_else(|poisoned| poisoned.into_inner())
1368 .has_deadlines();
1369 if !has_deadlines {
1370 self.scope_deadline_added.notified().await;
1371 continue;
1372 }
1373 tokio::time::sleep(Duration::from_secs(1)).await;
1374 if let Err(error) = self.sweep_expired_scopes() {
1375 warn!(%error, "scope expiry sweep failed");
1376 }
1377 }
1378 });
1379 }
1380
1381 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1382 let mut table = self
1383 .scopes
1384 .write()
1385 .unwrap_or_else(|poisoned| poisoned.into_inner());
1386 let swept = table.sweep_expired();
1387 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1388 drop(table);
1389 self.close_scope_drained_routes(drained);
1390 Ok(())
1391 }
1392
1393 fn publish_scope_changes(
1394 &self,
1395 changes: &[crate::scopes::ScopeTagChange],
1396 expired: &[crate::scopes::ScopeExpired],
1397 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1398 if changes.is_empty() {
1399 return Ok(Vec::new());
1400 }
1401 let drained = self
1402 .forwarding
1403 .publish_scope_changes(changes)
1404 .map_err(RouterError::Forwarding)?;
1405 let mut counts = HashMap::new();
1406 for route in &drained {
1407 *counts
1408 .entry((
1409 route.scope.owner.as_str(),
1410 route.scope.scope_ref.as_str(),
1411 route.scope.tag.scope_epoch,
1412 ))
1413 .or_insert(0usize) += 1;
1414 }
1415 for scope in expired {
1416 let routes_closed = counts
1417 .get(&(
1418 scope.owner.as_str(),
1419 scope.scope_ref.as_str(),
1420 scope.scope_epoch,
1421 ))
1422 .copied()
1423 .unwrap_or(0);
1424 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1425 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1426 routes_closed, "scope expired");
1427 }
1428 Ok(drained)
1429 }
1430
1431 fn runtime_capability_snapshot(
1432 &self,
1433 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1434 let runtime = self
1435 .supervisor
1436 .list()
1437 .into_iter()
1438 .map(|module| {
1439 let status = module.status().map_err(|err| {
1440 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1441 })?;
1442 Ok(RuntimeModule {
1443 module_id: status.module_id,
1444 state: status.state,
1445 enabled: status.enabled,
1446 })
1447 })
1448 .collect::<Result<Vec<_>, RouterError>>()?;
1449 let registrations = self.registry.list_registered_modules().map_err(|err| {
1455 RouterError::backend(
1456 0,
1457 0,
1458 format!("failed to list capability registrations: {err}"),
1459 )
1460 })?;
1461 let registrations = registrations
1462 .into_iter()
1463 .map(|registration| RegisteredModule {
1464 module_id: registration.manifest.module_id,
1465 module_version: registration.manifest.module_version,
1466 capabilities: registration.manifest.capabilities,
1467 })
1468 .collect();
1469 Ok((runtime, registrations))
1470 }
1471
1472 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1479 let cached_registration = RegisteredModule {
1480 module_id: registration.manifest.module_id.clone(),
1481 module_version: registration.manifest.module_version.clone(),
1482 capabilities: registration.manifest.capabilities.clone(),
1483 };
1484 let change = self.capability_evaluator.record_hello(&cached_registration);
1485 log_manifest_change(&cached_registration.module_id, &change);
1486 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1487 self.enforce_capability_denies();
1488 }
1489 self.refresh_capability_requirements();
1490 }
1491
1492 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1496 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1497 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1498 self.supervisor.set_swap_promotion_observer(observer);
1499 }
1500
1501 pub fn refresh_capability_requirements(&self) {
1502 match self.runtime_capability_snapshot() {
1503 Ok((runtime, registrations)) => {
1504 log_requirement_events(
1505 self.capability_evaluator
1506 .evaluate_now(&runtime, ®istrations),
1507 &self.daemon_shutdown,
1508 );
1509 }
1510 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1511 }
1512 }
1513
1514 fn enforce_capability_denies(&self) {
1518 let registrations = match self.registry.list_registered_modules() {
1519 Ok(snapshot) => snapshot,
1520 Err(err) => {
1521 warn!(error = %err, "failed to read registrations for capability deny census");
1522 return;
1523 }
1524 };
1525 let manifests = registrations
1526 .into_iter()
1527 .map(|registration| {
1528 (
1529 registration.manifest.module_id.clone(),
1530 registration.manifest,
1531 )
1532 })
1533 .collect::<BTreeMap<_, _>>();
1534 let census = match self.forwarding.route_census(None) {
1535 Ok(census) => census,
1536 Err(err) => {
1537 warn!(error = %err, "failed to read route census for capability deny enforcement");
1538 return;
1539 }
1540 };
1541
1542 for (target_module_id, routes) in census {
1543 let Some(target_manifest) = manifests.get(&target_module_id) else {
1544 continue;
1545 };
1546 let mut closed_routes = Vec::new();
1547 let mut goodbyes = Vec::new();
1548 for route in routes {
1549 let Principal::Reserved {
1550 module_id: opening_module_id,
1551 } = &route.principal
1552 else {
1553 continue;
1554 };
1555 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1556 continue;
1557 };
1558 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1559 continue;
1560 };
1561
1562 match self.forwarding.release_client_route(
1563 route.goodbye_target.connection_id,
1564 route.goodbye_target.channel,
1565 route.goodbye_target.epoch,
1566 ) {
1567 Ok(RouteRelease::Removed(module_goodbye)) => {
1568 warn!(
1569 opening_module_id,
1570 target_module_id,
1571 capability,
1572 "force-closing route because an attested capability deny edge now matches"
1573 );
1574 goodbyes.push(module_goodbye);
1575 goodbyes.push(route.goodbye_target.clone());
1576 closed_routes.push(route);
1577 }
1578 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1579 Err(err) => warn!(
1580 opening_module_id,
1581 target_module_id,
1582 capability,
1583 error = %err,
1584 "failed to force-close capability-denied route"
1585 ),
1586 }
1587 }
1588
1589 if closed_routes.is_empty() {
1590 continue;
1591 }
1592 send_route_control_pushes(
1593 &self.forwarding,
1594 closed_routes,
1595 ClientControlPush::RouteClosed {
1596 module_id: target_module_id,
1597 channels: Vec::new(),
1598 reason: RouteCloseReason::CapabilityDenied,
1599 drained: false,
1600 abandoned: 0,
1601 excluded_subscriptions: 0,
1602 terminal: Some(false),
1603 },
1604 );
1605 self.emit_route_goodbyes(goodbyes);
1608 }
1609 }
1610
1611 fn not_ready_reason(
1617 &self,
1618 registration: &crate::registry::ModuleRegistration,
1619 ) -> Option<NotReadyReason> {
1620 if !registration.ready {
1621 return Some(NotReadyReason {
1622 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1623 capability: None,
1624 });
1625 }
1626 self.first_unprovided_required_capability(registration)
1627 .map(|capability| NotReadyReason {
1628 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1629 capability: Some(capability),
1630 })
1631 }
1632
1633 fn first_unprovided_required_capability(
1651 &self,
1652 registration: &crate::registry::ModuleRegistration,
1653 ) -> Option<String> {
1654 let required = registration
1655 .manifest
1656 .capabilities
1657 .iter()
1658 .flat_map(|declarations| declarations.requires.iter())
1659 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1660 .map(|requirement| requirement.capability.as_str())
1661 .collect::<BTreeSet<_>>();
1662 if required.is_empty() {
1663 return None;
1664 }
1665 let module_id = registration.manifest.module_id.as_str();
1666 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1667 if required
1668 .iter()
1669 .any(|capability| verdict(capability).is_none())
1670 {
1671 self.refresh_capability_requirements();
1672 }
1673 required
1674 .into_iter()
1675 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1676 .map(str::to_string)
1677 }
1678
1679 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1680 self.capability_evaluator
1681 .statuses()
1682 .into_iter()
1683 .map(capability_requirement_status)
1684 .collect()
1685 }
1686
1687 fn deregister_connection(
1694 &self,
1695 connection_id: ConnectionId,
1696 reason: RegistrationEndReason,
1697 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1698 self.registry
1699 .deregister_connection_with_reason(connection_id, reason)
1700 }
1701
1702 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1703 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1704 return None;
1705 }
1706 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1707 parse_client_control_request(&frame.body)
1708 else {
1709 return None;
1710 };
1711 Some(target_module_id(&target).to_string())
1712 }
1713
1714 pub(crate) fn route_open_capacity_refusal(
1715 &self,
1716 ctx: &RouteCtx,
1717 frame: &Frame,
1718 target_module_id: &str,
1719 in_flight: usize,
1720 limit: usize,
1721 ) -> Result<Frame, RouterError> {
1722 let caller = self.route_open_caller_from_frame(frame);
1725 self.route_open_admission_refusal_frame(
1726 ctx,
1727 &caller,
1728 frame,
1729 target_module_id,
1730 "open_admission_full",
1731 (in_flight, limit),
1732 format!(
1733 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1734 ),
1735 )
1736 }
1737
1738 fn route_open_target_capacity_refusal(
1739 &self,
1740 ctx: &RouteCtx,
1741 caller: &RouteOpenCaller,
1742 frame: &Frame,
1743 target_module_id: &str,
1744 in_flight: usize,
1745 ) -> Result<Frame, RouterError> {
1746 self.route_open_admission_refusal_frame(
1747 ctx,
1748 caller,
1749 frame,
1750 target_module_id,
1751 "target_binds_full",
1752 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1753 format!(
1754 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1755 ),
1756 )
1757 }
1758
1759 #[allow(clippy::too_many_arguments)]
1767 fn route_open_admission_refusal_frame(
1768 &self,
1769 ctx: &RouteCtx,
1770 caller: &RouteOpenCaller,
1771 frame: &Frame,
1772 target_module_id: &str,
1773 reason: &'static str,
1774 (in_flight, limit): (usize, usize),
1775 message: impl Into<String>,
1776 ) -> Result<Frame, RouterError> {
1777 let code = error_codes::TARGET_UNAVAILABLE;
1778 self.counters.increment_route_open_refused(code);
1779 info!(
1780 target: "control",
1781 code,
1782 reason,
1783 module_id = ?target_module_id,
1784 connection_id = ctx.connection_id.get(),
1785 principal = %caller.principal,
1786 scope = %caller.scope,
1787 harness = %caller.harness,
1788 in_flight,
1789 limit,
1790 "route.open refused"
1791 );
1792 control_error_frame(frame, code, message.into())
1793 }
1794
1795 #[cfg(test)]
1801 pub fn handle_control(
1802 &self,
1803 connection_id: ConnectionId,
1804 frame: Frame,
1805 ) -> Result<Vec<Frame>, RouterError> {
1806 match frame.header.ty {
1807 FrameType::Ping => Ok(vec![pong(&frame)?]),
1808 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1809 FrameType::Goodbye => self.handle_goodbye(connection_id),
1810 ty => Ok(vec![control_error_frame(
1811 &frame,
1812 "unsupported_control_frame",
1813 format!("unsupported channel-0 frame {ty:?}"),
1814 )?]),
1815 }
1816 }
1817
1818 pub async fn handle_control_frame(
1819 &self,
1820 ctx: &RouteCtx,
1821 frame: Frame,
1822 ) -> Result<Vec<Frame>, RouterError> {
1823 self.handle_control_frame_timed(ctx, frame, None).await
1824 }
1825
1826 #[cfg(any(test, feature = "test-support"))]
1827 pub fn with_test_operator_app_policy(
1828 mut self,
1829 policy: crate::operator_apps::TestOperatorAppPolicy,
1830 ) -> Self {
1831 self.operator_app_policy = policy.into();
1832 self
1833 }
1834
1835 #[cfg(all(windows, any(test, feature = "test-support")))]
1839 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
1840 self.windows_verify = Some(verify);
1841 self
1842 }
1843
1844 #[cfg(all(windows, any(test, feature = "test-support")))]
1846 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
1847 self.windows_flash = Some(flash);
1848 self
1849 }
1850
1851 #[cfg(all(windows, any(test, feature = "test-support")))]
1855 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
1856 self.windows_track = Some(track);
1857 self
1858 }
1859
1860 pub(crate) fn record_captured_peer(
1861 &self,
1862 connection: ConnectionId,
1863 peer: subc_os::local_peer::CapturedPeer,
1864 ) {
1865 self.record_observed_peer(connection, peer.observation().clone());
1866 #[cfg(windows)]
1867 if let Some(stamp) = self.stamp_windows_launch(&peer) {
1868 self.operator_apps
1869 .lock()
1870 .unwrap_or_else(|p| p.into_inner())
1871 .insert(connection, stamp);
1872 }
1873 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1874 self.operator_apps
1875 .lock()
1876 .unwrap_or_else(|p| p.into_inner())
1877 .insert(connection, stamp);
1878 }
1879 }
1880
1881 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
1882 self.local_connections
1883 .lock()
1884 .unwrap_or_else(|p| p.into_inner())
1885 .insert(connection);
1886 }
1887
1888 fn connection_operator_app(
1891 &self,
1892 stamp: Option<&crate::operator_apps::OperatorAppStamp>,
1893 ) -> Option<subc_protocol::peer::OperatorApp> {
1894 stamp
1895 .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
1896 .map(|stamp| stamp.app.clone())
1897 }
1898
1899 fn handle_connection_describe(
1900 &self,
1901 ctx: &RouteCtx,
1902 frame: Frame,
1903 ) -> Result<Vec<Frame>, RouterError> {
1904 if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
1905 return Ok(vec![control_error_frame(
1906 &frame,
1907 "invalid_control_body",
1908 "malformed connection.describe body",
1909 )?]);
1910 }
1911 let local = self
1912 .local_connections
1913 .lock()
1914 .unwrap_or_else(|p| p.into_inner())
1915 .contains(&ctx.connection_id);
1916 let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
1917 let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
1918 Ok(vec![control_response_body_frame(
1919 &frame,
1920 &subc_control::ConnectionDescribeReply::new(local, operator_app),
1921 ops::CONNECTION_DESCRIBE,
1922 )?])
1923 }
1924
1925 pub(crate) fn record_observed_peer(
1926 &self,
1927 connection: ConnectionId,
1928 peer: subc_os::local_peer::ObservedPeer,
1929 ) {
1930 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1931 let peer = match peer {
1932 OsPeer::Unobserved => ObservedPeer::unobserved(),
1933 OsPeer::Observed(peer) => {
1934 let observed = match peer.user {
1935 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1936 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1937 };
1938 observed
1939 .with_exe(
1940 peer.exe
1941 .and_then(|path| path.into_os_string().into_string().ok()),
1942 )
1943 .with_signing_id(peer.signing_id)
1944 }
1945 };
1946 self.observed_peers
1947 .lock()
1948 .unwrap_or_else(|p| p.into_inner())
1949 .insert(connection, peer);
1950 }
1951
1952 pub(crate) fn record_caller_endpoints(
1953 &self,
1954 connection: ConnectionId,
1955 endpoints: Option<(SocketAddr, SocketAddr)>,
1956 ) {
1957 let mut callers = self
1958 .caller_endpoints
1959 .lock()
1960 .unwrap_or_else(|p| p.into_inner());
1961 callers.remove(&connection);
1962 if let Some(endpoints) = endpoints {
1963 callers.insert(connection, endpoints);
1964 }
1965 }
1966
1967 #[cfg(test)]
1968 pub(crate) fn with_caller_lookup_for_test(
1969 mut self,
1970 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1971 ) -> Self {
1972 self.caller_lookup = Some(Arc::new(lookup));
1973 self
1974 }
1975
1976 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1977 let endpoints = self
1978 .caller_endpoints
1979 .lock()
1980 .unwrap_or_else(|p| p.into_inner())
1981 .get(&connection)
1982 .copied();
1983 #[cfg(test)]
1984 let lookup = self.caller_lookup.clone();
1985 tokio::task::spawn_blocking(move || {
1986 #[cfg(test)]
1987 if let Some(lookup) = lookup {
1988 return lookup(endpoints);
1989 }
1990 endpoints
1991 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1992 .unwrap_or_default()
1993 })
1994 .await
1995 .unwrap_or_default()
1996 }
1997
1998 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1999 frame.header.channel == 0
2000 && frame.header.ty == FrameType::Request
2001 && self
2002 .forwarding
2003 .module_endpoint_for_connection(ctx.connection_id)
2004 .is_ok_and(|endpoint| endpoint.is_none())
2005 && parse_client_control_request(&frame.body)
2006 .ok()
2007 .is_some_and(|request| privileged_operation(&request).is_some())
2008 }
2009
2010 pub(crate) async fn handle_control_frame_timed(
2011 &self,
2012 ctx: &RouteCtx,
2013 frame: Frame,
2014 dispatch_started_at: Option<StdInstant>,
2015 ) -> Result<Vec<Frame>, RouterError> {
2016 match frame.header.ty {
2017 FrameType::Ping => Ok(vec![pong(&frame)?]),
2018 FrameType::Hello => {
2019 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
2020 }
2021 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2022 FrameType::Cancel => {
2023 if self
2026 .forwarding
2027 .operator_confirms()
2028 .cancel(ctx.connection_id, frame.header.corr)
2029 || self
2030 .supervisor
2031 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2032 || self.cancel_activity_subscription(ctx, &frame)?
2033 {
2034 Ok(Vec::new())
2035 } else {
2036 Ok(vec![control_error_frame(
2037 &frame,
2038 "unknown_subscription",
2039 "no subscription has this correlation id",
2040 )?])
2041 }
2042 }
2043 FrameType::Request => {
2044 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2047 if op
2048 .as_ref()
2049 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2050 {
2051 return self.handle_activity_record(ctx, frame).await;
2052 }
2053 if op.as_ref().is_some_and(|probe| {
2054 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2055 }) {
2056 return self.handle_activity_read(ctx, frame).await;
2057 }
2058 if op
2059 .as_ref()
2060 .is_some_and(|probe| probe.op == "operator.confirm")
2061 {
2062 return self.handle_operator_confirm(ctx, frame);
2063 }
2064 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2065 return self
2066 .observe_control_wait(
2067 ctx,
2068 None,
2069 ops::APP_OPEN,
2070 self.handle_app_open(ctx, frame),
2071 )
2072 .await;
2073 }
2074 if op
2075 .as_ref()
2076 .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2077 {
2078 return self.handle_connection_describe(ctx, frame);
2079 }
2080 if op
2081 .as_ref()
2082 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2083 {
2084 return self.handle_scope_served_by(ctx.connection_id, frame);
2085 }
2086 if self
2087 .forwarding
2088 .module_endpoint_for_connection(ctx.connection_id)
2089 .map_err(RouterError::Forwarding)?
2090 .is_some()
2091 {
2092 if !is_known_module_request_op(&frame.body) {
2093 return Ok(vec![control_error_frame(
2094 &frame,
2095 "unsupported_control_frame",
2096 "module-originated channel-0 REQUEST is not supported",
2097 )?]);
2098 }
2099 let request = match parse_module_control_request_from_module(&frame.body) {
2100 Ok(request) => request,
2101 Err((err, ControlRequestBodyError::UnknownOp)) => {
2102 return Ok(vec![control_error_frame(
2103 &frame,
2104 "unsupported_control_frame",
2105 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2106 )?])
2107 }
2108 Err((err, ControlRequestBodyError::InvalidBody)) => {
2109 return Ok(vec![control_error_frame(
2110 &frame,
2111 "invalid_control_body",
2112 format!("malformed module control body: {err}"),
2113 )?])
2114 }
2115 };
2116 let op = module_control_request_op(&request);
2117 let corr = frame.header.corr;
2118 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2119 let result =
2120 self.handle_module_control_request(ctx.connection_id, frame, request);
2121 log_slow_control_dispatch(
2122 dispatch_started_at,
2123 op,
2124 ctx.connection_id,
2125 corr,
2126 None,
2127 );
2128 return result;
2129 }
2130
2131 if is_known_module_request_op(&frame.body) {
2132 return Ok(vec![control_error_frame(
2133 &frame,
2134 "not_registered",
2135 "catalog.update requires an active module registration owned by this connection",
2136 )?]);
2137 }
2138
2139 let request = match parse_client_control_request(&frame.body) {
2140 Ok(request) => request,
2141 Err((err, ControlRequestBodyError::UnknownOp)) => {
2142 return Ok(vec![control_error_frame(
2143 &frame,
2144 "unknown_control_op",
2145 format!("unknown client control op: {err}"),
2146 )?])
2147 }
2148 Err((err, ControlRequestBodyError::InvalidBody)) => {
2149 return Ok(vec![control_error_frame(
2150 &frame,
2151 "invalid_control_body",
2152 format!("malformed client control body: {err}"),
2153 )?])
2154 }
2155 };
2156 let op = client_control_request_op(&request);
2157 let corr = frame.header.corr;
2158 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2159 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2160 #[cfg(test)]
2161 if let Some(delay) = self.control_dispatch_delay {
2162 tokio::time::sleep(delay).await;
2163 }
2164 let result = self
2165 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2166 .await;
2167 if let Some(timings) = route_open_timings.as_mut() {
2168 timings.finish();
2169 }
2170 log_slow_control_dispatch(
2171 dispatch_started_at,
2172 op,
2173 ctx.connection_id,
2174 corr,
2175 route_open_timings.as_ref(),
2176 );
2177 result
2178 }
2179 FrameType::Push => {
2180 let Some(endpoint) = self
2181 .forwarding
2182 .module_endpoint_for_connection(ctx.connection_id)
2183 .map_err(RouterError::Forwarding)?
2184 else {
2185 return Ok(vec![control_error_frame(
2186 &frame,
2187 "unsupported_control_frame",
2188 "client-originated channel-0 PUSH is not supported",
2189 )?]);
2190 };
2191 self.handle_status_update(endpoint, frame)
2192 }
2193 FrameType::Response | FrameType::Error
2194 if self
2195 .forwarding
2196 .module_endpoint_for_connection(ctx.connection_id)
2197 .map_err(RouterError::Forwarding)?
2198 .is_some() =>
2199 {
2200 self.handle_module_relay_response(ctx.connection_id, frame)
2201 }
2202 ty => Ok(vec![control_error_frame(
2203 &frame,
2204 "unsupported_control_frame",
2205 format!("unsupported channel-0 frame {ty:?}"),
2206 )?]),
2207 }
2208 }
2209
2210 pub fn cleanup_connection(
2211 &self,
2212 connection_id: ConnectionId,
2213 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2214 self.local_connections
2215 .lock()
2216 .unwrap_or_else(|p| p.into_inner())
2217 .remove(&connection_id);
2218 self.caller_endpoints
2219 .lock()
2220 .unwrap_or_else(|p| p.into_inner())
2221 .remove(&connection_id);
2222 self.build_class_warned
2223 .lock()
2224 .unwrap_or_else(|p| p.into_inner())
2225 .remove(&connection_id);
2226 self.observed_peers
2227 .lock()
2228 .unwrap_or_else(|p| p.into_inner())
2229 .remove(&connection_id);
2230 self.operator_apps
2231 .lock()
2232 .unwrap_or_else(|p| p.into_inner())
2233 .remove(&connection_id);
2234 self.forwarding
2235 .operator_confirms()
2236 .connection_closed(connection_id);
2237 self.cleanup_connection_with_end_reason(
2238 connection_id,
2239 RegistrationEndReason::ConnectionClosed,
2240 )
2241 }
2242
2243 fn cleanup_connection_with_end_reason(
2244 &self,
2245 connection_id: ConnectionId,
2246 requested_reason: RegistrationEndReason,
2247 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2248 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2249 self.registry
2250 .get_module_by_connection(connection_id)?
2251 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2252 .and_then(|module| module.registration_end_reason().ok().flatten())
2253 .unwrap_or(requested_reason)
2254 } else {
2255 requested_reason
2256 };
2257 let crash_closed = self
2258 .registry
2259 .get_module_by_connection(connection_id)?
2260 .and_then(|registration| {
2261 self.forwarding
2262 .module_endpoint_for_connection(connection_id)
2263 .ok()
2264 .flatten()
2265 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2266 .map(|routes| (registration.manifest.module_id, routes))
2267 });
2268 let crash_closed = crash_closed.map(|(module_id, routes)| {
2269 let terminal = match self.supervisor.get(&module_id) {
2270 None => false,
2271 Some(module) => match module.will_recover_after_connection_loss() {
2272 Ok(will_recover) => !will_recover,
2273 Err(err) => {
2274 warn!(
2275 %module_id,
2276 error = %err,
2277 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2278 );
2279 false
2280 }
2281 },
2282 };
2283 let (reason, drained) = match self
2287 .forwarding
2288 .daemon_drain_result_for_connection(connection_id)
2289 {
2290 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2291 Ok(None) => (RouteCloseReason::Crash, false),
2292 Err(err) => {
2293 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2294 (RouteCloseReason::Crash, false)
2295 }
2296 };
2297 (module_id, routes, reason, drained, terminal)
2298 });
2299 let registrations = self.deregister_connection(connection_id, end_reason);
2300 let cleanup = if crash_closed.is_some() {
2301 self.forwarding.cleanup_connection_counted(connection_id)
2302 } else {
2303 self.forwarding
2309 .cleanup_connection(connection_id)
2310 .map(|released| crate::forwarding::ConnectionCleanup {
2311 released,
2312 abandoned_relays: 0,
2313 })
2314 };
2315 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2320 let abandoned = cleanup
2321 .as_ref()
2322 .map_or(0, |cleanup| cleanup.abandoned_relays);
2323 send_route_control_pushes(
2324 &self.forwarding,
2325 routes,
2326 ClientControlPush::RouteClosed {
2327 module_id,
2328 channels: Vec::new(),
2329 reason,
2330 drained,
2331 abandoned,
2332 excluded_subscriptions: 0,
2333 terminal: Some(terminal),
2334 },
2335 );
2336 }
2337 if let Ok(cleanup) = cleanup {
2338 self.emit_route_goodbyes(cleanup.released);
2339 }
2340 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2344 crate::supervise::notify_registration_release();
2345 self.capability_evaluator.wake_deadline_loop();
2346 self.refresh_capability_requirements();
2347 }
2348 self.supervisor.remove_spawn_subscribers(connection_id);
2349 self.activity_streams
2350 .lock()
2351 .unwrap_or_else(|p| p.into_inner())
2352 .retain(|(id, _), task| {
2353 if *id == connection_id {
2354 task.abort();
2355 false
2356 } else {
2357 true
2358 }
2359 });
2360 self.hello_launch_nonces
2363 .lock()
2364 .unwrap_or_else(|poisoned| poisoned.into_inner())
2365 .forget(connection_id);
2366 self.scopes
2367 .write()
2368 .unwrap_or_else(|poisoned| poisoned.into_inner())
2369 .release_connection(connection_id);
2370 registrations
2371 }
2372
2373 pub(crate) fn handle_route_goodbye(
2374 &self,
2375 connection_id: ConnectionId,
2376 route_channel: u16,
2377 route_epoch: u32,
2378 ) -> Result<bool, RouterError> {
2379 debug!(
2380 connection_id = connection_id.get(),
2381 route_channel, route_epoch, "handling route GOODBYE"
2382 );
2383 let RouteRelease::Removed(released_route) = self
2384 .forwarding
2385 .release_client_route(connection_id, route_channel, route_epoch)
2386 .map_err(RouterError::Forwarding)?
2387 else {
2388 return Ok(false);
2389 };
2390 self.emit_route_goodbyes(vec![released_route]);
2391 Ok(true)
2392 }
2393
2394 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2395 for released in released_routes {
2396 let frame = match Frame::build_with_version(
2397 released.negotiated_ver,
2398 FrameType::Goodbye,
2399 control_flags(),
2400 released.channel,
2401 released.epoch,
2402 0,
2403 Vec::new(),
2404 ) {
2405 Ok(frame) => frame,
2406 Err(err) => {
2407 warn!(
2408 route_channel = released.channel,
2409 error = %err,
2410 "failed to build route GOODBYE frame"
2411 );
2412 continue;
2413 }
2414 };
2415 if !released.close_on_delivery_failure() {
2416 crate::forwarding::send_module_route_goodbye(
2417 &self.counters,
2418 &released.sink,
2419 frame,
2420 released.module_id.as_deref(),
2421 "client route released",
2422 );
2423 continue;
2424 }
2425 if let Err(err) = released.sink.try_send(frame) {
2426 warn!(
2427 target_connection_id = released.connection_id.get(),
2428 route_channel = released.channel,
2429 error = %err,
2430 "route GOODBYE was not delivered to client; closing target connection"
2431 );
2432 if self
2433 .forwarding
2434 .escalate_client_delivery_failure(
2435 released.connection_id,
2436 released.channel,
2437 released.epoch,
2438 CloseReason::new(
2439 "route_goodbye_delivery_failed",
2440 format!(
2441 "failed to enqueue route GOODBYE for channel {}: {err}",
2442 released.channel
2443 ),
2444 ),
2445 crate::forwarding::UndeliveredFrame {
2446 module_id: released.module_id.as_deref(),
2447 sink: &released.sink,
2448 },
2449 )
2450 .unwrap_or(false)
2451 {
2452 self.counters.increment_goodbye_relay_client_failed();
2453 }
2454 }
2455 }
2456 }
2457
2458 fn send_abandoned_route_bind_goodbye(
2470 &self,
2471 module_sink: &crate::FrameSink,
2472 negotiated_ver: u8,
2473 module_channel: u16,
2474 module_epoch: u32,
2475 ) {
2476 let frame = match Frame::build_with_version(
2477 negotiated_ver,
2478 FrameType::Goodbye,
2479 control_flags(),
2480 module_channel,
2481 module_epoch,
2482 0,
2483 Vec::new(),
2484 ) {
2485 Ok(frame) => frame,
2486 Err(err) => {
2487 warn!(
2488 route_channel = module_channel,
2489 error = %err,
2490 "failed to build GOODBYE for abandoned route.bind"
2491 );
2492 return;
2493 }
2494 };
2495 crate::forwarding::send_module_route_goodbye(
2496 &self.counters,
2497 module_sink,
2498 frame,
2499 None,
2500 "abandoned route.bind",
2501 );
2502 }
2503
2504 fn handle_hello(
2505 &self,
2506 connection_id: ConnectionId,
2507 sink: Option<crate::FrameSink>,
2508 frame: Frame,
2509 ) -> Result<Vec<Frame>, RouterError> {
2510 debug!(
2511 connection_id = connection_id.get(),
2512 corr = frame.header.corr,
2513 "handling HELLO"
2514 );
2515 if self
2519 .registry
2520 .get_module_by_connection(connection_id)
2521 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2522 .is_some()
2523 {
2524 return Ok(vec![control_error_frame(
2525 &frame,
2526 "invalid_hello",
2527 "connection is already registered as a module",
2528 )?]);
2529 }
2530 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2531 Ok(value) => value,
2532 Err(err) => {
2533 return Ok(vec![control_error_frame(
2534 &frame,
2535 "invalid_hello",
2536 format!("malformed HELLO body: {err}"),
2537 )?])
2538 }
2539 };
2540 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2541 return Ok(vec![control_error_frame(
2542 &frame,
2543 "invalid_capability_grammar",
2544 err.to_string(),
2545 )?]);
2546 }
2547 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2548 return Ok(vec![control_error_frame(
2549 &frame,
2550 "invalid_manifest",
2551 err.to_string(),
2552 )?]);
2553 }
2554 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2555 return Ok(vec![control_error_frame(
2556 &frame,
2557 "invalid_event_declaration",
2558 err.to_string(),
2559 )?]);
2560 }
2561 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2562 return Ok(vec![control_error_frame(
2563 &frame,
2564 "invalid_manifest",
2565 err.to_string(),
2566 )?]);
2567 }
2568 if let Some(provenance) = hello_value
2569 .get("manifest")
2570 .and_then(|manifest| manifest.get("provenance"))
2571 {
2572 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2573 return Ok(vec![control_error_frame(
2574 &frame,
2575 "invalid_manifest",
2576 format!("malformed manifest provenance: {err}"),
2577 )?]);
2578 }
2579 }
2580 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2581 let module_id = hello_value["manifest"]["module_id"]
2582 .as_str()
2583 .unwrap_or("<unknown>");
2584 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2585 }
2586 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2587 let module_id = hello_value["manifest"]["module_id"]
2588 .as_str()
2589 .unwrap_or("<unknown>");
2590 warn!(module_id, %reason, "dropping flow_operations from HELLO");
2591 }
2592 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2593 let module_id = hello_value["manifest"]["module_id"]
2594 .as_str()
2595 .unwrap_or("<unknown>");
2596 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2597 }
2598 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2599 Ok(hello) => hello,
2600 Err(err) => {
2601 return Ok(vec![control_error_frame(
2602 &frame,
2603 "invalid_hello",
2604 format!("malformed HELLO body: {err}"),
2605 )?])
2606 }
2607 };
2608
2609 if hello.protocol_ver != hello.manifest.protocol_ver {
2610 return Ok(vec![control_error_frame(
2611 &frame,
2612 "invalid_manifest",
2613 format!(
2614 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2615 hello.protocol_ver, hello.manifest.protocol_ver
2616 ),
2617 )?]);
2618 }
2619
2620 if hello.manifest.module_id.trim().is_empty() {
2621 return Ok(vec![control_error_frame(
2622 &frame,
2623 "invalid_manifest",
2624 "manifest module_id must not be empty",
2625 )?]);
2626 }
2627
2628 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2629 Ok(negotiated_ver) => negotiated_ver,
2630 Err(message) => {
2631 return Ok(vec![control_error_frame(
2632 &frame,
2633 "version_unsupported",
2634 message,
2635 )?])
2636 }
2637 };
2638
2639 let swap_admission = self
2647 .supervisor
2648 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2649 if swap_admission == SwapHelloAdmission::Refused {
2650 warn!(
2651 module_id = %hello.manifest.module_id,
2652 connection_id = connection_id.get(),
2653 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2654 );
2655 return Ok(vec![control_error_frame(
2656 &frame,
2657 "swap_token_invalid",
2658 format!(
2659 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2660 hello.manifest.module_id
2661 ),
2662 )?]);
2663 }
2664 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2665
2666 let nonce_verified = if swap_candidate {
2671 true
2672 } else {
2673 match self.supervisor.reserved_hello_verification(
2674 &hello.manifest.module_id,
2675 hello.launch_nonce.as_deref(),
2676 ) {
2677 Ok(verified) => verified,
2678 Err(rejection) => {
2679 let message = match rejection {
2680 ReservedHelloRejection::Exact { module_id } => format!(
2681 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2682 ),
2683 ReservedHelloRejection::Prefix {
2684 prefix,
2685 owner_module_id,
2686 } => format!(
2687 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2688 hello.manifest.module_id
2689 ),
2690 };
2691 return Ok(vec![control_error_frame(
2692 &frame,
2693 "reserved_module",
2694 message,
2695 )?]);
2696 }
2697 }
2698 };
2699
2700 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2701 &hello.manifest.module_id,
2702 hello.manifest.capabilities.as_ref(),
2703 );
2704 if let Some(refusal) = reserved_capability_refusals.first() {
2705 let capability = refusal.capability.clone();
2706 let bound_module = refusal.claimants[0].clone();
2707 log_duplicate_claim_events(reserved_capability_refusals);
2708 return Ok(vec![control_error_frame(
2709 &frame,
2710 "reserved_capability",
2711 format!(
2712 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2713 capability, bound_module, hello.manifest.module_id
2714 ),
2715 )?]);
2716 }
2717
2718 if self
2721 .forwarding
2722 .connection_has_client_routes(connection_id)
2723 .map_err(RouterError::Forwarding)?
2724 {
2725 return Ok(vec![control_error_frame(
2726 &frame,
2727 "invalid_hello",
2728 "connection has open client routes and cannot also register as a module",
2729 )?]);
2730 }
2731
2732 self.hello_launch_nonces
2737 .lock()
2738 .unwrap_or_else(|poisoned| poisoned.into_inner())
2739 .record(connection_id, hello.launch_nonce.as_deref());
2740 let control_ops = effective_module_control_ops(hello.control_ops);
2741 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2744 if swap_candidate {
2745 return self.register_swap_candidate(
2746 connection_id,
2747 sink,
2748 &frame,
2749 hello.manifest,
2750 negotiated_ver,
2751 control_ops,
2752 hello_ack,
2753 );
2754 }
2755 let registration = match self
2762 .registry
2763 .register_unpublished_verified_with_control_ops(
2764 hello.manifest,
2765 negotiated_ver,
2766 connection_id,
2767 control_ops,
2768 nonce_verified,
2769 ) {
2770 Ok(registration) => registration,
2771 Err(RegistryError::DuplicateModuleId { module_id }) => {
2772 return Ok(vec![control_error_frame(
2773 &frame,
2774 "duplicate_module_id",
2775 format!(
2776 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2777 ),
2778 )?])
2779 }
2780 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2781 return Ok(vec![control_error_frame(
2782 &frame,
2783 "invalid_module_id",
2784 err.to_string(),
2785 )?])
2786 }
2787 Err(err) => {
2788 return Ok(vec![control_error_frame(
2789 &frame,
2790 "registry_error",
2791 err.to_string(),
2792 )?])
2793 }
2794 };
2795 #[cfg(test)]
2796 self.hold_between_registry_and_forwarding();
2797
2798 let reply = if let Some(sink) = sink {
2799 let concurrency = manifest_concurrency(®istration.manifest);
2815 if let Err(err) = self.forwarding.register_module_connection_acked(
2816 connection_id,
2817 registration.manifest.module_id.clone(),
2818 negotiated_ver,
2819 concurrency,
2820 sink,
2821 hello_ack,
2822 ) {
2823 if matches!(
2828 self.deregister_connection(
2829 connection_id,
2830 RegistrationEndReason::RegistrationFailed,
2831 ),
2832 Ok(r) if !r.is_empty()
2833 ) {
2834 crate::supervise::notify_registration_release();
2835 }
2836 return Ok(vec![control_error_frame(
2837 &frame,
2838 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2839 "invalid_hello"
2840 } else {
2841 forwarding_error_code(&err)
2842 },
2843 err.to_string(),
2844 )?]);
2845 }
2846 Vec::new()
2847 } else {
2848 vec![hello_ack]
2851 };
2852
2853 if !self
2858 .registry
2859 .publish_registration(connection_id)
2860 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2861 {
2862 debug!(
2863 module_id = %registration.manifest.module_id,
2864 connection_id = connection_id.get(),
2865 "registration was removed before it could be published"
2866 );
2867 }
2868
2869 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2878 info!(
2879 module_id = %registration.manifest.module_id,
2880 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2881 );
2882 }
2883
2884 self.apply_registration_capabilities(®istration);
2885
2886 self.supervisor
2887 .record_activity_registration(®istration.manifest.module_id);
2888
2889 info!(
2890 module_id = %registration.manifest.module_id,
2891 module_version = %registration.manifest.module_version,
2892 negotiated_ver,
2893 routable_provider = manifest_provides_routable_role(®istration.manifest),
2894 connection_id = connection_id.get(),
2895 "module registered"
2896 );
2897
2898 Ok(reply)
2899 }
2900
2901 #[allow(clippy::too_many_arguments)]
2918 fn register_swap_candidate(
2919 &self,
2920 connection_id: ConnectionId,
2921 sink: Option<crate::FrameSink>,
2922 frame: &Frame,
2923 manifest: ModuleManifest,
2924 negotiated_ver: u8,
2925 control_ops: Vec<String>,
2926 hello_ack: Frame,
2927 ) -> Result<Vec<Frame>, RouterError> {
2928 let module_id = manifest.module_id.clone();
2929 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2930 manifest,
2931 negotiated_ver,
2932 connection_id,
2933 control_ops,
2934 ) {
2935 Ok(registration) => registration,
2936 Err(RegistryError::DuplicateModuleId { module_id }) => {
2937 return Ok(vec![control_error_frame(
2938 frame,
2939 "duplicate_module_id",
2940 format!(
2941 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2942 ),
2943 )?])
2944 }
2945 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2946 return Ok(vec![control_error_frame(
2947 frame,
2948 "invalid_module_id",
2949 err.to_string(),
2950 )?])
2951 }
2952 Err(err) => {
2953 return Ok(vec![control_error_frame(
2954 frame,
2955 "registry_error",
2956 err.to_string(),
2957 )?])
2958 }
2959 };
2960 #[cfg(test)]
2961 self.hold_between_registry_and_forwarding();
2962 let reply = if let Some(sink) = sink {
2963 let concurrency = manifest_concurrency(®istration.manifest);
2967 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2968 connection_id,
2969 module_id.clone(),
2970 negotiated_ver,
2971 concurrency,
2972 sink,
2973 hello_ack,
2974 ) {
2975 if matches!(
2976 self.deregister_connection(
2977 connection_id,
2978 RegistrationEndReason::RegistrationFailed,
2979 ),
2980 Ok(r) if !r.is_empty()
2981 ) {
2982 crate::supervise::notify_registration_release();
2983 }
2984 return Ok(vec![control_error_frame(
2985 frame,
2986 forwarding_error_code(&err),
2987 err.to_string(),
2988 )?]);
2989 }
2990 Vec::new()
2991 } else {
2992 vec![hello_ack]
2993 };
2994 if !self
2995 .registry
2996 .publish_registration(connection_id)
2997 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2998 {
2999 debug!(
3000 module_id = %module_id,
3001 connection_id = connection_id.get(),
3002 "swap candidate registration was removed before it could be published"
3003 );
3004 }
3005 self.supervisor.mark_swap_candidate_admitted(&module_id);
3006 info!(
3007 module_id = %module_id,
3008 module_version = %registration.manifest.module_version,
3009 negotiated_ver,
3010 ready = registration.ready,
3011 connection_id = connection_id.get(),
3012 "swap candidate registered; not routable until cutover"
3013 );
3014 Ok(reply)
3015 }
3016
3017 fn build_hello_ack(
3018 &self,
3019 frame: &Frame,
3020 negotiated_ver: u8,
3021 module_id: &str,
3022 ) -> Result<Frame, RouterError> {
3023 let ack = ModuleHelloAckBody {
3024 negotiated_ver,
3025 subc_ops: module_subc_ops(),
3026 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
3027 storage: self
3028 .storage_config
3029 .as_ref()
3030 .map(|cfg| cfg.descriptor_for(module_id)),
3031 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3032 };
3033 let body = serde_json::to_vec(&ack).map_err(|err| {
3034 RouterError::backend(
3035 0,
3036 frame.header.corr,
3037 format!("failed to encode HELLO_ACK: {err}"),
3038 )
3039 })?;
3040
3041 Frame::build_with_version(
3042 negotiated_ver,
3043 FrameType::HelloAck,
3044 control_flags(),
3045 0,
3046 0,
3047 frame.header.corr,
3048 body,
3049 )
3050 .map_err(RouterError::FrameBuild)
3051 }
3052
3053 async fn handle_client_control_request(
3054 &self,
3055 ctx: &RouteCtx,
3056 frame: Frame,
3057 request: ClientControlRequest,
3058 route_open_timings: &mut Option<RouteOpenTimings>,
3059 ) -> Result<Vec<Frame>, RouterError> {
3060 if let Some(operation) = privileged_operation(&request) {
3061 let caller = self.resolve_caller(ctx.connection_id).await;
3062 let op = client_control_request_op(&request);
3063 let text = prompt_text(operation, &caller);
3064 match self.privileged_operator_prompt {
3065 PrivilegedOperatorPrompt::Log => {
3066 info!(target: "subc_daemon::privileged_prompt", operation = op,
3067 target = ?operation, caller = %caller.description(), summary = %text,
3068 "privileged request would prompt the operator");
3069 }
3070 PrivilegedOperatorPrompt::Enforce => {
3071 let key =
3072 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
3073 let stamp = self
3074 .operator_apps
3075 .lock()
3076 .unwrap_or_else(|p| p.into_inner())
3077 .get(&ctx.connection_id)
3078 .cloned();
3079 let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
3080 &self.operator_app_policy,
3081 &self.launch_table,
3082 stamp.as_ref(),
3083 )
3084 .then(|| self.shortcut_context(stamp, None, None));
3085 let outcome = self
3086 .forwarding
3087 .operator_confirms()
3088 .approve_privileged_with_shortcut(
3089 ctx,
3090 &frame,
3091 op,
3092 text,
3093 caller.description(),
3094 key,
3095 shortcut,
3096 )
3097 .await;
3098 if outcome.code != "confirmed" {
3099 return Ok(vec![outcome.frame(&frame)]);
3100 }
3101 }
3102 }
3103 }
3104 match request {
3105 ClientControlRequest::DaemonShutdownIntent { intent } => {
3106 if intent.as_str() != Some("restart") {
3107 return Ok(vec![control_error_frame(
3108 &frame,
3109 "invalid_params",
3110 "intent must be 'restart'",
3111 )?]);
3112 }
3113 let response = ClientControlResponse::DaemonShutdownIntent {
3114 intent: "restart".to_owned(),
3115 expires_at_ms: self.supervisor.announce_daemon_restart(),
3116 };
3117 Ok(vec![control_response_body_frame(
3118 &frame,
3119 &response,
3120 "ClientControlResponse::DaemonShutdownIntent",
3121 )?])
3122 }
3123 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
3124 ClientControlRequest::CatalogList { module_id } => {
3125 self.handle_catalog_list(frame, module_id)
3126 }
3127 ClientControlRequest::RouteOpen {
3128 target,
3129 identity,
3130 consumer_identity,
3131 consumer_capabilities,
3132 role_versions,
3133 busy_refusals,
3134 admission_facts,
3135 scope,
3136 ..
3137 } => {
3138 self.handle_route_open(
3139 ctx,
3140 frame,
3141 RouteOpenRequest {
3142 target,
3143 identity,
3144 consumer_identity,
3145 consumer_capabilities,
3146 role_versions,
3147 busy_refusals,
3148 admission_facts,
3149 scope,
3150 },
3151 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
3152 )
3153 .await
3154 }
3155 ClientControlRequest::RoutePoll {
3156 route_channel,
3157 route_epoch,
3158 kind,
3159 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
3160 ClientControlRequest::SupervisorList {} => {
3161 let handler = self.clone();
3163 let sink = ctx.egress.clone();
3164 tokio::spawn(
3165 async move {
3166 let result = handler.handle_supervisor_list(frame.clone()).await;
3167 let replies = match result {
3168 Ok(replies) => replies,
3169 Err(error) => {
3170 control_error_frame(&frame, "backend_error", error.to_string())
3171 .into_iter()
3172 .collect()
3173 }
3174 };
3175 for reply in replies {
3176 if sink.send(reply).await.is_err() {
3177 break;
3178 }
3179 }
3180 }
3181 .with_current_subscriber(),
3182 );
3183 Ok(Vec::new())
3184 }
3185 ClientControlRequest::SupervisorSpawnSnapshot {} => {
3186 self.handle_supervisor_spawn_snapshot(frame)
3187 }
3188 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
3189 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
3190 }
3191 ClientControlRequest::SupervisorRestart {
3192 module_id,
3193 drain_timeout_ms,
3194 } => {
3195 self.log_supervisor_request_received(
3196 ctx,
3197 frame.header.corr,
3198 ops::SUPERVISOR_RESTART,
3199 Some(&module_id),
3200 None,
3201 )?;
3202 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3203 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
3204 .await
3205 }
3206 ClientControlRequest::SupervisorSwap {
3207 module_id,
3208 ready_timeout_ms,
3209 } => {
3210 self.log_supervisor_request_received(
3211 ctx,
3212 frame.header.corr,
3213 ops::SUPERVISOR_SWAP,
3214 Some(&module_id),
3215 None,
3216 )?;
3217 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3218 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3219 .await
3220 }
3221 ClientControlRequest::SupervisorReload { module_id } => {
3222 self.log_supervisor_request_received(
3223 ctx,
3224 frame.header.corr,
3225 ops::SUPERVISOR_RELOAD,
3226 Some(&module_id),
3227 None,
3228 )?;
3229 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3230 self.handle_supervisor_reload(frame, module_id, requester)
3231 .await
3232 }
3233 ClientControlRequest::SupervisorRescan { preview } => {
3234 if !preview {
3235 self.log_supervisor_request_received(
3236 ctx,
3237 frame.header.corr,
3238 ops::SUPERVISOR_RESCAN,
3239 None,
3240 None,
3241 )?;
3242 }
3243 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3244 let operation = self.handle_supervisor_rescan(
3245 frame,
3246 if preview {
3247 RescanMode::Preview
3248 } else {
3249 RescanMode::Execute
3250 },
3251 requester,
3252 );
3253 self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
3254 .await
3255 }
3256 ClientControlRequest::SupervisorRescanChecked {
3257 preview,
3258 candidate,
3259 expect,
3260 } => {
3261 self.observe_control_wait(
3262 ctx,
3263 None,
3264 ops::SUPERVISOR_RESCAN,
3265 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
3266 )
3267 .await
3268 }
3269 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3270 self.log_supervisor_request_received(
3271 ctx,
3272 frame.header.corr,
3273 ops::SUPERVISOR_RELEASE_RESERVED,
3274 Some(&module_id),
3275 None,
3276 )?;
3277 self.handle_supervisor_release_reserved(frame, module_id)
3278 .await
3279 }
3280 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3281 self.log_supervisor_request_received(
3282 ctx,
3283 frame.header.corr,
3284 ops::SUPERVISOR_SET_ENABLED,
3285 Some(&module_id),
3286 Some(enabled),
3287 )?;
3288 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3289 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3290 .await
3291 }
3292 ClientControlRequest::SupervisorHealthProbe { module_id } => {
3293 self.handle_supervisor_health_probe(ctx, frame, module_id)
3294 .await
3295 }
3296 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3297 ClientControlRequest::SupervisorRoutes { module_id } => {
3298 self.handle_supervisor_routes(frame, module_id)
3299 }
3300 ClientControlRequest::SupervisorProvenance { module_id } => {
3301 self.handle_supervisor_provenance(ctx, frame, module_id)
3302 .await
3303 }
3304 ClientControlRequest::SupervisorStderrTail {
3305 module_id,
3306 max_lines,
3307 max_bytes,
3308 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3309 ClientControlRequest::SupervisorTerminals { module_id } => {
3310 self.handle_supervisor_terminals(ctx, frame, module_id)
3311 .await
3312 }
3313 }
3314 }
3315
3316 fn log_supervisor_request_received(
3317 &self,
3318 ctx: &RouteCtx,
3319 corr: u64,
3320 op: &'static str,
3321 module_id: Option<&str>,
3322 enabled: Option<bool>,
3323 ) -> Result<(), RouterError> {
3324 let caller = self.supervisor_requester(ctx, corr)?;
3325 let caller = principal_label(&caller);
3326 info!(
3329 target: "control",
3330 op,
3331 module_id,
3332 enabled,
3333 connection_id = ctx.connection_id.get(),
3334 caller = %caller,
3335 "supervisor request received"
3336 );
3337 Ok(())
3338 }
3339
3340 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3341 Ok(self
3342 .registry
3343 .get_module_by_connection(ctx.connection_id)
3344 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3345 .map(|registration| Principal::Reserved {
3346 module_id: registration.manifest.module_id,
3347 })
3348 .unwrap_or(Principal::Direct))
3349 }
3350
3351 fn handle_module_control_request(
3352 &self,
3353 connection_id: ConnectionId,
3354 frame: Frame,
3355 request: ModuleControlRequestFromModule,
3356 ) -> Result<Vec<Frame>, RouterError> {
3357 match request {
3358 ModuleControlRequestFromModule::CatalogUpdate {
3359 provides,
3360 capabilities,
3361 ready,
3362 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3363 ModuleControlRequestFromModule::LiveRoots {} => {
3364 let registered = self
3365 .registry
3366 .get_module_by_connection(connection_id)
3367 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3368 let Some(registration) = registered else {
3369 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3370 };
3371 let response = self
3372 .forwarding
3373 .live_roots(®istration.manifest.module_id)
3374 .map_err(RouterError::Forwarding)?;
3375 Ok(vec![control_response_body_frame(
3376 &frame,
3377 &response,
3378 "ModuleControlResponseToModule::LiveRoots",
3379 )?])
3380 }
3381 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3382 self.handle_scope_sync(connection_id, frame, generation, scopes)
3383 }
3384 ModuleControlRequestFromModule::ScopeApply {
3385 generation,
3386 upsert,
3387 end,
3388 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3389 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3390 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3391 }
3392 }
3393 }
3394
3395 fn handle_scope_served_by(
3396 &self,
3397 connection_id: ConnectionId,
3398 frame: Frame,
3399 ) -> Result<Vec<Frame>, RouterError> {
3400 let registered = self
3401 .registry
3402 .get_module_by_connection(connection_id)
3403 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3404 let Some(registered) = registered else {
3405 return Ok(vec![control_error_frame(
3406 &frame,
3407 "not_registered",
3408 "scope.served_by requires an active module registration owned by this connection",
3409 )?]);
3410 };
3411 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3412 {
3413 Ok(request) => request,
3414 Err(err) => {
3415 return Ok(vec![control_error_frame(
3416 &frame,
3417 "invalid_control_body",
3418 format!("malformed scope.served_by body: {err}"),
3419 )?])
3420 }
3421 };
3422 let answer = self
3423 .scopes
3424 .read()
3425 .unwrap_or_else(|poisoned| poisoned.into_inner())
3426 .served_by(
3427 ®istered.manifest.module_id,
3428 &request.scope_ref,
3429 request.scope_epoch,
3430 &request.module_id,
3431 );
3432 let reply = subc_control::ScopeServedByReply::new(
3433 answer,
3434 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3435 );
3436 Ok(vec![control_response_body_frame(
3437 &frame,
3438 &reply,
3439 "ScopeServedByReply",
3440 )?])
3441 }
3442
3443 fn handle_operator_confirm(
3444 &self,
3445 ctx: &RouteCtx,
3446 frame: Frame,
3447 ) -> Result<Vec<Frame>, RouterError> {
3448 use crate::operator_confirm::{audit, Outcome};
3449 let registration = self
3450 .registry
3451 .get_module_by_connection(ctx.connection_id)
3452 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3453 let Some(registration) = registration else {
3454 let outcome = Outcome::refusal("not_registered");
3455 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3456 return Ok(vec![outcome.frame(&frame)]);
3457 };
3458 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3459 Ok(request) => request,
3460 Err(_) => {
3461 let outcome = Outcome::refusal("invalid_control_body");
3462 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3463 return Ok(vec![outcome.frame(&frame)]);
3464 }
3465 };
3466 let declared = matches!(
3467 request.kind,
3468 Some(subc_protocol::session::OperatorConfirmKind::Operation)
3469 ) && request.operation.as_ref().is_some_and(|operation| {
3470 registration
3471 .manifest
3472 .confirm_operations
3473 .as_ref()
3474 .is_some_and(|operations| operations.contains(operation))
3475 });
3476 let module_id = registration.manifest.module_id;
3477 let nonce = self
3481 .hello_launch_nonces
3482 .lock()
3483 .unwrap_or_else(|p| p.into_inner())
3484 .nonce(ctx.connection_id)
3485 .map(str::to_owned);
3486 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3487 self.supervisor
3488 .spawned_consumer_authorized(&module_id, nonce)
3489 });
3490 let confirms = self.forwarding.operator_confirms();
3491 self.forwarding
3492 .with_operator_route(
3493 ctx.connection_id,
3494 request.route_channel,
3495 request.route_epoch,
3496 |binding| {
3497 let shortcut = binding
3498 .filter(|binding| {
3499 nonce_proven
3500 && crate::operator_confirm::valid_summary(&request.summary)
3501 && declared
3502 && crate::operator_confirm::ShortcutContext::route_eligible(
3503 &self.operator_app_policy,
3504 &self.launch_table,
3505 binding,
3506 )
3507 })
3508 .map(|binding| {
3509 let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
3510 &self.operator_app_policy,
3511 binding,
3512 );
3513 let stamp = if phone.is_some() {
3514 None
3515 } else {
3516 binding.operator_app.clone()
3517 };
3518 self.shortcut_context(
3519 stamp,
3520 phone,
3521 Some((request.route_channel, request.route_epoch)),
3522 )
3523 });
3524 confirms.admit_with_shortcut(
3525 ctx,
3526 frame,
3527 module_id,
3528 nonce_proven,
3529 request,
3530 binding,
3531 shortcut,
3532 )
3533 },
3534 )
3535 .map_err(RouterError::Forwarding)
3536 }
3537
3538 fn shortcut_context(
3539 &self,
3540 stamp: Option<crate::operator_apps::OperatorAppStamp>,
3541 phone_fingerprint: Option<String>,
3542 route: Option<(u16, u32)>,
3543 ) -> crate::operator_confirm::ShortcutContext {
3544 crate::operator_confirm::ShortcutContext {
3545 ledger: self.activity.clone(),
3546 forwarding: self.forwarding.clone(),
3547 launches: self.launch_table.clone(),
3548 stamp,
3549 phone_fingerprint,
3550 route,
3551 connections: self.operator_apps.clone(),
3552 #[cfg(test)]
3553 hooks: Default::default(),
3554 }
3555 }
3556
3557 #[cfg(any(test, feature = "test-support"))]
3559 pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
3560 self.operator_app_policy.phone_enabled = enabled;
3561 self
3562 }
3563
3564 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3567 let mut scopes = ScopeRecords::from(scopes);
3568 for (index, record) in scopes.records.iter_mut().enumerate() {
3569 if let Some(refusal) = Self::canonicalize_workdir(record) {
3570 scopes.refusals.insert(index, refusal);
3571 }
3572 }
3573 scopes
3574 }
3575
3576 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3577 let workdir = record.workdir.as_mut()?;
3578 let path = std::path::Path::new(workdir);
3579 if !path.is_absolute() {
3580 return Some((
3581 error_codes::INVALID_WORKDIR,
3582 "workdir must be absolute".to_string(),
3583 ));
3584 }
3585 match ProjectRootId::from_path_allowing_missing(path) {
3586 Ok(root) => match root.as_path().to_str() {
3587 Some(canonical) => {
3588 *workdir = canonical.to_string();
3589 None
3590 }
3591 None => Some((
3592 error_codes::INVALID_WORKDIR,
3593 "canonical workdir is not UTF-8".to_string(),
3594 )),
3595 },
3596 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3597 }
3598 }
3599
3600 fn handle_scope_sync(
3604 &self,
3605 connection_id: ConnectionId,
3606 frame: Frame,
3607 generation: u64,
3608 scopes: Vec<ScopeRecord>,
3609 ) -> Result<Vec<Frame>, RouterError> {
3610 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3611 }
3612
3613 fn handle_scope_change(
3614 &self,
3615 connection_id: ConnectionId,
3616 frame: Frame,
3617 generation: u64,
3618 scopes: Vec<ScopeRecord>,
3619 end: Option<Vec<ScopeEnd>>,
3620 ) -> Result<Vec<Frame>, RouterError> {
3621 let is_apply = end.is_some();
3622 let Some(registration) = self
3623 .registry
3624 .get_module_by_connection(connection_id)
3625 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3626 else {
3627 return Ok(vec![control_error_frame(
3628 &frame,
3629 "not_registered",
3630 "scope.sync and scope.apply require an active module registration owned by this connection",
3631 )?]);
3632 };
3633 let owner = registration.manifest.module_id;
3634 let scopes = Self::prepare_scope_records(scopes);
3635 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3636 let is_current_launch = |connection: ConnectionId| {
3637 self.hello_launch_nonces
3638 .lock()
3639 .unwrap_or_else(|poisoned| poisoned.into_inner())
3640 .presented(connection, current_nonce.as_deref())
3641 };
3642 let mut table = self
3647 .scopes
3648 .write()
3649 .unwrap_or_else(|poisoned| poisoned.into_inner());
3650 let outcome = match end {
3651 Some(end) => table.apply(
3652 &owner,
3653 connection_id,
3654 is_current_launch,
3655 generation,
3656 scopes,
3657 end,
3658 ),
3659 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3660 };
3661 let drained = match &outcome {
3662 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3663 Err(_) => Vec::new(),
3664 };
3665 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3666 drop(table);
3667 if has_deadlines {
3668 self.scope_deadline_added.notify_one();
3669 }
3670 match outcome {
3671 Ok(applied) => {
3672 let counts = ScopeOutcomeCounts::of(&applied.results);
3673 info!(
3674 owner = %owner,
3675 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3676 generation,
3677 records = applied.results.len(),
3678 created = counts.created,
3679 replaced = counts.replaced,
3680 updated = counts.updated,
3681 unchanged = counts.unchanged,
3682 refused = counts.refused,
3683 ended = applied.ended.len(),
3684 tag_changes = applied.tag_changes.len(),
3685 routes_closed = drained.len(),
3686 "scope change accepted"
3687 );
3688 for refused in applied
3694 .results
3695 .iter()
3696 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3697 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3698 {
3699 warn!(
3700 owner = %owner,
3701 generation,
3702 scope_ref = %refused.scope_ref,
3703 scope_epoch = refused.scope_epoch,
3704 code = refused.code.as_deref().unwrap_or(""),
3705 "scope record refused"
3706 );
3707 }
3708 self.close_scope_drained_routes(drained);
3709 let response = if is_apply {
3710 ModuleControlResponseToModule::ScopeApply {
3711 generation,
3712 results: applied.results,
3713 end_results: applied.end_results,
3714 ended: applied.ended,
3715 }
3716 } else {
3717 ModuleControlResponseToModule::ScopeSync {
3718 generation,
3719 results: applied.results,
3720 ended: applied.ended,
3721 }
3722 };
3723 Ok(vec![control_response_body_frame(
3724 &frame,
3725 &response,
3726 "scope change response",
3727 )?])
3728 }
3729 Err(refusal) => {
3730 info!(
3731 owner = %owner,
3732 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3733 generation,
3734 code = refusal.code,
3735 "scope change refused"
3736 );
3737 Ok(vec![control_error_frame(
3738 &frame,
3739 refusal.code,
3740 refusal.message,
3741 )?])
3742 }
3743 }
3744 }
3745
3746 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3752 if drained.is_empty() {
3753 return;
3754 }
3755 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3756 BTreeMap::new();
3757 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3758 for route in drained {
3759 warn!(
3760 module_id = %route.module_id,
3761 reason = ?route.reason,
3762 client_connection_id = route.client.connection_id.get(),
3763 route_channel = route.client.channel,
3764 "closing route because its scope changed"
3765 );
3766 pushes
3767 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3768 .or_insert_with(|| (route.reason, Vec::new()))
3769 .1
3770 .push(EndpointRoute {
3771 goodbye_target: route.client.clone(),
3772 principal: Principal::Unverified,
3773 bound_at: Instant::now(),
3774 draining: false,
3775 drain_reason: None,
3776 });
3777 goodbyes.push(route.module);
3778 goodbyes.push(route.client);
3779 }
3780 for ((module_id, _), (reason, routes)) in pushes {
3781 send_route_control_pushes(
3782 &self.forwarding,
3783 routes,
3784 ClientControlPush::RouteClosed {
3785 module_id,
3786 channels: Vec::new(),
3787 reason,
3788 drained: false,
3789 abandoned: 0,
3790 excluded_subscriptions: 0,
3791 terminal: Some(false),
3792 },
3793 );
3794 }
3795 self.emit_route_goodbyes(goodbyes);
3796 }
3797
3798 fn handle_scope_describe(
3801 &self,
3802 connection_id: ConnectionId,
3803 frame: Frame,
3804 owner: Principal,
3805 scope_ref: String,
3806 ) -> Result<Vec<Frame>, RouterError> {
3807 let registered = self
3808 .registry
3809 .get_module_by_connection(connection_id)
3810 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3811 let Some(registered) = registered else {
3812 return Ok(vec![control_error_frame(
3813 &frame,
3814 "not_registered",
3815 "scope.describe requires an active module registration owned by this connection",
3816 )?]);
3817 };
3818 let mut description = self
3819 .scopes
3820 .read()
3821 .unwrap_or_else(|poisoned| poisoned.into_inner())
3822 .describe(&owner, &scope_ref);
3823 if !registered
3826 .manifest
3827 .capabilities
3828 .as_ref()
3829 .is_some_and(|caps| {
3830 caps.provides
3831 .iter()
3832 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3833 })
3834 {
3835 if let Some(stamp) = description.stamp.as_mut() {
3836 stamp.attributes.on_behalf_of_agent = None;
3837 }
3838 }
3839 if !registered
3840 .manifest
3841 .capabilities
3842 .as_ref()
3843 .is_some_and(|caps| {
3844 caps.provides
3845 .iter()
3846 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3847 })
3848 {
3849 if let Some(stamp) = description.stamp.as_mut() {
3850 stamp.attributes.flow_version = None;
3851 stamp.attributes.flow_code_hash = None;
3852 stamp.attributes.flow_install_id = None;
3853 }
3854 }
3855 let owner_configured = match &owner {
3856 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3861 _ => false,
3862 };
3863 let response = ModuleControlResponseToModule::ScopeDescribe {
3864 status: description.status,
3865 scope_epoch: description.scope_epoch,
3866 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3867 owner_synced: description.owner_synced,
3868 owner_configured,
3869 scope: description.stamp,
3870 };
3871 Ok(vec![control_response_body_frame(
3872 &frame,
3873 &response,
3874 "ModuleControlResponseToModule::ScopeDescribe",
3875 )?])
3876 }
3877
3878 fn handle_catalog_update(
3879 &self,
3880 connection_id: ConnectionId,
3881 frame: Frame,
3882 provides: Vec<ProviderRole>,
3883 capabilities: Option<CapabilityDeclarations>,
3884 ready: Option<bool>,
3885 ) -> Result<Vec<Frame>, RouterError> {
3886 self.refresh_capability_requirements();
3887 let Some(registration) = self
3888 .registry
3889 .get_module_by_connection(connection_id)
3890 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3891 else {
3892 return Ok(vec![control_error_frame(
3893 &frame,
3894 "not_registered",
3895 "catalog.update requires an active module registration owned by this connection",
3896 )?]);
3897 };
3898
3899 if let Some(message) =
3900 catalog_update_frozen_field_message(®istration.manifest, &provides)
3901 {
3902 return Ok(vec![control_error_frame(
3903 &frame,
3904 "catalog_update_frozen_field",
3905 message,
3906 )?]);
3907 }
3908
3909 let mut candidate = registration.manifest.clone();
3910 candidate.provides = provides.clone();
3911 candidate.capabilities = capabilities
3912 .clone()
3913 .or_else(|| registration.manifest.capabilities.clone());
3914 if let Some(rich_ui) = &candidate.rich_ui {
3915 if let Err(err) = rich_ui.validate() {
3916 return Ok(vec![control_error_frame(
3917 &frame,
3918 "invalid_manifest",
3919 err.to_string(),
3920 )?]);
3921 }
3922 }
3923 if let Err(err) = candidate.validate_capability_grammar() {
3924 return Ok(vec![control_error_frame(
3925 &frame,
3926 "invalid_capability_grammar",
3927 err.to_string(),
3928 )?]);
3929 }
3930
3931 let mut conflicts = self
3934 .capability_evaluator
3935 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3936 if let Some(conflict) = conflicts.first() {
3937 let message = format!(
3938 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3939 conflict.capability, conflict.claimants[0], candidate.module_id
3940 );
3941 for conflict in &mut conflicts {
3942 conflict.source = DuplicateClaimSource::CatalogUpdate;
3943 }
3944 log_duplicate_claim_events(conflicts);
3945 return Ok(vec![control_error_frame(
3946 &frame,
3947 "reserved_capability",
3948 message,
3949 )?]);
3950 }
3951
3952 let updated = self
3953 .registry
3954 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3955 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3956 if updated.is_none() {
3957 return Ok(vec![control_error_frame(
3958 &frame,
3959 "not_registered",
3960 "catalog.update requires an active module registration owned by this connection",
3961 )?]);
3962 }
3963 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3964 log_duplicate_claim_events(
3965 self.capability_evaluator
3966 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3967 );
3968 }
3969 if capability_census_trigger(
3970 registration.manifest.capabilities.as_ref(),
3971 updated
3972 .as_ref()
3973 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3974 ) {
3975 self.enforce_capability_denies();
3976 }
3977 self.refresh_capability_requirements();
3978
3979 let response = ModuleControlResponseToModule::CatalogUpdate {};
3980 control_response_body_frame(
3981 &frame,
3982 &response,
3983 "ModuleControlResponseToModule::CatalogUpdate",
3984 )
3985 .map(|frame| vec![frame])
3986 }
3987
3988 pub(crate) fn with_activity_ledger_config(
3989 mut self,
3990 run_dir: &Path,
3991 incarnation: String,
3992 config: crate::activity::ActivityConfig,
3993 ) -> Self {
3994 self.activity = Some(crate::activity::ActivityLedger::open_config(
3995 run_dir.join("activity.jsonl"),
3996 incarnation,
3997 config,
3998 ));
3999 self.supervisor.set_activity(self.activity.clone());
4000 self
4001 }
4002
4003 pub(crate) async fn configure_activity(
4004 &self,
4005 config: crate::activity::ActivityConfig,
4006 ) -> Result<(), &'static str> {
4007 match &self.activity {
4008 Some(ledger) => ledger.configure(config).await,
4009 None => Ok(()),
4010 }
4011 }
4012
4013 fn cancel_activity_subscription(
4014 &self,
4015 ctx: &RouteCtx,
4016 frame: &Frame,
4017 ) -> Result<bool, RouterError> {
4018 let task = self
4019 .activity_streams
4020 .lock()
4021 .unwrap_or_else(|p| p.into_inner())
4022 .remove(&(ctx.connection_id, frame.header.corr));
4023 let Some(task) = task else {
4024 return Ok(false);
4025 };
4026 task.abort();
4027 let end = Frame::build_with_version(
4028 response_version(frame),
4029 FrameType::StreamEnd,
4030 control_flags(),
4031 0,
4032 0,
4033 frame.header.corr,
4034 Vec::new(),
4035 )
4036 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
4037 let sink = ctx.egress.clone();
4038 tokio::spawn(async move {
4039 let _ = sink.send(end).await;
4040 });
4041 Ok(true)
4042 }
4043
4044 async fn handle_activity_read(
4045 &self,
4046 ctx: &RouteCtx,
4047 frame: Frame,
4048 ) -> Result<Vec<Frame>, RouterError> {
4049 use subc_control::{
4050 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
4051 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
4052 };
4053 #[derive(Deserialize)]
4054 struct Identity {
4055 #[serde(default)]
4056 consumer_identity: Option<ConsumerIdentity>,
4057 }
4058 let principal = match self
4062 .forwarding
4063 .module_id_for_connection(ctx.connection_id)
4064 .map_err(RouterError::Forwarding)?
4065 {
4066 Some(module_id) => Ok(Principal::Reserved { module_id }),
4067 None => {
4068 let identity: Identity = match serde_json::from_slice(&frame.body) {
4069 Ok(identity) => identity,
4070 Err(_) => {
4071 return Ok(vec![control_error_frame(
4072 &frame,
4073 ACTIVITY_MALFORMED,
4074 "malformed activity read",
4075 )?])
4076 }
4077 };
4078 self.route_open_principal(&frame, identity.consumer_identity)?
4079 }
4080 };
4081 let admitted = matches!(principal, Ok(Principal::Direct))
4082 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
4083 if !admitted {
4084 return Ok(vec![control_error_frame(
4085 &frame,
4086 ACTIVITY_READ_FORBIDDEN,
4087 "activity reads require an operator or trusted mirror principal",
4088 )?]);
4089 }
4090 let Some(ledger) = &self.activity else {
4091 return Ok(vec![control_error_frame(
4092 &frame,
4093 ACTIVITY_UNAVAILABLE,
4094 "activity ledger unavailable",
4095 )?]);
4096 };
4097 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
4098 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
4099 {
4100 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
4101 Ok(request) => ledger.query(request).await,
4102 Err(_) => Err(ACTIVITY_MALFORMED),
4103 };
4104 return match result {
4105 Ok(reply) => Ok(vec![control_response_body_frame(
4106 &frame,
4107 &reply,
4108 ops::ACTIVITY_QUERY,
4109 )?]),
4110 Err(code) => Ok(vec![control_error_frame(
4111 &frame,
4112 code,
4113 "activity query refused",
4114 )?]),
4115 };
4116 }
4117 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
4118 Ok(request) => request,
4119 Err(_) => {
4120 return Ok(vec![control_error_frame(
4121 &frame,
4122 ACTIVITY_MALFORMED,
4123 "malformed activity subscription",
4124 )?])
4125 }
4126 };
4127 let mut subscription = match ledger.subscribe(request.since).await {
4128 Ok(subscription) => subscription,
4129 Err(code) => {
4130 return Ok(vec![control_error_frame(
4131 &frame,
4132 code,
4133 "activity subscription refused",
4134 )?])
4135 }
4136 };
4137 let key = (ctx.connection_id, frame.header.corr);
4138 let sink = ctx.egress.clone();
4139 let mut streams = self
4140 .activity_streams
4141 .lock()
4142 .unwrap_or_else(|p| p.into_inner());
4143 streams.retain(|_, task| !task.is_finished());
4144 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
4145 return Ok(vec![control_error_frame(
4146 &frame,
4147 "duplicate_subscription",
4148 "activity subscription correlation id already in use",
4149 )?]);
4150 }
4151 let task = tokio::spawn(async move {
4152 loop {
4153 let entry = if let Some(entry) = subscription.replay.pop_front() {
4154 Some(entry)
4155 } else {
4156 subscription.receiver.recv().await
4157 };
4158 let Some(entry) = entry else {
4159 break;
4160 };
4161 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
4162 return;
4163 };
4164 let Ok(next) = Frame::build_with_version(
4165 response_version(&frame),
4166 FrameType::StreamData,
4167 control_flags(),
4168 0,
4169 0,
4170 frame.header.corr,
4171 body,
4172 ) else {
4173 return;
4174 };
4175 if sink.send(next).await.is_err() {
4176 return;
4177 }
4178 }
4179 if let Some(code) = subscription.terminal_code() {
4182 if let Ok(end) =
4183 control_error_frame(&frame, code, "activity subscriber fell behind")
4184 {
4185 let _ = sink.send(end).await;
4186 }
4187 }
4188 });
4189 streams.insert(key, task.abort_handle());
4190 Ok(Vec::new())
4191 }
4192
4193 async fn handle_activity_record(
4194 &self,
4195 ctx: &RouteCtx,
4196 frame: Frame,
4197 ) -> Result<Vec<Frame>, RouterError> {
4198 use subc_control::{
4199 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
4200 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
4201 };
4202 let result = async {
4203 let module = self
4204 .forwarding
4205 .module_id_for_connection(ctx.connection_id)
4206 .ok()
4207 .flatten()
4208 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
4209 let request: ActivityRecordRequest =
4210 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
4211 crate::activity::validate(&module, &request)?;
4212 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
4213 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
4214 let at_ms = std::time::SystemTime::now()
4215 .duration_since(std::time::UNIX_EPOCH)
4216 .unwrap_or_default()
4217 .as_millis() as u64;
4218 ledger.record(module, request, actor, at_ms).await
4219 }
4220 .await;
4221 match result {
4222 Ok(entry) => Ok(vec![control_response_body_frame(
4223 &frame,
4224 &ActivityRecordResponse { seq: entry.seq },
4225 "activity.record",
4226 )?]),
4227 Err(code) => Ok(vec![control_error_frame(
4228 &frame,
4229 code,
4230 "activity recording refused",
4231 )?]),
4232 }
4233 }
4234
4235 fn activity_actor(
4236 &self,
4237 connection: ConnectionId,
4238 module: &str,
4239 on: Option<&subc_control::ActivityOn>,
4240 ) -> Result<subc_control::ActivityActor, &'static str> {
4241 use crate::forwarding::{DataRoute, DataRouteState};
4242 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
4243 let principal = |id: &str| Principal::Reserved {
4244 module_id: id.to_owned(),
4245 };
4246 let mut actor = ActivityActor {
4247 principal: principal(module),
4248 scope_owner: None,
4249 agent_id: None,
4250 run_id: None,
4251 flow_id: None,
4252 project_root: None,
4253 workdir: None,
4254 project_id: None,
4255 };
4256 let scope = match on {
4257 None => return Ok(actor),
4258 Some(ActivityOn::Scope(scope)) => {
4259 let selector = subc_protocol::scope::ScopeSelector::new(
4260 principal(module),
4261 scope.scope_ref.clone(),
4262 Some(scope.epoch),
4263 );
4264 self.scopes
4265 .read()
4266 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4267 .attribute(&principal(module), module, &selector, true)
4268 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4269 }
4270 Some(ActivityOn::Route(route)) => {
4271 let DataRoute::Module(DataRouteState::Bound(binding)) = self
4272 .forwarding
4273 .lookup_data_route(connection, route.channel, route.epoch)
4274 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4275 else {
4276 return Err(ACTIVITY_ACTOR_UNKNOWN);
4277 };
4278 if binding.module_id != module {
4279 return Err(ACTIVITY_ACTOR_UNKNOWN);
4280 }
4281 actor.principal = binding.principal.clone();
4282 actor.project_root = binding
4283 .project_root
4284 .as_ref()
4285 .map(|root| root.as_path().to_path_buf());
4286 let Some(bound) = &binding.scope else {
4287 return Ok(actor);
4288 };
4289 let owner = principal(&bound.owner);
4290 let selector = subc_protocol::scope::ScopeSelector::new(
4291 owner.clone(),
4292 bound.scope_ref.clone(),
4293 Some(bound.tag.scope_epoch),
4294 );
4295 let admitted = self
4296 .scopes
4297 .read()
4298 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4299 .attribute(&owner, module, &selector, true)
4300 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4301 if admitted.tag != bound.tag {
4302 return Err(ACTIVITY_ACTOR_UNKNOWN);
4303 }
4304 admitted
4305 }
4306 };
4307 actor.scope_owner = Some(scope.stamp.owner);
4308 actor.agent_id = scope.stamp.attributes.agent_id;
4309 actor.run_id = scope.stamp.attributes.run_id;
4310 actor.flow_id = scope.stamp.attributes.flow_id;
4311 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4312 actor.project_id = scope.stamp.project_id;
4313 Ok(actor)
4314 }
4315
4316 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4317 self.refresh_capability_requirements();
4318 let mut counters = self.counters.snapshot();
4324 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4325 obj.insert(
4326 "activity_unparseable_lines".into(),
4327 ledger.read_errors().into(),
4328 );
4329 obj.insert(
4330 "activity_lifecycle_append_failures".into(),
4331 ledger.lifecycle_failures().into(),
4332 );
4333 obj.insert(
4334 "activity_started_digest_skips".into(),
4335 ledger.lifecycle_digest_skips().into(),
4336 );
4337 }
4338 if let (Ok((connections_with_routes, max)), Some(obj)) = (
4339 self.forwarding.client_route_concentration(),
4340 counters.as_object_mut(),
4341 ) {
4342 obj.insert(
4343 "client_connections_with_routes".into(),
4344 connections_with_routes.into(),
4345 );
4346 obj.insert("max_routes_on_one_connection".into(), max.into());
4347 }
4348 if let (Some(open_breakers), Some(obj)) = (
4354 self.route_bind_breakers.open_snapshot(),
4355 counters.as_object_mut(),
4356 ) {
4357 obj.insert("route_bind_breakers_open".into(), open_breakers);
4358 }
4359 if let (Some(reason), Some(obj)) = (
4364 self.local_endpoint_unavailable.clone(),
4365 counters.as_object_mut(),
4366 ) {
4367 obj.insert("local_endpoint_unavailable".into(), reason.into());
4368 }
4369 let response = ClientControlResponse::ServerDescribe {
4370 protocol_ver: PROTOCOL_VERSION,
4371 subc_ops: subc_ops(),
4372 capabilities: self.describe_capabilities(),
4373 connected_clients: self.connected_clients.count(),
4374 counters: Some(counters),
4375 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4376 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4377 capability_requirements: self.capability_requirement_statuses(),
4378 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4379 config_path: self
4380 .rescan
4381 .as_ref()
4382 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4383 };
4384 Ok(vec![control_response_body_frame(
4385 &frame,
4386 &response,
4387 "ClientControlResponse::ServerDescribe",
4388 )?])
4389 }
4390
4391 fn handle_catalog_list(
4392 &self,
4393 frame: Frame,
4394 module_id: Option<String>,
4395 ) -> Result<Vec<Frame>, RouterError> {
4396 let (generation, modules) = self
4397 .registry
4398 .list_modules_with_nonce_verification()
4399 .map_err(|err| {
4400 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4401 })?;
4402 let entries = modules
4403 .into_iter()
4404 .filter(|(registration, _)| {
4405 module_id
4406 .as_deref()
4407 .map(|wanted| registration.manifest.module_id == wanted)
4408 .unwrap_or(true)
4409 })
4410 .map(|(registration, nonce_verified)| {
4411 let not_ready = self.not_ready_reason(®istration);
4412 let roles = registration.manifest.provides;
4413 CatalogEntry::new(
4414 registration.manifest.module_id,
4415 roles,
4416 registration.control_ops,
4417 )
4418 .with_ready(not_ready.is_none())
4419 .with_not_ready(not_ready)
4420 .with_module_version(Some(registration.manifest.module_version))
4421 .with_provenance(registration.manifest.provenance)
4422 .with_capabilities(registration.manifest.capabilities)
4423 .with_self_signals(registration.manifest.self_signals)
4424 .with_events(registration.manifest.events)
4425 .with_operation_delivery(registration.manifest.operation_delivery)
4426 .with_operation_payload(registration.manifest.operation_payload)
4427 .with_flow_operations(registration.manifest.flow_operations)
4428 .with_confirm_operations(registration.manifest.confirm_operations)
4429 .with_rich_ui(registration.manifest.rich_ui)
4430 .with_role_declarations(registration.manifest.role_declarations)
4431 .with_nonce_verified(nonce_verified)
4432 })
4433 .collect();
4434 let response = ClientControlResponse::CatalogList {
4435 generation,
4436 modules: entries,
4437 subc_ops: subc_ops(),
4438 };
4439 Ok(vec![control_response_body_frame(
4440 &frame,
4441 &response,
4442 "ClientControlResponse::CatalogList",
4443 )?])
4444 }
4445
4446 fn attest_route_open_consumer(
4451 &self,
4452 consumer_identity: Option<&ConsumerIdentity>,
4453 ) -> Option<Principal> {
4454 let Some(consumer_identity) = consumer_identity else {
4455 return Some(Principal::Direct);
4456 };
4457 self.supervisor
4458 .spawned_consumer_authorized(
4459 &consumer_identity.module_id,
4460 &consumer_identity.launch_nonce,
4461 )
4462 .then(|| Principal::Reserved {
4463 module_id: consumer_identity.module_id.clone(),
4464 })
4465 }
4466
4467 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4471 let Ok(ClientControlRequest::RouteOpen {
4472 identity,
4473 consumer_identity,
4474 scope,
4475 ..
4476 }) = parse_client_control_request(&frame.body)
4477 else {
4478 return RouteOpenCaller::unknown();
4479 };
4480 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4481 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4482 }
4483
4484 fn route_open_principal(
4485 &self,
4486 frame: &Frame,
4487 consumer_identity: Option<ConsumerIdentity>,
4488 ) -> Result<Result<Principal, Frame>, RouterError> {
4489 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4490 self.route_open_principal_attested(frame, consumer_identity, attested)
4491 }
4492
4493 fn route_open_principal_attested(
4497 &self,
4498 frame: &Frame,
4499 consumer_identity: Option<ConsumerIdentity>,
4500 attested: Option<Principal>,
4501 ) -> Result<Result<Principal, Frame>, RouterError> {
4502 if let Some(principal) = attested {
4503 return Ok(Ok(principal));
4504 }
4505 let Some(consumer_identity) = consumer_identity else {
4506 return Ok(Ok(Principal::Direct));
4507 };
4508
4509 Ok(Err(control_error_frame(
4510 frame,
4511 "bad_consumer_identity",
4512 format!(
4513 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4514 consumer_identity.module_id
4515 ),
4516 )?))
4517 }
4518
4519 #[allow(clippy::too_many_arguments)]
4526 fn route_open_refusal_frame(
4527 &self,
4528 ctx: &RouteCtx,
4529 caller: &RouteOpenCaller,
4530 frame: &Frame,
4531 module_id: &str,
4532 reason: &'static str,
4533 code: &'static str,
4534 message: impl Into<String>,
4535 ) -> Result<Frame, RouterError> {
4536 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4537 control_error_frame(frame, code, message.into())
4538 }
4539
4540 #[allow(clippy::too_many_arguments)]
4549 fn route_bind_rejection(
4550 &self,
4551 ctx: &RouteCtx,
4552 caller: &RouteOpenCaller,
4553 frame: &Frame,
4554 target_module_id: &str,
4555 body: ErrorBody,
4556 daemon_authored: bool,
4557 ) -> Result<Vec<Frame>, RouterError> {
4558 if daemon_authored {
4564 let scope_code = match body.code.as_str() {
4565 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4566 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4567 _ => None,
4568 };
4569 if let Some(code) = scope_code {
4570 self.observe_route_open_refusal(
4571 ctx,
4572 caller,
4573 target_module_id,
4574 "scope_changed_before_commit",
4575 code,
4576 );
4577 return Ok(vec![control_error_body_frame(frame, body)?]);
4578 }
4579 }
4580 self.counters
4581 .increment_route_open_refused("module_rejected");
4582 info!(
4583 target: "control",
4584 code = "module_rejected",
4585 module_code = ?body.code,
4586 module_id = ?target_module_id,
4587 connection_id = ctx.connection_id.get(),
4588 principal = %caller.principal,
4589 scope = %caller.scope,
4590 harness = %caller.harness,
4591 "route.open refused"
4592 );
4593 let refusal = if daemon_authored {
4594 control_error_body_frame(frame, body)?
4595 } else {
4596 control_relayed_error_body_frame(frame, body)?
4597 };
4598 Ok(vec![refusal])
4599 }
4600
4601 #[allow(clippy::too_many_arguments)]
4619 fn route_open_breaker_refusal_frame(
4620 &self,
4621 ctx: &RouteCtx,
4622 caller: &RouteOpenCaller,
4623 frame: &Frame,
4624 module_id: &str,
4625 consecutive_timeouts: u32,
4626 retry_in: Duration,
4627 probe_in_flight: bool,
4628 ) -> Result<Frame, RouterError> {
4629 self.counters
4630 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4631 debug!(
4632 target: "control",
4633 code = "module_timeout",
4634 module_id = ?module_id,
4635 connection_id = ctx.connection_id.get(),
4636 principal = %caller.principal,
4637 scope = %caller.scope,
4638 harness = %caller.harness,
4639 consecutive_timeouts,
4640 retry_in_ms = retry_in.as_millis() as u64,
4641 probe_in_flight,
4642 "route.open refused by open bind-relay breaker"
4643 );
4644 let detail = if probe_in_flight {
4651 "one test route is already being tried; retry once it settles".to_string()
4652 } else {
4653 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4654 };
4655 control_error_frame(
4656 frame,
4657 "module_timeout",
4658 format!(
4659 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4660 timed out in a row); {detail}; its established routes are unaffected"
4661 ),
4662 )
4663 }
4664
4665 fn observe_route_open_refusal(
4678 &self,
4679 ctx: &RouteCtx,
4680 caller: &RouteOpenCaller,
4681 module_id: &str,
4682 reason: &'static str,
4683 code: &'static str,
4684 ) {
4685 self.counters.increment_route_open_refused(code);
4686 info!(
4687 target: "control",
4688 code,
4689 reason,
4690 module_id = ?module_id,
4691 connection_id = ctx.connection_id.get(),
4692 principal = %caller.principal,
4693 scope = %caller.scope,
4694 harness = %caller.harness,
4695 "route.open refused"
4696 );
4697 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4698 self.route_outages.record_not_serving(module_id, reason);
4699 }
4700 }
4701
4702 fn observe_route_open_accept(
4747 &self,
4748 ctx: &RouteCtx,
4749 module_id: &str,
4750 principal: &str,
4751 scope: &str,
4752 ) {
4753 self.route_outages.record_accepted(module_id);
4754 self.counters.increment_route_open_accepted(principal);
4755 info!(
4756 target: "control",
4757 principal,
4758 module_id,
4759 connection_id = ctx.connection_id.get(),
4760 scope = %scope,
4761 "route.open accepted"
4762 );
4763 }
4764
4765 fn supervised_absent_route_open_refusal_frame(
4766 &self,
4767 ctx: &RouteCtx,
4768 caller: &RouteOpenCaller,
4769 frame: &Frame,
4770 module_id: &str,
4771 code: &'static str,
4772 status: &crate::supervise::ModuleStatus,
4773 ) -> Result<Frame, RouterError> {
4774 self.counters.increment_route_open_refused(code);
4775 info!(
4776 target: "control",
4777 code,
4778 reason = "supervised_not_registered",
4779 module_id = ?module_id,
4780 connection_id = ctx.connection_id.get(),
4781 principal = %caller.principal,
4782 scope = %caller.scope,
4783 harness = %caller.harness,
4784 state = %status.state,
4785 enabled = status.enabled,
4786 live = status.live,
4787 "route.open refused"
4788 );
4789 self.route_outages
4793 .record_not_serving(module_id, "supervised_not_registered");
4794 let message = if let Some(pid) = status.waiting_for_previous_pid {
4795 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4796 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4797 })
4798 } else {
4799 format!(
4800 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4801 status.state, status.enabled, status.live
4802 )
4803 };
4804 control_error_frame(frame, code, message)
4805 }
4806
4807 async fn handle_route_open(
4808 &self,
4809 ctx: &RouteCtx,
4810 frame: Frame,
4811 request: RouteOpenRequest,
4812 timings: &mut RouteOpenTimings,
4813 ) -> Result<Vec<Frame>, RouterError> {
4814 let RouteOpenRequest {
4815 target,
4816 mut identity,
4817 consumer_identity,
4818 consumer_capabilities,
4819 role_versions,
4820 busy_refusals,
4821 admission_facts,
4822 scope,
4823 } = request;
4824 let target_module_id = target_module_id(&target).to_string();
4825 timings.module_id.clone_from(&target_module_id);
4826 if self
4827 .registry
4828 .get_module_by_connection(ctx.connection_id)
4829 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4830 .is_some()
4831 {
4832 return Ok(vec![control_error_frame(
4833 &frame,
4834 "invalid_request",
4835 "module connections cannot open client routes",
4836 )?]);
4837 }
4838 debug!(
4839 connection_id = ctx.connection_id.get(),
4840 corr = frame.header.corr,
4841 module_id = %target_module_id,
4842 "handling route.open"
4843 );
4844
4845 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4850 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4851
4852 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4858 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4859 self.observe_route_open_refusal(
4860 ctx,
4861 &caller,
4862 &target_module_id,
4863 "invalid_role_versions",
4864 error_codes::INVALID_REQUEST,
4865 );
4866 return Ok(vec![control_error_body_frame(
4867 &frame,
4868 ErrorBody {
4869 code: error_codes::INVALID_REQUEST.to_string(),
4870 message: error.to_string(),
4871 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4872 },
4873 )?]);
4874 }
4875
4876 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4877 if status.waiting_for_previous_pid.is_some() {
4878 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4879 ctx,
4880 &caller,
4881 &frame,
4882 &target_module_id,
4883 "module_warming",
4884 &status,
4885 )?]);
4886 }
4887 }
4888
4889 let Some(registration) = self
4908 .registry
4909 .get_module(&target_module_id)
4910 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4911 else {
4912 if let Some((status, warming)) =
4913 self.supervisor_status(&target_module_id, frame.header.corr)?
4914 {
4915 if status.protocol == ModuleProtocol::None {
4922 return Ok(vec![self.route_open_refusal_frame(
4923 ctx,
4924 &caller,
4925 &frame,
4926 &target_module_id,
4927 "protocol_none",
4928 error_codes::MODULE_NO_PROTOCOL,
4929 format!(
4930 "module_id '{target_module_id}' is declared protocol: none; \
4931 it speaks no subc wire and serves no routes"
4932 ),
4933 )?]);
4934 }
4935 let code = if warming {
4936 "module_warming"
4937 } else {
4938 "target_unavailable"
4939 };
4940 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4941 ctx,
4942 &caller,
4943 &frame,
4944 &target_module_id,
4945 code,
4946 &status,
4947 )?]);
4948 }
4949 if let Some(removed_ago_ms) =
4950 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4951 {
4952 return Ok(vec![self.route_open_refusal_frame(
4953 ctx,
4954 &caller,
4955 &frame,
4956 &target_module_id,
4957 "removed",
4958 error_codes::MODULE_REMOVED,
4959 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4960 )?]);
4961 }
4962 return Ok(vec![self.route_open_refusal_frame(
4963 ctx,
4964 &caller,
4965 &frame,
4966 &target_module_id,
4967 "not_registered",
4968 error_codes::UNKNOWN_MODULE,
4969 format!("module_id '{target_module_id}' is not registered"),
4970 )?]);
4971 };
4972
4973 if !registration.ready {
4977 self.counters
4978 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4979 info!(
4980 target: "control",
4981 code = error_codes::MODULE_WARMING,
4982 module_id = ?target_module_id,
4983 connection_id = ctx.connection_id.get(),
4984 principal = %caller.principal,
4985 scope = %caller.scope,
4986 harness = %caller.harness,
4987 reason = "declared_not_ready",
4988 "route.open refused"
4989 );
4990 self.route_outages
4993 .record_not_serving(&target_module_id, "declared_not_ready");
4994 return Ok(vec![control_error_body_frame(
4995 &frame,
4996 ErrorBody {
4997 code: error_codes::MODULE_WARMING.to_string(),
4998 message: format!(
4999 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
5000 ),
5001 detail: Some(serde_json::json!({
5002 "reason": "declared_not_ready"
5003 })),
5004 },
5005 )?]);
5006 }
5007
5008 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
5027 self.counters
5028 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
5029 info!(
5030 target: "control",
5031 code = error_codes::MODULE_WARMING,
5032 module_id = ?target_module_id,
5033 connection_id = ctx.connection_id.get(),
5034 principal = %caller.principal,
5035 scope = %caller.scope,
5036 harness = %caller.harness,
5037 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5038 capability = %capability,
5039 "route.open refused"
5040 );
5041 return Ok(vec![control_error_body_frame(
5042 &frame,
5043 ErrorBody {
5044 code: error_codes::MODULE_WARMING.to_string(),
5045 message: format!(
5046 "module_id '{target_module_id}' requires capability '{capability}', \
5047 which no registered module provides; retry"
5048 ),
5049 detail: Some(serde_json::json!({
5050 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5051 "capability": capability,
5052 })),
5053 },
5054 )?]);
5055 }
5056
5057 if !target_has_required_role(&target, ®istration.manifest.provides) {
5058 return Ok(vec![self.route_open_refusal_frame(
5059 ctx,
5060 &caller,
5061 &frame,
5062 &target_module_id,
5063 "role_not_provided",
5064 "target_unavailable",
5065 format!("module_id '{target_module_id}' does not provide the requested target"),
5066 )?]);
5067 }
5068
5069 if registration.state != ChannelState::Active {
5070 return Ok(vec![self.route_open_refusal_frame(
5071 ctx,
5072 &caller,
5073 &frame,
5074 &target_module_id,
5075 "registration_not_active",
5076 "target_unavailable",
5077 format!("module_id '{target_module_id}' is not active"),
5078 )?]);
5079 }
5080
5081 if self
5082 .forwarding
5083 .module_is_draining(&target_module_id)
5084 .map_err(RouterError::Forwarding)?
5085 {
5086 return Ok(vec![self.route_open_refusal_frame(
5087 ctx,
5088 &caller,
5089 &frame,
5090 &target_module_id,
5091 "reloading",
5092 "module_reloading",
5093 format!("module_id '{target_module_id}' is reloading"),
5094 )?]);
5095 }
5096
5097 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
5098 process_liveness.process_live(&target_module_id) == Some(false)
5099 }) {
5100 if process_liveness.process_replacing(&target_module_id) {
5107 return Ok(vec![self.route_open_refusal_frame(
5108 ctx,
5109 &caller,
5110 &frame,
5111 &target_module_id,
5112 "reloading",
5113 "module_reloading",
5114 format!("module_id '{target_module_id}' is reloading"),
5115 )?]);
5116 }
5117 return Ok(vec![self.route_open_refusal_frame(
5118 ctx,
5119 &caller,
5120 &frame,
5121 &target_module_id,
5122 "supervisor_not_live",
5123 "target_unavailable",
5124 format!("module_id '{target_module_id}' is not live"),
5125 )?]);
5126 }
5127
5128 if !self
5129 .forwarding
5130 .has_live_module_connection(&target_module_id)
5131 .map_err(RouterError::Forwarding)?
5132 {
5133 return Ok(vec![self.route_open_refusal_frame(
5134 ctx,
5135 &caller,
5136 &frame,
5137 &target_module_id,
5138 "no_forwarding_connection",
5139 "target_unavailable",
5140 format!("module_id '{target_module_id}' has no live forwarding connection"),
5141 )?]);
5142 }
5143
5144 if let Some(error) =
5145 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
5146 {
5147 self.observe_route_open_refusal(
5148 ctx,
5149 &caller,
5150 &target_module_id,
5151 "op_not_allowed",
5152 "op_not_allowed",
5153 );
5154 return Ok(vec![error]);
5155 }
5156
5157 let principal =
5158 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
5159 Ok(principal) => principal,
5160 Err(error) => {
5161 self.observe_route_open_refusal(
5162 ctx,
5163 &caller,
5164 &target_module_id,
5165 "bad_consumer_identity",
5166 "bad_consumer_identity",
5167 );
5168 return Ok(vec![error]);
5169 }
5170 };
5171 timings.principal = principal_label(&principal);
5172
5173 let build_class = identity.build_class.take().filter(|build_class| {
5176 if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
5177 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
5178 } else {
5179 if self
5180 .build_class_warned
5181 .lock()
5182 .unwrap_or_else(|p| p.into_inner())
5183 .insert(ctx.connection_id)
5184 {
5185 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
5186 "ignoring build_class from a non-Callosum opener");
5187 }
5188 false
5189 }
5190 });
5191
5192 if let Principal::Reserved {
5196 module_id: opening_module_id,
5197 } = &principal
5198 {
5199 if let Some(opening_registration) = self
5200 .registry
5201 .get_module(opening_module_id)
5202 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5203 {
5204 if let Some(capability) =
5205 denied_capability(&opening_registration.manifest, ®istration.manifest)
5206 {
5207 warn!(
5208 opening_module_id,
5209 target_module_id,
5210 capability,
5211 "refusing route.open because an attested capability deny edge matches"
5212 );
5213 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5214 &frame,
5215 &target_module_id,
5216 "capability_deny_edge",
5217 "capability_forbidden",
5218 format!(
5219 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
5220 ),
5221 )?]);
5222 }
5223 }
5224 }
5225
5226 if admission_facts.is_some() {
5227 let carrier_matches = matches!(
5228 &principal,
5229 Principal::Reserved { module_id }
5230 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
5231 );
5232 if !carrier_matches {
5233 return Ok(vec![self.route_open_refusal_frame(
5234 ctx,
5235 &caller,
5236 &frame,
5237 &target_module_id,
5238 "admission_facts_carrier_not_permitted",
5239 "admission_facts_not_permitted",
5240 "admission facts may only be carried by the configured reserved module",
5241 )?]);
5242 }
5243
5244 let target_allowed = self
5245 .admission_facts_targets
5246 .as_ref()
5247 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
5248 if !target_allowed {
5249 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5250 &frame,
5251 &target_module_id,
5252 "admission_facts_target_not_listed",
5253 "admission_facts_target_not_allowed",
5254 format!(
5255 "admission facts are not permitted for target module_id '{target_module_id}'"
5256 ),
5257 )?]);
5258 }
5259
5260 }
5264
5265 let (bound_scope, mut scope_stamp) = match &scope {
5271 None => (None, None),
5272 Some(selector) => {
5273 let owner_configured = match &selector.owner {
5274 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5280 _ => false,
5281 };
5282 let mut scopes = self
5283 .scopes
5284 .write()
5285 .unwrap_or_else(|poisoned| poisoned.into_inner());
5286 let admitted = scopes
5287 .admit(&principal, &target_module_id, selector, owner_configured)
5288 .and_then(|admission| {
5289 crate::scopes::check_target_flow_support(
5290 &admission.stamp,
5291 &target_module_id,
5292 registration.manifest.capabilities.as_ref(),
5293 )?;
5294 crate::scopes::check_target_agent_run_support(
5295 &admission.stamp,
5296 &target_module_id,
5297 registration.manifest.capabilities.as_ref(),
5298 )?;
5299 let bound = scopes.reserve_bind(
5300 &admission,
5301 &principal,
5302 ctx.connection_id,
5303 selector.hold_presence,
5304 );
5305 Ok((admission, bound))
5306 });
5307 match admitted {
5308 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
5309 Err(refusal) => {
5310 return Ok(vec![self.route_open_refusal_frame(
5311 ctx,
5312 &caller,
5313 &frame,
5314 &target_module_id,
5315 refusal.code,
5316 refusal.code,
5317 refusal.message,
5318 )?]);
5319 }
5320 }
5321 }
5322 };
5323 if let Some(stamp) = &scope_stamp {
5326 caller.scope = route_open_scope_label(stamp);
5327 }
5328
5329 timings.enter(RouteOpenPhase::ProjectRoot);
5337 let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
5338 Ok(project) => project,
5339 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5340 };
5341 timings.enter(RouteOpenPhase::Admission);
5342 let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
5343 Ok(root) => root,
5344 Err(err) => {
5345 return Ok(vec![control_error_frame(
5346 &frame,
5347 "invalid_project_root",
5348 err.to_string(),
5349 )?])
5350 }
5351 };
5352 let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
5358
5359 let _concurrency_guard = match self
5370 .route_bind_concurrency
5371 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5372 {
5373 Ok(guard) => guard,
5374 Err(in_flight) => {
5375 return Ok(vec![self.route_open_target_capacity_refusal(
5376 ctx,
5377 &caller,
5378 &frame,
5379 &target_module_id,
5380 in_flight,
5381 )?]);
5382 }
5383 };
5384
5385 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5388 RouteBindAdmission::Admitted { guard, probe } => {
5389 if probe {
5390 info!(
5391 module_id = %target_module_id,
5392 connection_id = ctx.connection_id.get(),
5393 "route.bind breaker half-open: admitting one probe"
5394 );
5395 }
5396 guard
5397 }
5398 RouteBindAdmission::Refused {
5399 consecutive_timeouts,
5400 retry_in,
5401 probe_in_flight,
5402 } => {
5403 return Ok(vec![self.route_open_breaker_refusal_frame(
5404 ctx,
5405 &caller,
5406 &frame,
5407 &target_module_id,
5408 consecutive_timeouts,
5409 retry_in,
5410 probe_in_flight,
5411 )?]);
5412 }
5413 };
5414
5415 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5420 let relay_deadline = Instant::now() + route_bind_relay_timeout;
5421 let operator_app = self
5422 .operator_apps
5423 .lock()
5424 .unwrap_or_else(|p| p.into_inner())
5425 .get(&ctx.connection_id)
5426 .cloned();
5427 timings.enter(RouteOpenPhase::EgressReservation);
5428 let pending = match self
5429 .forwarding
5430 .begin_route_bind_relay_for(
5431 ctx.connection_id,
5432 ctx.egress.clone(),
5433 response_version(&frame),
5434 frame.header.corr,
5435 &target_module_id,
5436 principal.clone(),
5437 bound_scope,
5438 project_root,
5439 identity.harness.clone(),
5440 build_class,
5441 operator_app.clone(),
5442 relay_deadline,
5443 busy_refusals == Some(true),
5444 )
5445 .await
5446 {
5447 Ok(pending) => pending,
5448 Err(err) => {
5449 return Ok(vec![self.route_open_refusal_frame(
5450 ctx,
5451 &caller,
5452 &frame,
5453 &target_module_id,
5454 "relay_reservation_failed",
5455 forwarding_error_code(&err),
5456 err.to_string(),
5457 )?])
5458 }
5459 };
5460 timings.enter(RouteOpenPhase::Admission);
5461 let crate::forwarding::PendingRouteBindRelay {
5462 endpoint,
5463 module_sink,
5464 negotiated_ver,
5465 client_channel,
5466 client_epoch,
5467 module_channel,
5468 module_epoch,
5469 corr: relay_corr,
5470 receiver,
5471 } = pending;
5472 let mut reservation =
5473 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5474
5475 let relay_registration = self
5480 .registry
5481 .get_module_by_connection(endpoint.connection_id)
5482 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5483 let relay_capabilities = relay_registration
5484 .as_ref()
5485 .and_then(|registration| registration.manifest.capabilities.as_ref());
5486 if let Some(selector) = &scope {
5487 if let Err(refusal) = self
5490 .scopes
5491 .read()
5492 .unwrap_or_else(|poisoned| poisoned.into_inner())
5493 .check_bind_proof(&principal, selector)
5494 {
5495 reservation.release_and_disarm();
5496 return Ok(vec![self.route_open_refusal_frame(
5497 ctx,
5498 &caller,
5499 &frame,
5500 &target_module_id,
5501 refusal.code,
5502 refusal.code,
5503 refusal.message,
5504 )?]);
5505 }
5506 }
5507 if let Some(stamp) = scope_stamp
5508 .as_ref()
5509 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5510 {
5511 if let Err(refusal) = crate::scopes::check_target_flow_support(
5512 stamp,
5513 &target_module_id,
5514 relay_capabilities,
5515 )
5516 .and_then(|()| {
5517 crate::scopes::check_target_agent_run_support(
5518 stamp,
5519 &target_module_id,
5520 relay_capabilities,
5521 )
5522 }) {
5523 reservation.release_and_disarm();
5524 return Ok(vec![self.route_open_refusal_frame(
5525 ctx,
5526 &caller,
5527 &frame,
5528 &target_module_id,
5529 refusal.code,
5530 refusal.code,
5531 refusal.message,
5532 )?]);
5533 }
5534 }
5535
5536 if !relay_capabilities.is_some_and(|caps| {
5539 caps.provides
5540 .iter()
5541 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5542 }) {
5543 if let Some(stamp) = scope_stamp.as_mut() {
5544 stamp.workdir = None;
5545 stamp.project_id = None;
5546 }
5547 }
5548 let worker_run = scope_stamp
5553 .as_ref()
5554 .is_some_and(crate::scopes::is_worker_run)
5555 && relay_capabilities.is_some_and(|caps| {
5556 caps.provides
5557 .iter()
5558 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5559 });
5560 if !worker_run
5561 && !relay_capabilities.is_some_and(|caps| {
5562 caps.provides
5563 .iter()
5564 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5565 })
5566 {
5567 if let Some(stamp) = scope_stamp.as_mut() {
5568 stamp.attributes.on_behalf_of_agent = None;
5569 }
5570 }
5571 if !relay_capabilities.is_some_and(|caps| {
5572 caps.provides
5573 .iter()
5574 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5575 }) {
5576 if let Some(stamp) = scope_stamp.as_mut() {
5577 stamp.attributes.flow_version = None;
5578 stamp.attributes.flow_code_hash = None;
5579 stamp.attributes.flow_install_id = None;
5580 }
5581 }
5582 let call_mode = if worker_run
5583 || relay_capabilities
5584 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5585 {
5586 Some(derive_call_mode(scope_stamp.as_ref()))
5587 } else {
5588 if let Some(stamp) = scope_stamp.as_mut() {
5589 stamp.expires_at_ms = None;
5590 }
5591 None
5592 };
5593
5594 debug!(
5595 connection_id = ctx.connection_id.get(),
5596 client_channel,
5597 client_epoch,
5598 module_channel,
5599 module_epoch,
5600 "reserved route handle pair"
5601 );
5602 let route_principal_label = principal_label(&principal);
5605 let relay = ModuleControlRequest::RouteBind {
5606 route_channel: module_channel,
5607 epoch: module_epoch,
5608 target,
5609 identity,
5610 principal: Some(principal),
5611 consumer_capabilities,
5612 role_versions,
5613 admission_facts,
5614 scope: scope_stamp,
5615 call_mode,
5616 };
5617 let peer = RouteBindPeer::new()
5618 .with_observed_peer(
5619 relay_capabilities
5620 .is_some_and(|caps| {
5621 caps.provides
5622 .iter()
5623 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5624 })
5625 .then(|| {
5626 self.observed_peers
5627 .lock()
5628 .unwrap_or_else(|p| p.into_inner())
5629 .get(&ctx.connection_id)
5630 .cloned()
5631 .unwrap_or_else(ObservedPeer::unobserved)
5632 }),
5633 )
5634 .with_operator_app(
5635 self.connection_operator_app(operator_app.as_ref())
5636 .filter(|_| {
5637 relay_capabilities.is_some_and(|caps| {
5638 caps.provides
5639 .iter()
5640 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5641 })
5642 }),
5643 );
5644 let relay_body = crate::bind_compat::encode_bind(
5645 &relay,
5646 &peer,
5647 &legacy_bind_project,
5648 relay_capabilities,
5649 )
5650 .map_err(|err| {
5651 RouterError::backend(
5652 0,
5653 frame.header.corr,
5654 format!("failed to encode route.bind request: {err}"),
5655 )
5656 })?;
5657 let relay_frame = Frame::build_with_version(
5658 negotiated_ver,
5659 FrameType::Request,
5660 control_flags(),
5661 0,
5662 0,
5663 relay_corr,
5664 relay_body,
5665 )
5666 .map_err(RouterError::FrameBuild)?;
5667
5668 timings.enter(RouteOpenPhase::BindSend);
5669 let sent = module_sink.send(relay_frame).await;
5670 timings.enter(RouteOpenPhase::Admission);
5671 if let Err(err) = sent {
5672 reservation.release_and_disarm();
5673 return Ok(vec![self.route_open_refusal_frame(
5674 ctx,
5675 &caller,
5676 &frame,
5677 &target_module_id,
5678 "relay_send_failed",
5679 "target_unavailable",
5680 err.to_string(),
5681 )?]);
5682 }
5683
5684 #[cfg(test)]
5685 if let Some(hold) = &self.route_bind_relay_sent_hold {
5686 hold.wait().await;
5690 hold.wait().await;
5691 }
5692
5693 if self
5698 .forwarding
5699 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5700 .map_err(RouterError::Forwarding)?
5701 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5702 {
5703 self.send_abandoned_route_bind_goodbye(
5704 &module_sink,
5705 negotiated_ver,
5706 module_channel,
5707 module_epoch,
5708 );
5709 }
5710
5711 timings.enter(RouteOpenPhase::BindAckWait);
5712 let outcome = timeout_at(relay_deadline, receiver).await;
5713 timings.enter(RouteOpenPhase::Admission);
5714 match outcome {
5715 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5716 reservation.disarm();
5717 if breaker.record_accepted() {
5718 info!(
5719 module_id = %target_module_id,
5720 "route.bind breaker closed: the probe was accepted"
5721 );
5722 }
5723 self.observe_route_open_accept(
5724 ctx,
5725 &target_module_id,
5726 &route_principal_label,
5727 &caller.scope,
5728 );
5729 Ok(Vec::new())
5730 }
5731 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5732 reservation.release_and_disarm();
5733 breaker.record_inconclusive();
5737 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5738 }
5739 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5740 reservation.release_and_disarm();
5741 breaker.record_inconclusive();
5742 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5743 }
5744 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5745 reservation.release_and_disarm();
5746 breaker.record_inconclusive();
5747 tracing::warn!(
5754 module_id = %target_module_id,
5755 "route.bind relay abandoned: {message}"
5756 );
5757 Ok(vec![self.route_open_refusal_frame(
5758 ctx,
5759 &caller,
5760 &frame,
5761 &target_module_id,
5762 "relay_abandoned",
5763 "target_unavailable",
5764 message,
5765 )?])
5766 }
5767 Ok(Err(_)) => {
5768 reservation.release_and_disarm();
5769 breaker.record_inconclusive();
5770 Ok(vec![self.route_open_refusal_frame(
5771 ctx,
5772 &caller,
5773 &frame,
5774 &target_module_id,
5775 "relay_waiter_canceled",
5776 "target_unavailable",
5777 "route.bind relay waiter was canceled before the module responded",
5778 )?])
5779 }
5780 Err(_) => {
5781 reservation.release_and_disarm();
5782 if let Some(opened) = breaker.record_timeout(
5786 self.route_bind_breaker_threshold,
5787 self.route_bind_breaker_cooldown,
5788 ) {
5789 warn!(
5790 module_id = %target_module_id,
5791 consecutive_timeouts = opened.consecutive_timeouts,
5792 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5793 reopened_after_probe = opened.reopened_after_probe,
5794 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5795 );
5796 }
5797 tracing::warn!(
5805 module_id = %target_module_id,
5806 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5807 "route.bind relay timed out: module did not ack within budget"
5808 );
5809 Ok(vec![self.route_open_refusal_frame(
5810 ctx,
5811 &caller,
5812 &frame,
5813 &target_module_id,
5814 "relay_timed_out",
5815 "module_timeout",
5816 format!(
5817 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5818 route_bind_relay_timeout
5819 ),
5820 )?])
5821 }
5822 }
5823 }
5824
5825 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5826 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5827 snapshot: self.supervisor.spawn_snapshot(),
5828 };
5829 Ok(vec![control_response_body_frame(
5830 &frame,
5831 &response,
5832 "ClientControlResponse::SupervisorSpawnSnapshot",
5833 )?])
5834 }
5835
5836 fn handle_supervisor_spawn_subscribe(
5837 &self,
5838 ctx: &RouteCtx,
5839 frame: Frame,
5840 since: Option<SpawnCursor>,
5841 ) -> Result<Vec<Frame>, RouterError> {
5842 match self.supervisor.subscribe_spawns(
5843 ctx.connection_id,
5844 frame.header.corr,
5845 response_version(&frame),
5846 since,
5847 ctx.egress.clone(),
5848 ) {
5849 Ok(()) => Ok(Vec::new()),
5850 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5851 Ok(vec![control_error_body_frame(
5852 &frame,
5853 ErrorBody {
5854 code: "spawn_cursor_incarnation_mismatch".to_string(),
5855 message: "spawn cursor belongs to a different daemon incarnation"
5856 .to_string(),
5857 detail: Some(serde_json::json!({
5858 "current_daemon_incarnation": current
5859 })),
5860 },
5861 )?])
5862 }
5863 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5864 &frame,
5865 ErrorBody {
5866 code: "spawn_cursor_too_old".to_string(),
5867 message: "spawn cursor predates the retained event ring".to_string(),
5868 detail: Some(serde_json::json!({
5869 "oldest_retained_cursor": oldest
5870 })),
5871 },
5872 )?]),
5873 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5874 0,
5875 frame.header.corr,
5876 format!("failed to open supervisor spawn subscription: {error}"),
5877 )),
5878 }
5879 }
5880
5881 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5882 let generation = self
5883 .registry
5884 .generation()
5885 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5886 let supervised = self.supervisor.list();
5887 let targets = supervised
5888 .iter()
5889 .map(|module| module.subtree_target())
5890 .collect();
5891 let subtree = tokio::spawn(crate::subtree_resources::read(targets));
5892 let mut modules = Vec::new();
5893 for module in supervised {
5894 let status = module.status_for_control("list").map_err(|err| {
5895 RouterError::backend(
5896 0,
5897 frame.header.corr,
5898 format!("failed to read supervisor status: {err}"),
5899 )
5900 })?;
5901 let (configured, _) = module.configuration().map_err(|err| {
5902 RouterError::backend(
5903 0,
5904 frame.header.corr,
5905 format!("failed to read module configuration: {err}"),
5906 )
5907 })?;
5908 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5909 RouterError::backend(
5910 0,
5911 frame.header.corr,
5912 format!("failed to read retry time: {err}"),
5913 )
5914 })?;
5915 let image = module.running_image_agreement().await;
5917 let resources = Some(crate::subtree_resources::read_process(module.clone()).await);
5920 let mut verdict =
5921 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5922 verdict.spec = Some(launch_spec_agreement(
5923 &configured,
5924 status.process_alive,
5925 status.launch_spec_digest.as_ref(),
5926 ));
5927 let pending_reload = Some(verdict);
5928 let configured_enabled = self
5929 .capability_evaluator
5930 .configured_enabled(&status.module_id);
5931 modules.push(
5934 SupervisorEntry::new(
5935 status.module_id,
5936 status.state.to_string(),
5937 status.enabled,
5938 status.live,
5939 status.health.status,
5940 )
5941 .with_configured_enabled(configured_enabled)
5942 .with_next_retry_at_ms(next_retry_at_ms)
5943 .with_launch_nonce_env(Some(!cfg!(unix)))
5944 .with_launch_nonce_source(status.launch_nonce_source)
5945 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5946 .with_protocol(status.protocol)
5947 .with_pending_reload(pending_reload)
5948 .with_last_probe_ms(status.health.last_probe_ms)
5949 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5950 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5951 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5952 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5953 .with_restart_count(Some(status.restart_count))
5954 .with_max_restarts(Some(status.max_restarts))
5955 .with_lifetime_restarts(Some(status.lifetime_restarts))
5956 .with_spawn_generation(Some(status.spawn_generation))
5957 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5958 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5959 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5960 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5961 .with_resources(resources),
5962 );
5963 }
5964 let readings = subtree
5965 .await
5966 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5967 for (module, reading) in modules.iter_mut().zip(readings) {
5968 if reading.reason.as_deref() == Some("timeout") {
5969 crate::subtree_resources::log_timeout(&module.module_id, "subtree");
5970 }
5971 module.subtree = Some(reading);
5972 }
5973 let response = ClientControlResponse::SupervisorList {
5974 generation,
5975 modules,
5976 };
5977 Ok(vec![control_response_body_frame(
5978 &frame,
5979 &response,
5980 "ClientControlResponse::SupervisorList",
5981 )?])
5982 }
5983
5984 fn handle_supervisor_stderr_tail(
5985 &self,
5986 frame: Frame,
5987 module_id: String,
5988 max_lines: Option<u32>,
5989 max_bytes: Option<u32>,
5990 ) -> Result<Vec<Frame>, RouterError> {
5991 let Some(module) = self.supervisor.get(&module_id) else {
5992 return Ok(vec![control_error_frame(
5993 &frame,
5994 "unknown_module",
5995 format!("module_id '{module_id}' is not supervised"),
5996 )?]);
5997 };
5998
5999 let snapshot = module.stderr_tail(
6000 max_lines.map(|value| value as usize),
6001 max_bytes.map(|value| value as usize),
6002 );
6003
6004 let response = ClientControlResponse::SupervisorStderrTail {
6005 module_id,
6006 tail: StderrTail {
6007 capture: match snapshot.capture {
6008 CaptureState::Captured => StderrCaptureState::Captured,
6009 CaptureState::Incomplete { reason } => {
6010 StderrCaptureState::Incomplete { reason }
6011 }
6012 CaptureState::NotCaptured { reason } => {
6013 StderrCaptureState::NotCaptured { reason }
6014 }
6015 },
6016 entries: snapshot
6017 .entries
6018 .into_iter()
6019 .map(|entry| match entry {
6020 TailEntry::Line {
6021 text,
6022 truncated,
6023 at_ms,
6024 } => StderrTailEntry::Line {
6025 text,
6026 truncated,
6027 at_ms,
6028 },
6029 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
6030 })
6031 .collect(),
6032 dropped_lines: snapshot.dropped_lines,
6033 },
6034 };
6035 Ok(vec![control_response_body_frame(
6036 &frame,
6037 &response,
6038 "ClientControlResponse::SupervisorStderrTail",
6039 )?])
6040 }
6041
6042 async fn observe_control_wait<F: std::future::Future>(
6043 &self,
6044 ctx: &RouteCtx,
6045 target: Option<&str>,
6046 kind: &'static str,
6047 future: F,
6048 ) -> F::Output {
6049 self.counters
6050 .observe_reader_wait(
6051 || {
6052 let module = self
6053 .forwarding
6054 .module_id_for_connection(ctx.connection_id)
6055 .ok()
6056 .flatten();
6057 let principal = if module.is_some() { "module" } else { "direct" };
6058 (
6059 self.resolve_reader_wait_target(kind, target)
6060 .or(module)
6061 .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
6062 principal.into(),
6063 )
6064 },
6065 kind,
6066 future,
6067 )
6068 .await
6069 }
6070
6071 pub(crate) fn resolve_reader_wait_target(
6074 &self,
6075 op: &str,
6076 selector: Option<&str>,
6077 ) -> Option<String> {
6078 let selector = selector?;
6079 match op {
6080 ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
6081 .supervisor
6082 .get(selector)
6083 .map(|module| module.module_id().to_owned()),
6084 ops::SUPERVISOR_HEALTH_PROBE => self
6085 .registry
6086 .get_module(selector)
6087 .ok()
6088 .flatten()
6089 .map(|registration| registration.manifest.module_id),
6090 _ => None,
6091 }
6092 }
6093
6094 async fn handle_supervisor_terminals(
6095 &self,
6096 ctx: &RouteCtx,
6097 frame: Frame,
6098 module_id: String,
6099 ) -> Result<Vec<Frame>, RouterError> {
6100 let Some(module) = self.supervisor.get(&module_id) else {
6101 return Ok(vec![control_error_frame(
6102 &frame,
6103 "unknown_module",
6104 format!("module_id '{module_id}' is not supervised"),
6105 )?]);
6106 };
6107
6108 let terminals = self
6111 .observe_control_wait(
6112 ctx,
6113 Some(module.module_id()),
6114 ops::SUPERVISOR_TERMINALS,
6115 module.read_durable_terminal_history(),
6116 )
6117 .await
6118 .map_err(|error| {
6119 RouterError::backend(
6120 0,
6121 frame.header.corr,
6122 format!("failed to read terminal history: {error}"),
6123 )
6124 })?;
6125 let response = ClientControlResponse::SupervisorTerminals {
6126 module_id,
6127 terminals,
6128 };
6129 Ok(vec![control_response_body_frame(
6130 &frame,
6131 &response,
6132 "ClientControlResponse::SupervisorTerminals",
6133 )?])
6134 }
6135
6136 fn handle_supervisor_routes(
6137 &self,
6138 frame: Frame,
6139 module_id: Option<String>,
6140 ) -> Result<Vec<Frame>, RouterError> {
6141 let modules = self
6142 .forwarding
6143 .route_census(module_id.as_deref())
6144 .map_err(RouterError::Forwarding)?
6145 .into_iter()
6146 .map(|(module_id, routes)| SupervisorRouteModule {
6147 module_id,
6148 routes: routes
6149 .into_iter()
6150 .map(|route| SupervisorRoute {
6151 consumer: match route.principal {
6152 Principal::Reserved { module_id } => {
6153 SupervisorRouteConsumer::Reserved { module_id }
6154 }
6155 Principal::Direct | Principal::Unverified => {
6156 SupervisorRouteConsumer::Direct {
6157 connection_id: route.goodbye_target.connection_id.get(),
6158 }
6159 }
6160 },
6161 age_ms: Instant::now()
6162 .saturating_duration_since(route.bound_at)
6163 .as_millis()
6164 .try_into()
6165 .unwrap_or(u64::MAX),
6166 draining: route.draining,
6167 drain_reason: route.drain_reason,
6168 })
6169 .collect(),
6170 })
6171 .collect();
6172 let response = ClientControlResponse::SupervisorRoutes { modules };
6173 Ok(vec![control_response_body_frame(
6174 &frame,
6175 &response,
6176 "ClientControlResponse::SupervisorRoutes",
6177 )?])
6178 }
6179
6180 async fn handle_supervisor_provenance(
6181 &self,
6182 ctx: &RouteCtx,
6183 frame: Frame,
6184 module_id: Option<String>,
6185 ) -> Result<Vec<Frame>, RouterError> {
6186 let targeted = module_id.is_some();
6187 let selected = if let Some(module_id) = module_id {
6188 let Some(module) = self.supervisor.get(&module_id) else {
6189 return Ok(vec![control_error_frame(
6190 &frame,
6191 "unknown_module",
6192 format!("module_id '{module_id}' is not supervised"),
6193 )?]);
6194 };
6195 vec![module]
6196 } else {
6197 self.supervisor.list()
6198 };
6199
6200 let target = targeted.then(|| selected[0].module_id());
6201 self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
6202 let mut modules = Vec::with_capacity(selected.len());
6203 for module in &selected {
6204 let (status, observed_image) = module
6205 .status_and_running_image_agreement()
6206 .await
6207 .map_err(|err| {
6208 RouterError::backend(
6209 0,
6210 frame.header.corr,
6211 format!("failed to read supervisor status: {err}"),
6212 )
6213 })?;
6214 let module_declared = self
6215 .registry
6216 .get_module(&status.module_id)
6217 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6218 .and_then(|registration| registration.manifest.provenance)
6219 .map(|build| ModuleDeclaredProvenance::Reported { build })
6220 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
6221 #[cfg(test)]
6222 let running_image = match &self.provenance_probe_override {
6223 Some(result) => result.clone(),
6224 None => observed_image,
6225 };
6226 #[cfg(not(test))]
6227 let running_image = observed_image;
6228 modules.push(SupervisorModuleProvenance {
6229 module_id: status.module_id,
6230 module_declared,
6231 daemon_observed: SupervisorObservedProcess {
6232 launch_nonce_source: status.launch_nonce_source,
6233 pid: status.pid,
6234 spawned_at_ms: status.spawned_at_ms,
6235 spawned_from: status.spawned_from,
6236 running_image,
6237 },
6238 });
6239 }
6240 let daemon = SupervisorDaemonProvenance {
6241 daemon_build: self.daemon_provenance.build.clone(),
6242 daemon_observed: DaemonObservedProcess {
6243 pid: self.daemon_provenance.pid,
6244 started_at_ms: self
6245 .daemon_provenance
6246 .start_clock
6247 .map(|clock| clock.started_at_ms())
6248 .or(self.daemon_provenance.started_at_ms),
6249 running_image: self
6250 .daemon_provenance
6251 .probe
6252 .observe(
6253 self.daemon_provenance.pid,
6254 self.daemon_provenance.executable_path.as_deref(),
6255 self.daemon_provenance.executable_identity.clone(),
6256 self.daemon_provenance.process_start_time,
6257 )
6258 .await,
6259 },
6260 };
6261 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
6262 Ok(vec![control_response_body_frame(
6263 &frame,
6264 &response,
6265 "ClientControlResponse::SupervisorProvenance",
6266 )?])
6267 })
6268 .await
6269 }
6270
6271 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6272 self.refresh_capability_requirements();
6273 let generation = self
6274 .registry
6275 .generation()
6276 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6277 let modules = self
6278 .supervisor
6279 .list()
6280 .into_iter()
6281 .map(|module| {
6282 let status = module.status_for_control("health").map_err(|err| {
6283 RouterError::backend(
6284 0,
6285 frame.header.corr,
6286 format!("failed to read supervisor health: {err}"),
6287 )
6288 })?;
6289 let module_id = status.module_id;
6290 let capability_detail = self
6291 .capability_evaluator
6292 .required_problem_detail(&module_id);
6293 Ok(SupervisorHealthEntry {
6294 module_id,
6295 status: status.health.status,
6296 detail: append_capability_problem_detail(
6297 status.health.detail,
6298 capability_detail,
6299 ),
6300 metrics: status.health.metrics,
6301 consecutive_failures: status.health.consecutive_failures,
6302 late_answer_count: status.health.late_answer_count,
6303 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
6304 last_action: status.health.last_action,
6305 last_action_ms: status.health.last_action_ms,
6306 last_probe_ms: status.health.last_probe_ms,
6307 })
6308 })
6309 .collect::<Result<Vec<_>, RouterError>>()?;
6310 let response = ClientControlResponse::SupervisorHealth {
6311 generation,
6312 modules,
6313 };
6314 Ok(vec![control_response_body_frame(
6315 &frame,
6316 &response,
6317 "ClientControlResponse::SupervisorHealth",
6318 )?])
6319 }
6320
6321 async fn handle_supervisor_restart(
6322 &self,
6323 frame: Frame,
6324 module_id: String,
6325 drain_timeout_ms: Option<u64>,
6326 requester: Principal,
6327 ) -> Result<Vec<Frame>, RouterError> {
6328 let operation_lock = self.supervisor.operation_lock();
6329 let _operation_guard = operation_lock.lock().await;
6330 let Some(module) = self.supervisor.get(&module_id) else {
6331 return Ok(vec![control_error_frame(
6332 &frame,
6333 "unknown_module",
6334 format!("module_id '{module_id}' is not supervised"),
6335 )?]);
6336 };
6337
6338 self.route_outages.mark_operator_action(&module_id);
6339 if let Err(err) = module
6340 .restart_requested(drain_timeout_ms, Some(requester))
6341 .await
6342 {
6343 self.route_outages
6344 .operator_action_ended_unrefused(&module_id);
6345 let (code, message) = match err {
6346 crate::supervise::SuperviseError::Disabled { .. } => {
6347 ("module_disabled", err.to_string())
6348 }
6349 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6350 ("swap_in_progress", err.to_string())
6351 }
6352 _ => (
6353 "target_unavailable",
6354 format!("failed to restart module_id '{module_id}': {err}"),
6355 ),
6356 };
6357 return Ok(vec![control_error_frame(&frame, code, message)?]);
6358 }
6359
6360 let response = ClientControlResponse::SupervisorAck {
6361 module_id,
6362 applied: true,
6363 };
6364 Ok(vec![control_response_body_frame(
6365 &frame,
6366 &response,
6367 "ClientControlResponse::SupervisorAck",
6368 )?])
6369 }
6370
6371 async fn handle_supervisor_swap(
6375 &self,
6376 frame: Frame,
6377 module_id: String,
6378 ready_timeout_ms: Option<u64>,
6379 requester: Principal,
6380 ) -> Result<Vec<Frame>, RouterError> {
6381 let module = {
6388 let operation_lock = self.supervisor.operation_lock();
6389 let _operation_guard = operation_lock.lock().await;
6390 self.supervisor.get(&module_id)
6391 };
6392 let Some(module) = module else {
6393 return Ok(vec![control_error_frame(
6394 &frame,
6395 "unknown_module",
6396 format!("module_id '{module_id}' is not supervised"),
6397 )?]);
6398 };
6399
6400 self.route_outages.mark_operator_action(&module_id);
6401 if let Err(err) = module
6402 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6403 .await
6404 {
6405 self.route_outages
6406 .operator_action_ended_unrefused(&module_id);
6407 use crate::supervise::SuperviseError;
6408 let message = err.to_string();
6409 let error = match err {
6410 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6411 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6412 code: "swap_refused".to_string(),
6413 message,
6414 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6415 },
6416 SuperviseError::SwapFailed {
6417 arm,
6418 candidate_exit,
6419 ..
6420 } => ErrorBody {
6421 code: "swap_failed".to_string(),
6422 message,
6423 detail: Some(serde_json::json!({
6424 "arm": arm.as_str(),
6425 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6426 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6427 })),
6428 },
6429 _ => ErrorBody::new(
6430 "target_unavailable",
6431 format!("failed to swap module_id '{module_id}': {message}"),
6432 ),
6433 };
6434 return Ok(vec![control_error_body_frame(&frame, error)?]);
6435 }
6436 self.route_outages
6440 .operator_action_ended_unrefused(&module_id);
6441
6442 let response = ClientControlResponse::SupervisorAck {
6443 module_id,
6444 applied: true,
6445 };
6446 Ok(vec![control_response_body_frame(
6447 &frame,
6448 &response,
6449 "ClientControlResponse::SupervisorAck",
6450 )?])
6451 }
6452
6453 async fn handle_supervisor_reload(
6454 &self,
6455 frame: Frame,
6456 module_id: String,
6457 requester: Principal,
6458 ) -> Result<Vec<Frame>, RouterError> {
6459 let operation_lock = self.supervisor.operation_lock();
6460 let _operation_guard = operation_lock.lock().await;
6461 let Some(module) = self.supervisor.get(&module_id) else {
6462 return Ok(vec![control_error_frame(
6463 &frame,
6464 "unknown_module",
6465 format!("module_id '{module_id}' is not supervised"),
6466 )?]);
6467 };
6468
6469 self.route_outages.mark_operator_action(&module_id);
6470 if let Err(err) = module.reload_requested(Some(requester)).await {
6471 self.route_outages
6472 .operator_action_ended_unrefused(&module_id);
6473 let (code, message) = match err {
6474 crate::supervise::SuperviseError::Disabled { .. } => {
6475 ("module_disabled", err.to_string())
6476 }
6477 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6478 ("swap_in_progress", err.to_string())
6479 }
6480 _ => (
6481 "reload_failed",
6482 format!("failed to reload module_id '{module_id}': {err}"),
6483 ),
6484 };
6485 return Ok(vec![control_error_frame(&frame, code, message)?]);
6486 }
6487
6488 let response = ClientControlResponse::SupervisorAck {
6489 module_id,
6490 applied: true,
6491 };
6492 Ok(vec![control_response_body_frame(
6493 &frame,
6494 &response,
6495 "ClientControlResponse::SupervisorAck",
6496 )?])
6497 }
6498
6499 async fn handle_supervisor_rescan(
6503 &self,
6504 frame: Frame,
6505 mode: RescanMode,
6506 requester: Principal,
6507 ) -> Result<Vec<Frame>, RouterError> {
6508 let preview = mode.previews();
6509 let Some(context) = self.rescan.clone() else {
6510 return Ok(vec![control_error_frame(
6511 &frame,
6512 "rescan_unavailable",
6513 "the daemon was not started with a reloadable config path".to_string(),
6514 )?]);
6515 };
6516
6517 let operation_lock = self.supervisor.operation_lock();
6518 let _operation_guard = operation_lock.lock().await;
6519 let loaded = match self.load_rescan_config(&mode, &context) {
6523 Ok(config) => config,
6524 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6525 };
6526 let Some((config, activity_config)) = loaded else {
6534 return Ok(vec![control_error_frame(
6535 &frame,
6536 "invalid_daemon_config",
6537 format!(
6538 "daemon config not found at {}; refusing to rescan (an absent config would \
6539 retire every supervised module)",
6540 context.config_path.display()
6541 ),
6542 )?]);
6543 };
6544 let (
6545 configured_port,
6546 storage_config,
6547 admission_facts_carrier_module_id,
6548 admission_facts_targets,
6549 scope_authority_owners,
6550 privileged_operator_prompt,
6551 modules,
6552 reserved_capabilities,
6553 ) = (
6554 config.port,
6555 config.storage,
6556 config.admission_facts_carrier_module_id,
6557 config.admission_facts_targets,
6558 config.scope_authority_owners,
6559 config.privileged_operator_prompt,
6560 config.modules,
6561 config.reserved_capabilities,
6562 );
6563
6564 let mut restart_required = Vec::new();
6573 for section in RestartRequiredSection::ALL {
6574 let changed = match section {
6575 RestartRequiredSection::Port => configured_port != context.configured_port,
6576 RestartRequiredSection::Storage => storage_config != context.storage_config,
6577 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6578 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6579 }
6580 RestartRequiredSection::AdmissionFactsTargets => {
6581 admission_facts_targets != context.admission_facts_targets
6582 }
6583 RestartRequiredSection::ScopeAuthorityOwners => {
6584 scope_authority_owners != context.scope_authority_owners
6585 }
6586 RestartRequiredSection::PrivilegedOperatorPrompt => {
6587 privileged_operator_prompt != context.privileged_operator_prompt
6588 }
6589 };
6590 if changed {
6591 restart_required.push(section.label().to_string());
6592 }
6593 }
6594 if !restart_required.is_empty() {
6595 warn!(
6596 config_path = %context.config_path.display(),
6597 sections = %restart_required.join(", "),
6598 "daemon config changed outside the modules section; restart the daemon to apply those changes"
6599 );
6600 }
6601
6602 for configured in &modules {
6603 if let Err(err) = validate_spec(&configured.module_spec()) {
6604 return Ok(vec![control_error_frame(
6605 &frame,
6606 "invalid_daemon_config",
6607 format!("supervisor rescan rejected daemon config: {err}"),
6608 )?]);
6609 }
6610 }
6611
6612 let configured_capabilities = modules
6613 .iter()
6614 .map(|module| (module.module_id.clone(), module.enabled))
6615 .collect::<Vec<_>>();
6616 let preview_capability_warnings = if preview {
6617 let (_, registrations) = self.runtime_capability_snapshot()?;
6618 let current_modules = self
6619 .supervisor
6620 .list()
6621 .into_iter()
6622 .map(|module| module.module_id().to_string())
6623 .collect::<BTreeSet<_>>();
6624 let resulting_modules = configured_capabilities.clone();
6625 let removed = current_modules
6626 .into_iter()
6627 .filter(|module_id| {
6628 !resulting_modules
6629 .iter()
6630 .any(|(configured_id, _)| configured_id == module_id)
6631 })
6632 .collect::<Vec<_>>();
6633 let disabled = match &mode {
6636 RescanMode::Candidate(_) => resulting_modules
6637 .iter()
6638 .filter(|(_, enabled)| !enabled)
6639 .map(|(module_id, _)| module_id.clone())
6640 .collect::<Vec<_>>(),
6641 _ => Vec::new(),
6642 };
6643 self.capability_evaluator.preview_warnings(
6644 resulting_modules,
6645 &removed,
6646 &disabled,
6647 ®istrations,
6648 )
6649 } else {
6650 Vec::new()
6651 };
6652 let reconciled = match &mode {
6653 RescanMode::Candidate(candidate) => self
6654 .preview_candidate(&context, modules, candidate)
6655 .map(|result| (result, Vec::new())),
6656 _ => {
6657 self.reconcile_supervised_modules(
6658 &context.supervisor,
6659 modules,
6660 preview,
6661 requester.clone(),
6662 )
6663 .await
6664 }
6665 };
6666 let (result, changed) = match reconciled {
6667 Ok(result) => result,
6668 Err(message) => {
6669 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6670 }
6671 };
6672 if !preview {
6673 if let Some(ledger) = &self.activity {
6674 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6675 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6676 }
6677 if let Err(code) = self.configure_activity(activity_config).await {
6678 warn!(
6679 code,
6680 "activity retention could not be applied during rescan"
6681 );
6682 }
6683 self.capability_evaluator
6684 .configure(configured_capabilities, reserved_capabilities);
6685 self.capability_evaluator.wake_deadline_loop();
6686 self.refresh_capability_requirements();
6687 }
6688 let mut result = result;
6689 result.restart_required = restart_required;
6690 result.capability_warnings = preview_capability_warnings;
6691 let response = ClientControlResponse::SupervisorRescan { result };
6692 Ok(vec![control_response_body_frame(
6693 &frame,
6694 &response,
6695 "ClientControlResponse::SupervisorRescan",
6696 )?])
6697 }
6698
6699 async fn handle_supervisor_release_reserved(
6700 &self,
6701 frame: Frame,
6702 module_id: String,
6703 ) -> Result<Vec<Frame>, RouterError> {
6704 let Some(context) = self.rescan.clone() else {
6705 return Ok(vec![control_error_frame(
6706 &frame,
6707 "release_unavailable",
6708 "reserved-id release requires a daemon started with a reloadable config path",
6709 )?]);
6710 };
6711 let operation_lock = self.supervisor.operation_lock();
6712 let _operation_guard = operation_lock.lock().await;
6713 let loaded = match crate::daemon_config::load(&context.config_path) {
6714 Ok(Some(config)) => config,
6715 Ok(None) => {
6716 return Ok(vec![control_error_frame(
6717 &frame,
6718 "invalid_daemon_config",
6719 format!(
6720 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6721 context.config_path.display()
6722 ),
6723 )?])
6724 }
6725 Err(err) => {
6726 return Ok(vec![control_error_frame(
6727 &frame,
6728 "invalid_daemon_config",
6729 format!("unable to verify reserved-id release against daemon config: {err}"),
6730 )?])
6731 }
6732 };
6733 if loaded
6734 .modules
6735 .iter()
6736 .any(|configured| configured.module_id == module_id)
6737 {
6738 return Ok(vec![control_error_frame(
6739 &frame,
6740 "reserved_module_configured",
6741 format!(
6742 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6743 ),
6744 )?]);
6745 }
6746 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6747 return Ok(vec![control_error_frame(
6748 &frame,
6749 "reserved_gate_not_retained",
6750 format!(
6751 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6752 ),
6753 )?]);
6754 }
6755
6756 let response = ClientControlResponse::SupervisorAck {
6757 module_id,
6758 applied: true,
6759 };
6760 Ok(vec![control_response_body_frame(
6761 &frame,
6762 &response,
6763 "ClientControlResponse::SupervisorAck",
6764 )?])
6765 }
6766
6767 fn plan_rescan(
6771 &self,
6772 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6773 ) -> Result<RescanPlan, String> {
6774 let mut current = BTreeMap::new();
6775 for module in self.supervisor.list() {
6776 let (spec, health) = module.configuration().map_err(|err| {
6777 format!(
6778 "failed to read configuration for module_id '{}': {err}",
6779 module.module_id()
6780 )
6781 })?;
6782 let enabled = module
6783 .status()
6784 .map_err(|err| {
6785 format!(
6786 "failed to read status for module_id '{}': {err}",
6787 module.module_id()
6788 )
6789 })?
6790 .enabled;
6791 current.insert(
6792 module.module_id().to_string(),
6793 (module, spec, health, enabled),
6794 );
6795 }
6796 let configured = configured_modules
6797 .into_iter()
6798 .map(|module| (module.module_id.clone(), module))
6799 .collect::<BTreeMap<_, _>>();
6800
6801 let added = configured
6802 .keys()
6803 .filter(|module_id| !current.contains_key(*module_id))
6804 .cloned()
6805 .collect::<Vec<_>>();
6806 let removed = current
6807 .keys()
6808 .filter(|module_id| !configured.contains_key(*module_id))
6809 .cloned()
6810 .collect::<Vec<_>>();
6811 let mut changed_pending_reload = Vec::new();
6812 let mut configuration_changes = BTreeSet::new();
6813 let mut enabled_changes = BTreeSet::new();
6814 let mut unchanged = 0_u32;
6815
6816 for (module_id, configured_module) in &configured {
6817 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6818 else {
6819 continue;
6820 };
6821 let launch_changed = *current_spec != configured_module.module_spec();
6825 let configuration_changed =
6826 launch_changed || *current_health != configured_module.health;
6827 let enabled_changed = *current_enabled != configured_module.enabled
6830 || self
6831 .capability_evaluator
6832 .configured_enabled(module_id)
6833 .is_some_and(|enabled| enabled != configured_module.enabled);
6834 if configuration_changed {
6835 configuration_changes.insert(module_id.clone());
6836 }
6837 if launch_changed {
6838 changed_pending_reload.push(module_id.clone());
6839 }
6840 if enabled_changed {
6841 enabled_changes.insert(module_id.clone());
6842 }
6843 if !configuration_changed && !enabled_changed {
6844 unchanged = unchanged.saturating_add(1);
6845 }
6846 }
6847 Ok(RescanPlan {
6848 current,
6849 configured,
6850 added,
6851 removed,
6852 changed_pending_reload,
6853 configuration_changes,
6854 enabled_changes,
6855 unchanged,
6856 })
6857 }
6858
6859 async fn reconcile_supervised_modules(
6868 &self,
6869 supervisor: &Supervisor,
6870 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6871 preview: bool,
6872 requester: Principal,
6873 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6874 let plan = self.plan_rescan(configured_modules)?;
6875
6876 if preview {
6882 return Ok((plan.preview_result(), Vec::new()));
6883 }
6884 let changed = plan
6885 .configuration_changes
6886 .union(&plan.enabled_changes)
6887 .cloned()
6888 .collect::<Vec<_>>();
6889 let RescanPlan {
6890 current,
6891 configured,
6892 added,
6893 removed,
6894 changed_pending_reload,
6895 configuration_changes,
6896 enabled_changes,
6897 unchanged,
6898 } = plan;
6899 if !(added.is_empty()
6904 && removed.is_empty()
6905 && configuration_changes.is_empty()
6906 && enabled_changes.is_empty())
6907 {
6908 if let Some(context) = &self.rescan {
6909 context
6910 .generation
6911 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6912 }
6913 }
6914
6915 for module_id in &removed {
6916 let module = ¤t
6917 .get(module_id)
6918 .expect("removed module came from current supervisor state")
6919 .0;
6920 module
6921 .retire_requested(Some(requester.clone()))
6922 .await
6923 .map_err(|err| {
6924 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6925 })?;
6926 self.supervisor.record_rescan_removal(module_id);
6960 self.supervisor.retire(module_id);
6961 self.route_outages.forget(module_id);
6962 }
6963
6964 for module_id in configured.keys() {
6965 let Some((module, _, _, current_enabled)) = current.get(module_id) else {
6966 continue;
6967 };
6968 let configured_module = configured
6969 .get(module_id)
6970 .expect("configured module id came from configured map");
6971 if configuration_changes.contains(module_id) {
6972 module
6973 .update_configuration(
6974 configured_module.module_spec(),
6975 configured_module.health.clone(),
6976 configured_module.drain_timeout_ms,
6977 )
6978 .await
6979 .map_err(|err| {
6980 format!(
6981 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6982 )
6983 })?;
6984 }
6985 if *current_enabled != configured_module.enabled {
6988 self.route_outages.mark_operator_action(module_id);
6991 module
6992 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6993 .await
6994 .map_err(|err| {
6995 self.route_outages.operator_action_ended_unrefused(module_id);
6996 format!(
6997 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6998 configured_module.enabled
6999 )
7000 })?;
7001 }
7002 }
7003
7004 for module_id in &added {
7005 let configured_module = configured
7006 .get(module_id)
7007 .expect("added module id came from configured map");
7008 supervisor
7009 .supervise_configured_requested(
7010 configured_module.module_spec(),
7011 configured_module.enabled,
7012 configured_module.health.clone(),
7013 configured_module.drain_timeout_ms,
7014 configured_module.restart,
7015 configured_module.register_timeout,
7016 Some(requester.clone()),
7017 )
7018 .map_err(|err| {
7019 format!("failed to add module_id '{module_id}' during rescan: {err}")
7020 })?;
7021 }
7022
7023 Ok((
7024 SupervisorRescanResult {
7025 added,
7026 removed,
7027 changed_pending_reload,
7028 enabled_changes: enabled_changes.iter().cloned().collect(),
7029 unchanged,
7030 preview: false,
7031 restart_required: Vec::new(),
7034 capability_warnings: Vec::new(),
7035 module_actions: Vec::new(),
7036 earlier_pending: Vec::new(),
7037 expect: None,
7038 },
7039 changed,
7040 ))
7041 }
7042
7043 async fn handle_supervisor_set_enabled(
7044 &self,
7045 frame: Frame,
7046 module_id: String,
7047 enabled: bool,
7048 requester: Principal,
7049 ) -> Result<Vec<Frame>, RouterError> {
7050 let operation_lock = self.supervisor.operation_lock();
7051 let _operation_guard = operation_lock.lock().await;
7052 let Some(module) = self.supervisor.get(&module_id) else {
7053 return Ok(vec![control_error_frame(
7054 &frame,
7055 "unknown_module",
7056 format!("module_id '{module_id}' is not supervised"),
7057 )?]);
7058 };
7059
7060 self.route_outages.mark_operator_action(&module_id);
7063 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
7064 Ok(applied) => applied,
7065 Err(err) => {
7066 self.route_outages
7067 .operator_action_ended_unrefused(&module_id);
7068 return Ok(vec![control_error_frame(
7069 &frame,
7070 "target_unavailable",
7071 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
7072 )?]);
7073 }
7074 };
7075 if !applied {
7076 self.route_outages
7079 .operator_action_ended_unrefused(&module_id);
7080 }
7081
7082 self.capability_evaluator.wake_deadline_loop();
7083 self.refresh_capability_requirements();
7084 let response = ClientControlResponse::SupervisorAck { module_id, applied };
7085 Ok(vec![control_response_body_frame(
7086 &frame,
7087 &response,
7088 "ClientControlResponse::SupervisorAck",
7089 )?])
7090 }
7091
7092 async fn handle_supervisor_health_probe(
7093 &self,
7094 ctx: &RouteCtx,
7095 frame: Frame,
7096 module_id: String,
7097 ) -> Result<Vec<Frame>, RouterError> {
7098 self.refresh_capability_requirements();
7099 let Some(registration) = self
7100 .registry
7101 .get_module(&module_id)
7102 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
7103 else {
7104 return Ok(vec![control_error_frame(
7105 &frame,
7106 "unknown_module",
7107 format!("module_id '{module_id}' is not registered"),
7108 )?]);
7109 };
7110
7111 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
7124 {
7125 return Ok(vec![control_error_frame(
7126 &frame,
7127 "health_not_advertised",
7128 format!("module_id '{module_id}' did not advertise health.check"),
7129 )?]);
7130 }
7131
7132 self.observe_control_wait(
7133 ctx,
7134 Some(®istration.manifest.module_id),
7135 ops::SUPERVISOR_HEALTH_PROBE,
7136 async {
7137 let deadline = Instant::now() + self.health_probe_timeout;
7138 let pending = match self.forwarding.begin_module_control_rpc_for(
7139 &module_id,
7140 MODULE_CONTROL_OP_HEALTH_CHECK,
7141 deadline,
7142 ) {
7143 Ok(pending) => pending,
7144 Err(err) => {
7145 return Ok(vec![control_error_frame(
7146 &frame,
7147 forwarding_error_code(&err),
7148 err.to_string(),
7149 )?])
7150 }
7151 };
7152
7153 let PendingModuleControlRpc {
7154 endpoint,
7155 module_sink,
7156 negotiated_ver,
7157 corr: probe_corr,
7158 receiver,
7159 } = pending;
7160 let mut guard =
7161 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
7162 let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
7163 .map_err(|err| {
7164 RouterError::backend(
7165 0,
7166 frame.header.corr,
7167 format!("failed to encode health.check request: {err}"),
7168 )
7169 })?;
7170 let probe_frame = Frame::build_with_version(
7171 negotiated_ver,
7172 FrameType::Request,
7173 control_flags(),
7174 0,
7175 0,
7176 probe_corr,
7177 probe_body,
7178 )
7179 .map_err(RouterError::FrameBuild)?;
7180
7181 match timeout_at(deadline, module_sink.send(probe_frame)).await {
7184 Ok(Ok(())) => {}
7185 Ok(Err(err)) => {
7186 return Ok(vec![control_error_frame(
7187 &frame,
7188 "target_unavailable",
7189 err.to_string(),
7190 )?]);
7191 }
7192 Err(_) => {
7193 return Ok(vec![control_error_frame(
7194 &frame,
7195 "module_timeout",
7196 format!(
7197 "module_id '{module_id}' did not answer health.check within {:?}",
7198 self.health_probe_timeout
7199 ),
7200 )?]);
7201 }
7202 }
7203
7204 match timeout_at(deadline, receiver).await {
7205 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
7206 guard.disarm();
7207 let Some(report) = response.health_report() else {
7208 return Ok(vec![control_error_frame(
7209 &frame,
7210 "invalid_control_body",
7211 "health.check RPC returned a non-health response",
7212 )?]);
7213 };
7214 let HealthReport {
7219 status,
7220 detail,
7221 metrics,
7222 } = report;
7223 let capability_detail = self
7224 .capability_evaluator
7225 .required_problem_detail(&module_id);
7226 let response = ClientControlResponse::SupervisorHealthProbe {
7227 module_id,
7228 status,
7229 detail: append_capability_problem_detail(detail, capability_detail),
7230 metrics,
7231 };
7232 Ok(vec![control_response_body_frame(
7233 &frame,
7234 &response,
7235 "ClientControlResponse::SupervisorHealthProbe",
7236 )?])
7237 }
7238 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
7239 guard.disarm();
7240 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
7245 }
7246 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
7247 guard.disarm();
7248 Ok(vec![control_error_frame(
7249 &frame,
7250 "target_unavailable",
7251 message,
7252 )?])
7253 }
7254 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
7255 guard.disarm();
7256 Ok(vec![control_error_frame(
7257 &frame,
7258 "invalid_control_body",
7259 message,
7260 )?])
7261 }
7262 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
7263 guard.disarm();
7264 Ok(vec![control_error_frame(
7265 &frame,
7266 "invalid_control_body",
7267 format!("expected module-control op '{expected}', got '{actual}'"),
7268 )?])
7269 }
7270 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
7271 guard.disarm();
7272 Ok(vec![control_error_frame(
7273 &frame,
7274 "module_timeout",
7275 format!(
7276 "module_id '{module_id}' answered health.check after {:?}",
7277 self.health_probe_timeout
7278 ),
7279 )?])
7280 }
7281 Ok(Err(_)) => Ok(vec![control_error_frame(
7282 &frame,
7283 "target_unavailable",
7284 "health.check waiter was canceled before the module responded",
7285 )?]),
7286 Err(_) => Ok(vec![control_error_frame(
7287 &frame,
7288 "module_timeout",
7289 format!(
7290 "module_id '{module_id}' did not answer health.check within {:?}",
7291 self.health_probe_timeout
7292 ),
7293 )?]),
7294 }
7295 },
7296 )
7297 .await
7298 }
7299
7300 fn supervisor_status(
7301 &self,
7302 module_id: &str,
7303 corr: u64,
7304 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
7305 self.supervisor
7306 .get(module_id)
7307 .map(|module| {
7308 let warming = module.is_warming_for_control("status").map_err(|err| {
7309 RouterError::backend(
7310 0,
7311 corr,
7312 format!(
7313 "failed to read supervisor warming state for module_id '{module_id}': {err}"
7314 ),
7315 )
7316 })?;
7317 module.status_for_control("status").map_err(|err| {
7318 RouterError::backend(
7319 0,
7320 corr,
7321 format!(
7322 "failed to read supervisor status for module_id '{module_id}': {err}"
7323 ),
7324 )
7325 }).map(|status| (status, warming))
7326 })
7327 .transpose()
7328 }
7329
7330 fn guard_module_control_op(
7331 &self,
7332 frame: &Frame,
7333 module_id: &str,
7334 op: &str,
7335 ) -> Result<Option<Frame>, RouterError> {
7336 if self.module_grants_op(module_id, op, frame.header.corr)? {
7337 return Ok(None);
7338 }
7339
7340 Ok(Some(control_error_frame(
7341 frame,
7342 "op_not_allowed",
7343 format!("module_id '{module_id}' did not grant control op '{op}'"),
7344 )?))
7345 }
7346
7347 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
7348 let Some(registration) = self
7349 .registry
7350 .get_module(module_id)
7351 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
7352 else {
7353 return Ok(false);
7354 };
7355 Ok(module_registration_grants_op(®istration.control_ops, op))
7356 }
7357
7358 fn handle_status_update(
7359 &self,
7360 endpoint: ModuleEndpointId,
7361 frame: Frame,
7362 ) -> Result<Vec<Frame>, RouterError> {
7363 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
7364 Ok(update) => update,
7365 Err(err) => {
7366 if is_known_module_push_op(&frame.body) {
7371 return Ok(vec![control_error_frame(
7372 &frame,
7373 "invalid_control_body",
7374 format!("malformed module control push body: {err}"),
7375 )?]);
7376 }
7377 return Ok(Vec::new());
7378 }
7379 };
7380
7381 match update {
7382 ModuleControlPush::RouteStatus {
7383 route_channel,
7384 route_epoch,
7385 status,
7386 } => {
7387 self.forwarding
7388 .cache_status(endpoint, route_channel, route_epoch, status)
7389 .map_err(RouterError::Forwarding)?;
7390 }
7391 }
7392 Ok(Vec::new())
7393 }
7394
7395 fn handle_route_poll(
7396 &self,
7397 ctx: &RouteCtx,
7398 frame: Frame,
7399 route_channel: u16,
7400 route_epoch: u32,
7401 kind: PollKind,
7402 ) -> Result<Vec<Frame>, RouterError> {
7403 let snapshot = self
7404 .forwarding
7405 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7406 .map_err(RouterError::Forwarding)?;
7407 let response = match (kind, snapshot) {
7408 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7409 ClientControlResponse::RoutePoll {
7410 route_channel,
7411 route_epoch,
7412 status,
7413 live: None,
7414 }
7415 }
7416 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7417 route_channel,
7418 route_epoch,
7419 status: None,
7420 live: None,
7421 },
7422 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7423 let live = self
7449 .process_liveness
7450 .as_ref()
7451 .and_then(|source| source.process_live(&module_id))
7452 .unwrap_or(true);
7453 ClientControlResponse::RoutePoll {
7454 route_channel,
7455 route_epoch,
7456 status: None,
7457 live: Some(live),
7458 }
7459 }
7460 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7461 route_channel,
7462 route_epoch,
7463 status: None,
7464 live: Some(false),
7465 },
7466 };
7467
7468 Ok(vec![control_response_body_frame(
7469 &frame,
7470 &response,
7471 "ClientControlResponse::RoutePoll",
7472 )?])
7473 }
7474
7475 pub(crate) fn observe_module_control_completion(
7476 &self,
7477 completion: ModuleControlRpcCompletion,
7478 ) -> bool {
7479 match completion {
7480 ModuleControlRpcCompletion::Unknown => false,
7481 ModuleControlRpcCompletion::Settled => true,
7482 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7483 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7484 info!(
7485 module_id = %module_id,
7486 latency_ms,
7487 "late health.check answer proves the module is alive"
7488 );
7489 match self
7490 .supervisor
7491 .record_late_health_answer(&module_id, latency_ms)
7492 {
7493 Ok(true) => {}
7494 Ok(false) => debug!(
7495 module_id = %module_id,
7496 latency_ms,
7497 "late health.check answer has no active supervisor snapshot"
7498 ),
7499 Err(err) => warn!(
7500 module_id = %module_id,
7501 latency_ms,
7502 error = %err,
7503 "failed to record late health.check answer"
7504 ),
7505 }
7506 true
7507 }
7508 }
7509 }
7510
7511 fn refuse_to_end_module_connection_for_a_client(
7532 &self,
7533 module_connection_id: ConnectionId,
7534 corr: u64,
7535 err: ForwardingError,
7536 ) -> Result<(), RouterError> {
7537 if let ForwardingError::ConnectionClosing { connection_id } = err {
7538 if connection_id != module_connection_id {
7539 warn!(
7540 module_connection_id = module_connection_id.get(),
7541 client_connection_id = connection_id.get(),
7542 corr,
7543 "dropping a route.bind response for a closing client; the module connection keeps serving"
7544 );
7545 return Ok(());
7546 }
7547 }
7548 Err(RouterError::Forwarding(err))
7549 }
7550
7551 fn handle_module_relay_response(
7552 &self,
7553 connection_id: ConnectionId,
7554 frame: Frame,
7555 ) -> Result<Vec<Frame>, RouterError> {
7556 let mut secondary_error = None;
7557 let outcome = match frame.header.ty {
7558 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7559 Ok(probe) if probe.op == "route.bind" => {
7560 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7561 Ok(ModuleControlResponse::RouteBindAck {}) => {
7562 RouteBindRelayOutcome::Accepted
7563 }
7564 Ok(other) => {
7565 let message =
7566 format!("route.bind response carried unexpected body: {other:?}");
7567 secondary_error = Some(control_error_frame(
7568 &frame,
7569 "invalid_control_body",
7570 message.clone(),
7571 )?);
7572 RouteBindRelayOutcome::ModuleGone(message)
7573 }
7574 Err(err) => {
7575 let message = format!("malformed route.bind response body: {err}");
7576 secondary_error = Some(control_error_frame(
7577 &frame,
7578 "invalid_control_body",
7579 message.clone(),
7580 )?);
7581 RouteBindRelayOutcome::ModuleGone(message)
7582 }
7583 }
7584 }
7585 Ok(probe) => {
7586 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7587 {
7588 Ok(response) => ModuleControlRpcOutcome::Response(response),
7589 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7590 "malformed {} response body: {err}",
7591 probe.op
7592 )),
7593 };
7594 let completion = self
7595 .forwarding
7596 .complete_module_control_rpc(
7597 connection_id,
7598 frame.header.corr,
7599 Some(&probe.op),
7600 outcome,
7601 )
7602 .map_err(RouterError::Forwarding)?;
7603 if !self.observe_module_control_completion(completion) {
7604 debug!(
7605 connection_id = connection_id.get(),
7606 corr = frame.header.corr,
7607 op = %probe.op,
7608 "dropping late or unknown module-control RPC response"
7609 );
7610 }
7611 return Ok(Vec::new());
7612 }
7613 Err(err) => {
7614 if let Some(expected_op) = self
7615 .forwarding
7616 .pending_module_control_op(connection_id, frame.header.corr)
7617 .map_err(RouterError::Forwarding)?
7618 {
7619 let completion = self
7620 .forwarding
7621 .complete_module_control_rpc(
7622 connection_id,
7623 frame.header.corr,
7624 None,
7625 ModuleControlRpcOutcome::MalformedResponse(format!(
7626 "malformed {expected_op} response body: {err}"
7627 )),
7628 )
7629 .map_err(RouterError::Forwarding)?;
7630 if !self.observe_module_control_completion(completion) {
7631 debug!(
7632 connection_id = connection_id.get(),
7633 corr = frame.header.corr,
7634 "dropping late malformed module-control RPC response"
7635 );
7636 }
7637 return Ok(Vec::new());
7638 }
7639 let message = format!("malformed route.bind response body: {err}");
7640 secondary_error = Some(control_error_frame(
7641 &frame,
7642 "invalid_control_body",
7643 message.clone(),
7644 )?);
7645 RouteBindRelayOutcome::ModuleGone(message)
7646 }
7647 },
7648 FrameType::Error => {
7649 if self
7650 .forwarding
7651 .pending_module_control_op(connection_id, frame.header.corr)
7652 .map_err(RouterError::Forwarding)?
7653 .is_some()
7654 {
7655 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7656 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7657 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7658 "malformed module-control ERROR body: {err}"
7659 )),
7660 };
7661 let completion = self
7662 .forwarding
7663 .complete_module_control_rpc(
7664 connection_id,
7665 frame.header.corr,
7666 None,
7667 outcome,
7668 )
7669 .map_err(RouterError::Forwarding)?;
7670 if !self.observe_module_control_completion(completion) {
7671 debug!(
7672 connection_id = connection_id.get(),
7673 corr = frame.header.corr,
7674 "dropping late or unknown module-control RPC error"
7675 );
7676 }
7677 return Ok(Vec::new());
7678 }
7679 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7680 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7681 Err(err) => {
7682 let message = format!("malformed route.bind ERROR body: {err}");
7683 secondary_error = Some(control_error_frame(
7684 &frame,
7685 "invalid_control_body",
7686 message.clone(),
7687 )?);
7688 RouteBindRelayOutcome::ModuleGone(message)
7689 }
7690 }
7691 }
7692 ty => {
7693 return Ok(vec![control_error_frame(
7694 &frame,
7695 "unsupported_control_frame",
7696 format!("unsupported module channel-0 frame {ty:?}"),
7697 )?])
7698 }
7699 };
7700
7701 let settled =
7702 self.forwarding
7703 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7704 let completion = match settled {
7705 Ok(completion) => completion,
7706 Err(err) => {
7707 self.refuse_to_end_module_connection_for_a_client(
7708 connection_id,
7709 frame.header.corr,
7710 err,
7711 )?;
7712 return Ok(secondary_error.into_iter().collect());
7713 }
7714 };
7715 if let Some(target) = completion.abandoned.as_ref() {
7716 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7717 }
7718 if !completion.settled {
7719 debug!(
7720 connection_id = connection_id.get(),
7721 corr = frame.header.corr,
7722 frame_type = ?frame.header.ty,
7723 "dropping late or unknown route.bind relay response"
7724 );
7725 }
7726 Ok(secondary_error.into_iter().collect())
7727 }
7728
7729 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7730 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7731 self.cleanup_connection_with_end_reason(
7735 connection_id,
7736 RegistrationEndReason::ExplicitGoodbye,
7737 )
7738 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7739 Ok(Vec::new())
7740 }
7741}
7742
7743impl Default for ControlHandler {
7744 fn default() -> Self {
7745 Self::new(Arc::new(Registry::default()))
7746 }
7747}
7748
7749impl crate::supervise::SwapPromotionObserver for ControlHandler {
7750 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7751 self.apply_registration_capabilities(registration);
7752 }
7753}
7754
7755fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7756 CapabilityRequirementStatus {
7757 consumer: status.consumer,
7758 capability: status.capability,
7759 need: match status.need {
7760 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7761 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7762 },
7763 verdict: status.verdict.as_str().to_string(),
7764 episode_seq: status.episode_seq,
7765 config_satisfiable: status.config_satisfiable,
7766 runtime_available: status.runtime_available,
7767 detail: status.detail,
7768 }
7769}
7770
7771fn append_capability_problem_detail(
7772 detail: Option<String>,
7773 capability_detail: Option<String>,
7774) -> Option<String> {
7775 match (detail, capability_detail) {
7776 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7777 (Some(detail), None) => Some(detail),
7778 (None, Some(capability_detail)) => Some(capability_detail),
7779 (None, None) => None,
7780 }
7781}
7782
7783fn subc_ops() -> Vec<String> {
7784 SUBC_CONTROL_OPS
7785 .iter()
7786 .map(|op| (*op).to_string())
7787 .collect()
7788}
7789
7790fn module_subc_ops() -> Vec<String> {
7791 SUBC_CONTROL_OPS
7792 .iter()
7793 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7794 .map(|op| (*op).to_string())
7795 .collect()
7796}
7797
7798#[cfg(test)]
7799fn module_baseline_control_ops() -> Vec<String> {
7800 MODULE_BASELINE_CONTROL_OPS
7801 .iter()
7802 .map(|op| (*op).to_string())
7803 .collect()
7804}
7805
7806fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7807 let mut seen = HashSet::new();
7808 let mut effective = Vec::new();
7809 for op in MODULE_BASELINE_CONTROL_OPS {
7810 if seen.insert((*op).to_string()) {
7811 effective.push((*op).to_string());
7812 }
7813 }
7814 for op in declared.unwrap_or_default() {
7815 if seen.insert(op.clone()) {
7816 effective.push(op);
7817 }
7818 }
7819 effective
7820}
7821
7822fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7823 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7824}
7825
7826fn target_module_id(target: &RouteTarget) -> &str {
7827 match target {
7828 RouteTarget::ToolProvider { module_id }
7829 | RouteTarget::ManagementSurface { module_id }
7830 | RouteTarget::InternalService { module_id, .. } => module_id,
7831 }
7832}
7833
7834fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7835 roles.iter().any(|role| match (target, role) {
7836 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7837 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7838 (
7839 RouteTarget::InternalService { service_id, .. },
7840 ProviderRole::InternalService {
7841 service_id: provided,
7842 ..
7843 },
7844 ) => service_id == provided,
7845 _ => false,
7846 })
7847}
7848
7849fn is_routable_role(role: &ProviderRole) -> bool {
7850 matches!(
7851 role,
7852 ProviderRole::ToolProvider { .. }
7853 | ProviderRole::ManagementSurface { .. }
7854 | ProviderRole::InternalService { .. }
7855 )
7856}
7857
7858#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7859enum ControlRequestBodyError {
7860 UnknownOp,
7861 InvalidBody,
7862}
7863
7864#[derive(Debug, Deserialize)]
7865struct ControlOpProbe {
7866 op: String,
7867}
7868
7869const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7872
7873fn is_known_module_push_op(body: &[u8]) -> bool {
7874 serde_json::from_slice::<ControlOpProbe>(body)
7875 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7876 .unwrap_or(false)
7877}
7878
7879fn is_known_module_request_op(body: &[u8]) -> bool {
7880 serde_json::from_slice::<ControlOpProbe>(body)
7881 .map(|probe| is_module_to_subc_op(&probe.op))
7882 .unwrap_or(false)
7883}
7884
7885fn is_module_to_subc_op(op: &str) -> bool {
7886 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7887}
7888
7889fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7890 debug!(
7891 op = %op,
7892 connection_id = connection_id.get(),
7893 corr,
7894 "control dispatch"
7895 );
7896}
7897
7898fn log_slow_control_dispatch(
7899 dispatch_started_at: Option<StdInstant>,
7900 op: &'static str,
7901 connection_id: ConnectionId,
7902 corr: u64,
7903 route_open_timings: Option<&RouteOpenTimings>,
7904) {
7905 let Some(dispatch_started_at) = dispatch_started_at else {
7906 return;
7907 };
7908 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7910 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7911 if let Some(timings) = route_open_timings {
7912 warn!(
7913 op = %op,
7914 connection_id = connection_id.get(),
7915 corr,
7916 elapsed_ms = elapsed.as_millis() as u64,
7917 module_id = ?timings.module_id,
7918 principal = %timings.principal,
7919 admission_ms = timings.admission.as_millis() as u64,
7920 project_root_ms = timings.project_root.as_millis() as u64,
7921 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7922 bind_send_ms = timings.bind_send.as_millis() as u64,
7923 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7924 "slow control dispatch"
7925 );
7926 return;
7927 }
7928 warn!(
7929 op = %op,
7930 connection_id = connection_id.get(),
7931 corr,
7932 elapsed_ms = elapsed.as_millis() as u64,
7933 "slow control dispatch"
7934 );
7935 }
7936}
7937
7938#[cfg(test)]
7940type TestCallerLookup =
7941 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7942
7943fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7945 match request {
7946 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7947 Some(PrivilegedOperation::Restart { module_id })
7948 }
7949 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7950 Some(PrivilegedOperation::Swap { module_id })
7951 }
7952 ClientControlRequest::SupervisorReload { module_id } => {
7953 Some(PrivilegedOperation::Reload { module_id })
7954 }
7955 ClientControlRequest::SupervisorRescan { preview } => {
7956 if *preview {
7957 None
7958 } else {
7959 Some(PrivilegedOperation::Rescan)
7960 }
7961 }
7962 ClientControlRequest::SupervisorRescanChecked {
7967 preview,
7968 candidate,
7969 expect,
7970 } => (!*preview && candidate.is_none() && expect.is_some())
7971 .then_some(PrivilegedOperation::Rescan),
7972 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7973 Some(PrivilegedOperation::SetEnabled {
7974 module_id,
7975 enabled: *enabled,
7976 })
7977 }
7978 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7979 Some(PrivilegedOperation::ReleaseReserved { module_id })
7980 }
7981 ClientControlRequest::ServerDescribe {}
7982 | ClientControlRequest::DaemonShutdownIntent { .. }
7985 | ClientControlRequest::CatalogList { .. }
7986 | ClientControlRequest::RouteOpen { .. }
7987 | ClientControlRequest::RoutePoll { .. }
7988 | ClientControlRequest::SupervisorList {}
7989 | ClientControlRequest::SupervisorHealth {}
7990 | ClientControlRequest::SupervisorHealthProbe { .. }
7991 | ClientControlRequest::SupervisorRoutes { .. }
7992 | ClientControlRequest::SupervisorProvenance { .. }
7993 | ClientControlRequest::SupervisorStderrTail { .. }
7994 | ClientControlRequest::SupervisorTerminals { .. }
7995 | ClientControlRequest::SupervisorSpawnSnapshot {}
7996 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7997 }
7998}
7999
8000fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
8001 match request {
8002 ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
8003 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
8004 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
8005 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
8006 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
8007 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
8008 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
8009 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
8010 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
8011 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
8012 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
8013 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
8014 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
8015 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
8016 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
8017 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
8018 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
8019 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
8020 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
8021 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
8022 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
8023 }
8024}
8025
8026struct RouteOpenCaller {
8034 principal: String,
8038 scope: String,
8041 harness: String,
8044}
8045
8046impl RouteOpenCaller {
8047 fn new(
8048 principal: Option<&Principal>,
8049 scope: Option<&ScopeSelector>,
8050 identity: &BindIdentity,
8051 ) -> Self {
8052 Self {
8053 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
8054 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
8055 harness: bounded_log_text(&identity.harness),
8056 }
8057 }
8058
8059 fn unknown() -> Self {
8062 Self {
8063 principal: "unknown".to_string(),
8064 scope: "unknown".to_string(),
8065 harness: "unknown".to_string(),
8066 }
8067 }
8068}
8069
8070const MAX_LOGGED_TEXT_CHARS: usize = 64;
8073
8074fn bounded_log_text(value: &str) -> String {
8078 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
8079 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
8080 bounded.push('…');
8081 }
8082 format!("{bounded:?}")
8083}
8084
8085fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
8091 use subc_protocol::scope::ScopeKind;
8092 let kind = match stamp.kind {
8093 ScopeKind::Head => "head",
8094 ScopeKind::Worker => "worker",
8095 ScopeKind::Ephemeral => "ephemeral",
8096 };
8097 format!(
8098 "{kind}/{}/{}@{}",
8099 principal_label(&stamp.owner),
8100 bounded_log_text(&stamp.scope_ref),
8101 stamp.scope_epoch
8102 )
8103}
8104
8105fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
8112 let epoch = selector
8113 .scope_epoch
8114 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
8115 format!(
8116 "requested/{}/{}@{epoch}",
8117 bounded_log_text(&principal_label(&selector.owner)),
8118 bounded_log_text(&selector.scope_ref),
8119 )
8120}
8121
8122fn principal_label(principal: &Principal) -> String {
8123 match principal {
8124 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
8125 Principal::Direct => "direct".to_string(),
8126 other => format!("{other:?}"),
8127 }
8128}
8129
8130fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
8131 match request {
8132 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
8133 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
8134 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
8135 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
8136 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
8137 }
8138}
8139
8140fn parse_client_control_request(
8141 body: &[u8],
8142) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
8143 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
8144 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8145 Ok(probe)
8146 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
8147 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
8148 {
8149 ControlRequestBodyError::InvalidBody
8150 }
8151 Ok(_) => ControlRequestBodyError::UnknownOp,
8152 Err(_) => ControlRequestBodyError::InvalidBody,
8153 };
8154 (err, classification)
8155 })
8156}
8157
8158fn parse_module_control_request_from_module(
8159 body: &[u8],
8160) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
8161 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
8162 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8163 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
8164 Ok(_) => ControlRequestBodyError::UnknownOp,
8165 Err(_) => ControlRequestBodyError::InvalidBody,
8166 };
8167 (err, classification)
8168 })
8169}
8170
8171#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
8172enum ProviderRoleKind {
8173 ToolProvider,
8174 PipelineStage,
8175 ManagementSurface,
8176 InternalService,
8177}
8178
8179fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
8180 match role {
8181 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
8182 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
8183 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
8184 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
8185 }
8186}
8187
8188fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
8189 roles.iter().map(provider_role_kind).collect()
8190}
8191
8192const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
8195
8196#[derive(Debug, Default, PartialEq, Eq)]
8198struct ScopeOutcomeCounts {
8199 created: usize,
8200 replaced: usize,
8201 updated: usize,
8202 unchanged: usize,
8203 refused: usize,
8204}
8205
8206impl ScopeOutcomeCounts {
8207 fn of(results: &[ScopeRecordResult]) -> Self {
8208 let mut counts = Self::default();
8209 for result in results {
8210 let slot = match result.outcome {
8211 ScopeRecordOutcome::Created => &mut counts.created,
8212 ScopeRecordOutcome::Replaced => &mut counts.replaced,
8213 ScopeRecordOutcome::Updated => &mut counts.updated,
8214 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
8215 ScopeRecordOutcome::Refused => &mut counts.refused,
8216 };
8217 *slot += 1;
8218 }
8219 counts
8220 }
8221}
8222
8223#[cfg(test)]
8224mod scope_outcome_count_tests {
8225 use super::*;
8226
8227 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
8228 ScopeRecordResult {
8229 scope_ref: "r".to_string(),
8230 scope_epoch: 1,
8231 outcome,
8232 code: None,
8233 message: None,
8234 version: None,
8235 parent_state: None,
8236 }
8237 }
8238
8239 #[test]
8242 fn every_outcome_is_counted_in_its_own_field() {
8243 let results = [
8244 result(ScopeRecordOutcome::Created),
8245 result(ScopeRecordOutcome::Created),
8246 result(ScopeRecordOutcome::Replaced),
8247 result(ScopeRecordOutcome::Updated),
8248 result(ScopeRecordOutcome::Unchanged),
8249 result(ScopeRecordOutcome::Refused),
8250 result(ScopeRecordOutcome::Refused),
8251 result(ScopeRecordOutcome::Refused),
8252 ];
8253 assert_eq!(
8254 ScopeOutcomeCounts::of(&results),
8255 ScopeOutcomeCounts {
8256 created: 2,
8257 replaced: 1,
8258 updated: 1,
8259 unchanged: 1,
8260 refused: 3,
8261 }
8262 );
8263 }
8264}
8265
8266fn capability_census_trigger(
8270 old: Option<&CapabilityDeclarations>,
8271 new: Option<&CapabilityDeclarations>,
8272) -> bool {
8273 let old_provides = old
8274 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
8275 .unwrap_or_default();
8276 let old_denies = old
8277 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
8278 .unwrap_or_default();
8279 let new = new.cloned().unwrap_or(CapabilityDeclarations {
8280 provides: Vec::new(),
8281 requires: Vec::new(),
8282 must_never_reach: Vec::new(),
8283 });
8284
8285 new.provides
8286 .iter()
8287 .any(|capability| !old_provides.contains(capability))
8288 || new
8289 .must_never_reach
8290 .iter()
8291 .any(|capability| !old_denies.contains(capability))
8292}
8293
8294fn denied_capability<'a>(
8297 opening_manifest: &'a ModuleManifest,
8298 target_manifest: &ModuleManifest,
8299) -> Option<&'a str> {
8300 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
8301 let target_capabilities = target_manifest.capabilities.as_ref()?;
8302 opening_capabilities
8303 .must_never_reach
8304 .iter()
8305 .find(|denied| {
8306 target_capabilities
8307 .provides
8308 .iter()
8309 .any(|provided| provided == *denied)
8310 })
8311 .map(String::as_str)
8312}
8313
8314fn catalog_update_frozen_field_message(
8315 registered: &ModuleManifest,
8316 provides: &[ProviderRole],
8317) -> Option<String> {
8318 let old_has_provides = !registered.provides.is_empty();
8319 let new_has_provides = !provides.is_empty();
8320 if old_has_provides != new_has_provides {
8321 return Some(format!(
8322 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
8323 registered.module_id
8324 ));
8325 }
8326
8327 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
8328 return Some(format!(
8329 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
8330 registered.module_id
8331 ));
8332 }
8333
8334 let registered_concurrency = manifest_concurrency(registered);
8335 let mut candidate = registered.clone();
8336 candidate.provides = provides.to_vec();
8337 let candidate_concurrency = manifest_concurrency(&candidate);
8338 if candidate_concurrency != registered_concurrency {
8339 return Some(format!(
8340 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
8341 registered.module_id, registered_concurrency, candidate_concurrency
8342 ));
8343 }
8344
8345 None
8348}
8349
8350fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
8351 manifest.provides.iter().any(is_routable_role)
8352}
8353
8354fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
8360 manifest
8361 .provides
8362 .iter()
8363 .find_map(|provider| match provider {
8364 ProviderRole::ToolProvider { concurrency, .. }
8365 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
8366 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
8367 })
8368 .unwrap_or(Concurrency::ModuleManaged)
8369}
8370
8371fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
8378 let has_management_surface = manifest
8379 .provides
8380 .iter()
8381 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
8382 if !has_management_surface {
8383 return false;
8384 }
8385 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
8386 return false;
8387 };
8388 let Some(provides) = raw
8389 .get("manifest")
8390 .and_then(|manifest| manifest.get("provides"))
8391 .and_then(serde_json::Value::as_array)
8392 else {
8393 return false;
8394 };
8395 provides.iter().any(|role| {
8400 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8401 && role.get("concurrency").is_none()
8402 })
8403}
8404
8405fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8406 if peer_version != PROTOCOL_VERSION {
8407 return Err(format!(
8408 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8409 ));
8410 }
8411 Ok(PROTOCOL_VERSION)
8412}
8413
8414fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8415 Frame::build_with_version(
8416 response_version(frame),
8417 FrameType::Pong,
8418 frame.header.flags,
8419 0,
8420 0,
8421 frame.header.corr,
8422 Vec::new(),
8423 )
8424 .map_err(RouterError::FrameBuild)
8425}
8426
8427pub(crate) fn control_error_frame(
8428 frame: &Frame,
8429 code: &'static str,
8430 message: impl Into<String>,
8431) -> Result<Frame, RouterError> {
8432 control_error_body_frame(
8433 frame,
8434 ErrorBody {
8435 code: code.to_string(),
8436 message: message.into(),
8437 detail: None,
8438 },
8439 )
8440}
8441
8442fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8450 control_error_frame_with_flags(frame, error, daemon_error_flags())
8451}
8452
8453fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8459 control_error_frame_with_flags(frame, error, control_flags())
8460}
8461
8462fn control_error_frame_with_flags(
8463 frame: &Frame,
8464 error: ErrorBody,
8465 flags: Flags,
8466) -> Result<Frame, RouterError> {
8467 let body = serde_json::to_vec(&error).map_err(|err| {
8468 RouterError::backend(
8469 0,
8470 frame.header.corr,
8471 format!("failed to encode control ERROR: {err}"),
8472 )
8473 })?;
8474
8475 Frame::build_with_version(
8476 response_version(frame),
8477 FrameType::Error,
8478 flags,
8479 0,
8480 0,
8481 frame.header.corr,
8482 body,
8483 )
8484 .map_err(RouterError::FrameBuild)
8485}
8486
8487pub(crate) fn control_response_body_frame<T: Serialize>(
8488 frame: &Frame,
8489 reply: &T,
8490 label: &'static str,
8491) -> Result<Frame, RouterError> {
8492 let body = serde_json::to_vec(reply).map_err(|err| {
8493 RouterError::backend(
8494 0,
8495 frame.header.corr,
8496 format!("failed to encode {label}: {err}"),
8497 )
8498 })?;
8499
8500 Frame::build_with_version(
8501 response_version(frame),
8502 FrameType::Response,
8503 control_flags(),
8504 0,
8505 0,
8506 frame.header.corr,
8507 body,
8508 )
8509 .map_err(RouterError::FrameBuild)
8510}
8511
8512fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8542 match err {
8543 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8544 ForwardingError::NoModuleConnection => "target_unavailable",
8545 ForwardingError::ModuleReloading { .. } => "module_reloading",
8546 ForwardingError::ClientRouteChannelExhausted { .. }
8547 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8548 ForwardingError::StaleModuleEndpoint
8549 | ForwardingError::UnknownReservation { .. }
8550 | ForwardingError::ConnectionClosing { .. }
8551 | ForwardingError::ClientEgressClosed { .. }
8552 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8553 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8556 ForwardingError::RelayCorrelationExhausted
8557 | ForwardingError::RouteOpenBuild(_)
8558 | ForwardingError::Poisoned => "forwarding_error",
8559 }
8560}
8561
8562fn response_version(frame: &Frame) -> u8 {
8563 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8564 frame.header.ver
8565 } else {
8566 PROTOCOL_VERSION
8567 }
8568}
8569
8570fn control_flags() -> Flags {
8571 Flags::new(false, Priority::Passive, false)
8572}
8573
8574fn send_goodbye_target_best_effort(
8579 counters: &DaemonCounters,
8580 target: &GoodbyeTarget,
8581 context: &'static str,
8582) {
8583 let Ok(frame) = Frame::build_with_version(
8584 target.negotiated_ver,
8585 FrameType::Goodbye,
8586 control_flags(),
8587 target.channel,
8588 target.epoch,
8589 0,
8590 Vec::new(),
8591 ) else {
8592 return;
8593 };
8594 crate::forwarding::send_module_route_goodbye(
8595 counters,
8596 &target.sink,
8597 frame,
8598 target.module_id.as_deref(),
8599 context,
8600 );
8601}
8602
8603pub(crate) fn send_route_control_pushes(
8604 forwarding: &ForwardingTable,
8605 routes: Vec<EndpointRoute>,
8606 push: ClientControlPush,
8607) {
8608 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8609 for route in routes {
8610 let target = route.goodbye_target;
8611 if let Some((existing, channels)) = targets
8612 .iter_mut()
8613 .find(|(existing, _)| existing.connection_id == target.connection_id)
8614 {
8615 debug_assert_eq!(
8616 existing.negotiated_ver, target.negotiated_ver,
8617 "one connection cannot negotiate multiple frame versions"
8618 );
8619 if !channels.contains(&target.channel) {
8620 channels.push(target.channel);
8621 }
8622 continue;
8623 }
8624 let channel = target.channel;
8625 targets.push((target, vec![channel]));
8626 }
8627 for (target, mut channels) in targets {
8628 channels.sort_unstable();
8629 let mut push = push.clone();
8630 match &mut push {
8631 ClientControlPush::RouteClosing {
8632 channels: covered, ..
8633 }
8634 | ClientControlPush::RouteClosed {
8635 channels: covered, ..
8636 } => *covered = channels,
8637 }
8638 let body = match serde_json::to_vec(&push) {
8639 Ok(body) => body,
8640 Err(err) => {
8641 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8642 continue;
8643 }
8644 };
8645 let frame = match Frame::build_with_version(
8646 target.negotiated_ver,
8647 FrameType::Push,
8648 control_flags(),
8649 0,
8650 0,
8651 0,
8652 body.clone(),
8653 ) {
8654 Ok(frame) => frame,
8655 Err(err) => {
8656 warn!(
8657 route_channel = target.channel,
8658 error = %err,
8659 "failed to build route lifecycle control PUSH frame"
8660 );
8661 continue;
8662 }
8663 };
8664 if let Err(err) = target.sink.try_send(frame) {
8665 if target.close_on_delivery_failure() {
8666 warn!(
8667 target_connection_id = target.connection_id.get(),
8668 route_channel = target.channel,
8669 error = %err,
8670 "route lifecycle control PUSH was not delivered to client; closing target connection"
8671 );
8672 let _ = forwarding.escalate_client_delivery_failure(
8673 target.connection_id,
8674 target.channel,
8675 target.epoch,
8676 CloseReason::new(
8677 "route_lifecycle_push_delivery_failed",
8678 format!(
8679 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8680 target.channel
8681 ),
8682 ),
8683 crate::forwarding::UndeliveredFrame {
8684 module_id: target.module_id.as_deref(),
8685 sink: &target.sink,
8686 },
8687 );
8688 }
8689 }
8690 }
8691}
8692
8693#[cfg(test)]
8694mod tests {
8695 #[tokio::test]
8696 async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
8697 let registry = Arc::new(Registry::default());
8698 let handler = ControlHandler::new(registry);
8699 assert_eq!(
8700 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8701 None
8702 );
8703 let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
8704 hello_via_sink(
8705 &handler,
8706 &ctx,
8707 &mut rx,
8708 hello_frame("daemon", PROTOCOL_VERSION, 1),
8709 )
8710 .await;
8711 assert_eq!(
8712 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8713 Some("daemon".into())
8714 );
8715 assert_eq!(
8716 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
8717 None
8718 );
8719 assert_eq!(
8720 handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
8721 None,
8722 "a registered module is not necessarily supervised"
8723 );
8724 assert_eq!(
8725 handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
8726 None,
8727 "aggregate provenance has no single target"
8728 );
8729 assert_eq!(
8730 handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
8731 None
8732 );
8733 }
8734
8735 mod peer_manifest {
8736 include!("peer_manifest_tests.rs");
8737 }
8738 mod activity_read {
8739 include!("activity_read_tests.rs");
8740 }
8741
8742 mod lifecycle_activity {
8743 include!("control_lifecycle_tests.rs");
8744 }
8745 #[cfg(unix)]
8746 #[tokio::test]
8747 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8748 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8749 let path = dir.join("subc.jsonc");
8750 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8751 "program":"/bin/sleep","args":["60"],"protocol":"none",
8752 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8753 }}}).to_string()).unwrap();
8754 let mut configured = crate::daemon_config::load(&path)
8755 .unwrap()
8756 .unwrap()
8757 .modules
8758 .pop()
8759 .unwrap();
8760 let registry = std::sync::Arc::new(crate::Registry::default());
8761 let handle = crate::SupervisorHandle::new();
8762 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8763 .with_handle(handle.clone());
8764 let module = supervisor
8765 .supervise_configured_with_health(
8766 configured.module_spec(),
8767 true,
8768 configured.health.clone(),
8769 None,
8770 configured.restart,
8771 )
8772 .unwrap();
8773 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8774 let before = module.status().unwrap().pid;
8775 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8776 configured.health.cadence = std::time::Duration::from_secs(3600);
8777 let health_only = handler
8778 .reconcile_supervised_modules(
8779 &supervisor,
8780 vec![configured.clone()],
8781 false,
8782 Principal::Direct,
8783 )
8784 .await
8785 .unwrap();
8786 assert!(
8787 health_only.0.changed_pending_reload.is_empty(),
8788 "health policy is already applied live"
8789 );
8790 assert_eq!(module.status().unwrap().pid, before);
8791 assert_eq!(
8792 module.configuration().unwrap().1.http,
8793 configured.health.http
8794 );
8795 configured.args = vec!["61".into()];
8796 let launch = handler
8797 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8798 .await
8799 .unwrap();
8800 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8801 assert_eq!(
8802 module.status().unwrap().pid,
8803 before,
8804 "a launch edit is stored until reload"
8805 );
8806 module.drain().await.unwrap();
8807 }
8808 use cortexkit_test_support::ScratchDir;
8809 use std::{
8810 collections::BTreeMap,
8811 fmt,
8812 path::PathBuf,
8813 sync::{Arc, Mutex},
8814 time::Duration,
8815 };
8816
8817 use serde_json::{json, Value};
8818 use subc_protocol::{
8819 manifest::{
8820 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8821 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8822 },
8823 session::HealthStatus,
8824 FrameType,
8825 };
8826
8827 use super::*;
8828 use crate::{
8829 forwarding::{DataRoute, DataRouteState},
8830 registry::ChannelState,
8831 router::FrameSink,
8832 stderr_tail::DEFAULT_MAX_LINE_BYTES,
8833 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8834 RouteCtx, Router,
8835 };
8836 use tokio::{
8837 sync::mpsc,
8838 time::{sleep, Instant},
8839 };
8840 use tracing::{
8841 field::{Field, Visit},
8842 Event, Subscriber,
8843 };
8844 use tracing_subscriber::{layer::Context, prelude::*, Layer};
8845
8846 fn fake_aft_stub_path() -> PathBuf {
8865 let mut path = std::env::current_exe().expect("current_exe available in tests");
8866 path.pop(); path.pop(); path.push(if cfg!(windows) {
8869 "fake-aft-stub.exe"
8870 } else {
8871 "fake-aft-stub"
8872 });
8873 assert!(
8874 path.exists(),
8875 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8876 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8877 path.display()
8878 );
8879 path
8880 }
8881
8882 fn client_retries(code: &str) -> bool {
8887 subc_protocol::error_codes::is_retryable_route_open(code)
8888 }
8889
8890 #[test]
8901 fn retryability_of_forwarding_codes_matches_the_failure() {
8902 let transient = [
8905 ForwardingError::NoModuleConnection,
8906 ForwardingError::ModuleReloading {
8907 module_id: "m".into(),
8908 },
8909 ForwardingError::StaleModuleEndpoint,
8910 ForwardingError::UnknownReservation {
8911 client_channel: 1,
8912 module_channel: 1,
8913 },
8914 ForwardingError::ConnectionClosing {
8915 connection_id: ConnectionId::new(1),
8916 },
8917 ForwardingError::ClientEgressClosed {
8918 connection_id: ConnectionId::new(1),
8919 },
8920 ForwardingError::ModuleEgressUnavailable {
8921 connection_id: ConnectionId::new(1),
8922 },
8923 ];
8924 for err in transient {
8925 let code = forwarding_error_code(&err);
8926 assert!(
8927 client_retries(code),
8928 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8929 );
8930 }
8931
8932 let permanent = [
8937 ForwardingError::ConnectionRoleConflict {
8938 connection_id: ConnectionId::new(1),
8939 },
8940 ForwardingError::ClientRouteChannelExhausted {
8941 connection_id: ConnectionId::new(1),
8942 },
8943 ForwardingError::ModuleRouteChannelExhausted {
8944 endpoint: ModuleEndpointId {
8945 connection_id: ConnectionId::new(1),
8946 generation: 1,
8947 },
8948 },
8949 ForwardingError::RelayCorrelationExhausted,
8950 ForwardingError::RouteOpenBuild("x".into()),
8951 ForwardingError::Poisoned,
8952 ];
8953 for err in permanent {
8954 let code = forwarding_error_code(&err);
8955 assert!(
8956 !client_retries(code),
8957 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8958 );
8959 }
8960 }
8961
8962 #[tokio::test]
8990 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8991 let handler = ControlHandler::default();
8992 let frame =
8993 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8994
8995 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8999 assert_eq!(
9000 stamped,
9001 Principal::Direct,
9002 "a caller that proved nothing must not be stamped as a supervised module"
9003 );
9004
9005 let forged = handler
9010 .route_open_principal(
9011 &frame,
9012 Some(ConsumerIdentity {
9013 module_id: "aft".to_string(),
9014 launch_nonce: "not-a-real-nonce".to_string(),
9015 }),
9016 )
9017 .unwrap();
9018 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
9019 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
9020 }
9021
9022 #[test]
9039 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
9040 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
9041 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
9042 let ClientControlRequest::RouteOpen {
9043 consumer_identity, ..
9044 } = parsed
9045 else {
9046 panic!("route.open body must parse as RouteOpen");
9047 };
9048 assert_eq!(
9049 consumer_identity,
9050 Some(ConsumerIdentity {
9051 module_id: "aft".to_string(),
9052 launch_nonce: "n".to_string(),
9053 }),
9054 "the wire field name must reach the value route_open_principal reads"
9055 );
9056 }
9057
9058 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
9059 ModuleManifest::builder(module_id, "0.1.0")
9060 .protocol_ver(protocol_ver)
9061 .provides(vec![ProviderRole::ToolProvider {
9062 tools: vec![Tool {
9063 name: "read".to_string(),
9064 description: None,
9065 execution_mode: ExecutionMode::Pure,
9066 schema: json!({"type": "object"}),
9067 }],
9068 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
9069 concurrency: Concurrency::ModuleManaged,
9070 emits_push: true,
9071 sub_supervises: true,
9072 }])
9073 .build()
9074 }
9075
9076 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
9077 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
9078 }
9079
9080 fn hello_frame_with_control_ops(
9081 module_id: &str,
9082 protocol_ver: u8,
9083 corr: u64,
9084 control_ops: Option<Vec<String>>,
9085 ) -> Frame {
9086 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
9087 }
9088
9089 fn hello_frame_with_nonce(
9090 module_id: &str,
9091 protocol_ver: u8,
9092 corr: u64,
9093 launch_nonce: Option<&str>,
9094 ) -> Frame {
9095 hello_frame_full(
9096 module_id,
9097 protocol_ver,
9098 corr,
9099 None,
9100 launch_nonce.map(ToOwned::to_owned),
9101 )
9102 }
9103
9104 fn hello_frame_full(
9105 module_id: &str,
9106 protocol_ver: u8,
9107 corr: u64,
9108 control_ops: Option<Vec<String>>,
9109 launch_nonce: Option<String>,
9110 ) -> Frame {
9111 let body = serde_json::to_vec(&ModuleHelloBody {
9112 manifest: manifest(module_id, protocol_ver),
9113 protocol_ver,
9114 control_ops,
9115 launch_nonce,
9116 })
9117 .unwrap();
9118 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9119 }
9120
9121 fn hello_frame_with_rich_ui(
9122 module_id: &str,
9123 protocol_ver: u8,
9124 corr: u64,
9125 launch_nonce: Option<&str>,
9126 rich_ui: Value,
9127 ) -> Frame {
9128 let mut body = serde_json::to_value(ModuleHelloBody {
9129 manifest: manifest(module_id, protocol_ver),
9130 protocol_ver,
9131 control_ops: None,
9132 launch_nonce: launch_nonce.map(str::to_string),
9133 })
9134 .unwrap();
9135 body["manifest"]["rich_ui"] = rich_ui;
9136 Frame::build(
9137 FrameType::Hello,
9138 control_flags(),
9139 0,
9140 0,
9141 corr,
9142 serde_json::to_vec(&body).unwrap(),
9143 )
9144 .unwrap()
9145 }
9146
9147 fn non_routable_hello_frame_with_control_ops(
9148 module_id: &str,
9149 corr: u64,
9150 control_ops: Option<Vec<String>>,
9151 ) -> Frame {
9152 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9153 manifest.provides.clear();
9154 let body = serde_json::to_vec(&ModuleHelloBody {
9155 manifest,
9156 protocol_ver: PROTOCOL_VERSION,
9157 control_ops,
9158 launch_nonce: None,
9159 })
9160 .unwrap();
9161 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9162 }
9163
9164 fn capability_grammar_hello_frame(
9165 capabilities: Value,
9166 runtime_computed: Option<Value>,
9167 corr: u64,
9168 ) -> Frame {
9169 let mut body = serde_json::to_value(ModuleHelloBody {
9170 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
9171 protocol_ver: PROTOCOL_VERSION,
9172 control_ops: None,
9173 launch_nonce: None,
9174 })
9175 .expect("HELLO body serializes");
9176 body["manifest"]["capabilities"] = capabilities;
9177 if let Some(runtime_computed) = runtime_computed {
9178 body["runtime_computed"] = runtime_computed;
9179 }
9180 Frame::build(
9181 FrameType::Hello,
9182 control_flags(),
9183 0,
9184 0,
9185 corr,
9186 serde_json::to_vec(&body).expect("HELLO body reserializes"),
9187 )
9188 .expect("HELLO frame builds")
9189 }
9190
9191 fn channel_request(channel: u16, corr: u64) -> Frame {
9192 Frame::build(
9193 FrameType::Request,
9194 Flags::new(true, Priority::Interactive, false),
9195 channel,
9196 0,
9197 corr,
9198 b"opaque".to_vec(),
9199 )
9200 .unwrap()
9201 }
9202
9203 fn route_ctx(
9204 connection_id: ConnectionId,
9205 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
9206 let (tx, rx) = mpsc::channel(8);
9207 (
9208 RouteCtx {
9209 connection_id,
9210 egress: FrameSink::new(tx),
9211 },
9212 rx,
9213 )
9214 }
9215
9216 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
9217 serde_json::from_slice(&frame.body).unwrap()
9218 }
9219
9220 async fn hello_via_sink(
9225 handler: &ControlHandler,
9226 ctx: &RouteCtx,
9227 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9228 hello: Frame,
9229 ) -> Frame {
9230 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
9231 assert!(
9232 replies.is_empty(),
9233 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
9234 );
9235 let ack = rx
9236 .try_recv()
9237 .expect("HELLO_ACK is queued on the module sink")
9238 .frame;
9239 assert_eq!(ack.header.ty, FrameType::HelloAck);
9240 ack
9241 }
9242
9243 fn parse_error(frame: &Frame) -> Value {
9244 serde_json::from_slice(&frame.body).unwrap()
9245 }
9246
9247 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
9248 serde_json::from_slice(&frame.body).unwrap()
9249 }
9250
9251 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
9252 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
9253 route_channel,
9254 route_epoch: 0,
9255 kind,
9256 })
9257 .unwrap();
9258 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9259 }
9260
9261 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
9262 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
9263 module_id: module_id.to_string(),
9264 })
9265 .unwrap();
9266 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9267 }
9268
9269 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
9270 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
9271 }
9272
9273 fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
9275 let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
9276 let mut frame = route_open_frame(corr, module_id, root);
9277 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
9278 body["identity"]["project_root"] = serde_json::json!(path.as_path());
9279 frame.body = serde_json::to_vec(&body).unwrap();
9280 frame
9281 }
9282
9283 fn route_open_frame_with_consumer_capabilities(
9284 corr: u64,
9285 module_id: &str,
9286 _project_root: ScratchDir,
9287 consumer_capabilities: Option<Vec<String>>,
9288 ) -> Frame {
9289 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9290 busy_refusals: None,
9291 target: RouteTarget::ToolProvider {
9292 module_id: module_id.to_string(),
9293 },
9294 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
9295 consumer_identity: None,
9296 consumer_capabilities,
9297 role_versions: None,
9298 admission_facts: None,
9299 scope: None,
9300 })
9301 .unwrap();
9302 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9303 }
9304
9305 fn route_open_frame_with_role_versions(
9306 corr: u64,
9307 module_id: &str,
9308 _project_root: ScratchDir,
9309 role_versions: Option<BTreeMap<String, String>>,
9310 ) -> Frame {
9311 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9312 busy_refusals: None,
9313 target: RouteTarget::ToolProvider {
9314 module_id: module_id.to_string(),
9315 },
9316 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9317 consumer_identity: None,
9318 consumer_capabilities: None,
9319 role_versions,
9320 admission_facts: None,
9321 scope: None,
9322 })
9323 .unwrap();
9324 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9325 }
9326
9327 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
9328 entries
9329 .iter()
9330 .map(|(role, version)| (role.to_string(), version.to_string()))
9331 .collect()
9332 }
9333
9334 fn route_open_frame_with_admission_facts(
9335 corr: u64,
9336 module_id: &str,
9337 _project_root: ScratchDir,
9338 consumer_identity: Option<subc_control::ConsumerIdentity>,
9339 facts: Option<Value>,
9340 ) -> Frame {
9341 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9342 busy_refusals: None,
9343 target: RouteTarget::ToolProvider {
9344 module_id: module_id.to_string(),
9345 },
9346 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9347 consumer_identity,
9348 consumer_capabilities: None,
9349 role_versions: None,
9350 admission_facts: facts,
9351 scope: None,
9352 })
9353 .unwrap();
9354 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9355 }
9356
9357 #[derive(Clone, Default)]
9358 struct EventCapture {
9359 events: Arc<Mutex<Vec<CapturedEvent>>>,
9360 }
9361
9362 #[derive(Clone, Debug)]
9363 struct CapturedEvent {
9364 target: String,
9365 level: tracing::Level,
9366 fields: BTreeMap<String, String>,
9367 }
9368
9369 impl EventCapture {
9370 fn events(&self) -> Vec<CapturedEvent> {
9371 self.events.lock().unwrap().clone()
9372 }
9373 }
9374
9375 impl<S> Layer<S> for EventCapture
9376 where
9377 S: Subscriber,
9378 {
9379 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
9380 let mut visitor = EventFieldVisitor::default();
9381 event.record(&mut visitor);
9382 self.events.lock().unwrap().push(CapturedEvent {
9383 target: event.metadata().target().to_string(),
9384 level: *event.metadata().level(),
9385 fields: visitor.fields,
9386 });
9387 }
9388 }
9389
9390 #[derive(Default)]
9391 struct EventFieldVisitor {
9392 fields: BTreeMap<String, String>,
9393 }
9394
9395 impl Visit for EventFieldVisitor {
9396 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
9397 self.fields
9398 .insert(field.name().to_string(), format!("{value:?}"));
9399 }
9400 }
9401
9402 fn health_response(corr: u64, status: HealthStatus) -> Frame {
9403 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
9404 status,
9405 detail: Some("warming".to_string()),
9406 metrics: Some(json!({"queue_depth": 3})),
9407 })
9408 .unwrap();
9409 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9410 }
9411
9412 fn route_bind_ack(corr: u64) -> Frame {
9413 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
9414 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9415 }
9416
9417 fn unique_project_root(label: &str) -> ScratchDir {
9418 ScratchDir::new(label)
9419 }
9420
9421 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
9422 match parse_route_poll(frame) {
9423 ClientControlResponse::RoutePoll {
9424 status: None,
9425 live: Some(live),
9426 ..
9427 } => assert_eq!(live, expected_live),
9428 other => panic!("unexpected route.poll response: {other:?}"),
9429 }
9430 }
9431
9432 fn bind_liveness_route(
9433 registry: &Registry,
9434 forwarding: &ForwardingTable,
9435 module_id: &str,
9436 ) -> (RouteCtx, u16, u32) {
9437 let module_connection = ConnectionId::new(101);
9438 let client_connection = ConnectionId::new(202);
9439 let registration = registry
9440 .register_with_control_ops(
9441 manifest(module_id, PROTOCOL_VERSION),
9442 PROTOCOL_VERSION,
9443 module_connection,
9444 module_baseline_control_ops(),
9445 )
9446 .unwrap();
9447 let (module_tx, _module_rx) = mpsc::channel(8);
9448 let endpoint = forwarding
9449 .register_module_connection(
9450 module_connection,
9451 module_id.to_string(),
9452 PROTOCOL_VERSION,
9453 manifest_concurrency(®istration.manifest),
9454 FrameSink::new(module_tx),
9455 )
9456 .unwrap();
9457 let (client_ctx, _client_rx) = route_ctx(client_connection);
9458 let pending = forwarding
9459 .begin_route_bind_relay_for_test(
9460 client_connection,
9461 client_ctx.egress.clone(),
9462 1,
9463 module_id,
9464 )
9465 .unwrap();
9466 assert_eq!(pending.endpoint, endpoint);
9467 let route_channel = pending.client_channel;
9468 let route_epoch = pending.client_epoch;
9469 forwarding
9470 .complete_pending_relay(
9471 module_connection,
9472 pending.corr,
9473 RouteBindRelayOutcome::Accepted,
9474 )
9475 .unwrap();
9476 (client_ctx, route_channel, route_epoch)
9477 }
9478
9479 struct FakeProcessLiveness {
9480 live: Option<bool>,
9481 }
9482
9483 impl ModuleProcessLiveness for FakeProcessLiveness {
9484 fn process_live(&self, _module_id: &str) -> Option<bool> {
9485 self.live
9486 }
9487 }
9488
9489 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9490 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9491 {
9492 let registry = Arc::new(Registry::default());
9493 let supervisor_handle = SupervisorHandle::new();
9494 let supervisor = Supervisor::new_for_test(
9495 Arc::clone(®istry),
9496 RestartPolicy::new(1, Duration::from_millis(10)),
9497 )
9498 .with_handle(supervisor_handle.clone());
9499 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9500 let module = supervisor
9501 .spawn(ModuleSpec {
9502 carry_containment: false,
9503 module_id: "stderr-tail-wire".to_string(),
9504 program: fake_aft_stub_path(),
9505 args: Vec::new(),
9506 env: vec![
9507 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9508 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9509 ],
9510 reserved: false,
9511 reserved_prefixes: Vec::new(),
9512 protocol: ModuleProtocol::Subc,
9513 overlap: Default::default(),
9514 })
9515 .unwrap();
9516
9517 let deadline = Instant::now() + Duration::from_secs(5);
9518 loop {
9519 let tail = module.stderr_tail(None, None);
9520 if tail
9521 .entries
9522 .iter()
9523 .any(|entry| matches!(entry, TailEntry::ProcessStart))
9524 && tail.entries.iter().any(|entry| {
9525 matches!(
9526 entry,
9527 TailEntry::Line {
9528 truncated: true,
9529 ..
9530 }
9531 )
9532 })
9533 {
9534 break;
9535 }
9536 assert!(
9537 Instant::now() < deadline,
9538 "module did not produce a truncated line and restart boundary: {tail:?}"
9539 );
9540 sleep(Duration::from_millis(10)).await;
9541 }
9542
9543 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9544 let request = ClientControlRequest::SupervisorStderrTail {
9545 module_id: "stderr-tail-wire".to_string(),
9546 max_lines: None,
9547 max_bytes: None,
9548 };
9549 let frame = Frame::build(
9550 FrameType::Request,
9551 control_flags(),
9552 0,
9553 0,
9554 1,
9555 serde_json::to_vec(&request).unwrap(),
9556 )
9557 .unwrap();
9558 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9559 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9560 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9561 serde_json::from_slice(&responses[0].body).unwrap()
9562 else {
9563 panic!("expected supervisor.stderr_tail response");
9564 };
9565
9566 assert!(
9567 tail.entries
9568 .iter()
9569 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9570 "the control response lost the restart boundary"
9571 );
9572 let Some(StderrTailEntry::Line {
9573 text,
9574 truncated,
9575 at_ms,
9576 }) = tail.entries.iter().find(|entry| {
9577 matches!(
9578 entry,
9579 StderrTailEntry::Line {
9580 truncated: true,
9581 ..
9582 }
9583 )
9584 })
9585 else {
9586 panic!("the control response lost the truncated line");
9587 };
9588 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9589 assert!(*truncated);
9590 assert!(
9591 at_ms.is_some(),
9592 "the control response lost the line's capture time"
9593 );
9594 }
9595
9596 #[tokio::test(flavor = "current_thread")]
9601 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9602 let dir = ScratchDir::new("terminals-off-worker");
9603 let journal_path = dir.join("terminals.jsonl");
9604 let registry = Arc::new(Registry::default());
9605 let supervisor_handle = SupervisorHandle::new();
9606 let supervisor =
9607 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9608 .with_handle(supervisor_handle.clone())
9609 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9610 let module = supervisor
9611 .spawn(ModuleSpec {
9612 carry_containment: false,
9613 module_id: "terminal-off-worker".to_string(),
9614 program: fake_aft_stub_path(),
9615 args: Vec::new(),
9616 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9617 reserved: false,
9618 reserved_prefixes: Vec::new(),
9619 protocol: ModuleProtocol::Subc,
9620 overlap: Default::default(),
9621 })
9622 .unwrap();
9623 let deadline = Instant::now() + Duration::from_secs(5);
9624 while module.terminal_history().entries.len() != 2 {
9625 assert!(Instant::now() < deadline, "module did not record two exits");
9626 sleep(Duration::from_millis(10)).await;
9627 }
9628
9629 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9630 let handler =
9631 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
9632 let frame = Frame::build(
9633 FrameType::Request,
9634 control_flags(),
9635 0,
9636 0,
9637 1,
9638 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9639 module_id: "terminal-off-worker".to_string(),
9640 })
9641 .unwrap(),
9642 )
9643 .unwrap();
9644 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9645 let spawned_at = std::time::Instant::now();
9646 let read = tokio::spawn({
9647 let handler = Arc::clone(&handler);
9648 async move { handler.handle_control_frame(&ctx, frame).await }
9649 });
9650 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9653 .await
9654 .unwrap()
9655 .expect("the history read reached its pause");
9656 let elapsed = spawned_at.elapsed();
9657 assert!(
9658 elapsed < Duration::from_secs(2) && !read.is_finished(),
9659 "this task could not run while the history read was paused \
9660 (resumed after {elapsed:?}, read finished: {})",
9661 read.is_finished()
9662 );
9663
9664 drop(release);
9665 let responses = read.await.unwrap().unwrap();
9666 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9667 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9668 panic!("expected supervisor.terminals response");
9669 };
9670 assert_eq!(terminals.entries.len(), 2);
9671 assert_eq!(terminals.journal_skipped_lines, 0);
9672 assert_eq!(terminals.journal_read_errors, 0);
9673 }
9674
9675 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9676 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9677 let registry = Arc::new(Registry::default());
9678 let supervisor_handle = SupervisorHandle::new();
9679 let supervisor =
9680 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9681 .with_handle(supervisor_handle.clone());
9682 let module = supervisor
9683 .spawn(ModuleSpec {
9684 carry_containment: false,
9685 module_id: "terminal-golden".to_string(),
9686 program: fake_aft_stub_path(),
9687 args: Vec::new(),
9688 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9689 reserved: false,
9690 reserved_prefixes: Vec::new(),
9691 protocol: ModuleProtocol::Subc,
9692 overlap: Default::default(),
9693 })
9694 .unwrap();
9695
9696 let deadline = Instant::now() + Duration::from_secs(5);
9697 while module.terminal_history().entries.len() != 2 {
9698 assert!(
9699 Instant::now() < deadline,
9700 "module did not retain two terminal exits: {:?}",
9701 module.terminal_history()
9702 );
9703 sleep(Duration::from_millis(10)).await;
9704 }
9705
9706 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9707 let request = ClientControlRequest::SupervisorTerminals {
9708 module_id: "terminal-golden".to_string(),
9709 };
9710 let frame = Frame::build(
9711 FrameType::Request,
9712 control_flags(),
9713 0,
9714 0,
9715 1,
9716 serde_json::to_vec(&request).unwrap(),
9717 )
9718 .unwrap();
9719 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9720 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9721 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9722 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9723 panic!("expected supervisor.terminals response");
9724 };
9725 assert_eq!(terminals.entries.len(), 2);
9726 assert_eq!(terminals.dropped, 0);
9727
9728 let mut rendered = serde_json::to_value(response).unwrap();
9729 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9732 for (index, entry) in rendered["entries"]
9733 .as_array_mut()
9734 .expect("terminal response entries array")
9735 .iter_mut()
9736 .enumerate()
9737 {
9738 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9739 }
9740
9741 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9742 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9743 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9744 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9745 std::fs::write(&golden_path, &serialized).unwrap();
9746 }
9747 let expected: Value =
9748 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9749 assert_eq!(rendered, expected);
9750 }
9751
9752 #[test]
9753 fn hello_registers_manifest_and_returns_ack() {
9754 let registry = Arc::new(Registry::default());
9755 let handler = ControlHandler::new(Arc::clone(®istry));
9756 let conn = ConnectionId::new(1);
9757
9758 let responses = handler
9759 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9760 .unwrap();
9761
9762 assert_eq!(responses.len(), 1);
9763 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9764 assert_eq!(responses[0].header.channel, 0);
9765 assert_eq!(responses[0].header.corr, 7);
9766 let ack = parse_ack(&responses[0]);
9767 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9768 assert!(ack
9769 .subc_capabilities
9770 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9771 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9772 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9773 assert!(ack
9774 .subc_ops
9775 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9776 assert!(ack
9777 .subc_ops
9778 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9779
9780 let registration = registry.get_module("aft").unwrap().unwrap();
9781 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9782 assert_eq!(registration.state, ChannelState::Active);
9783 assert_eq!(registration.connection_id, conn);
9784 assert_eq!(registration.control_ops, module_baseline_control_ops());
9785 }
9786
9787 #[test]
9788 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9789 let invalid_identifiers = [
9790 ("case_change", "credentials-Provider/v1"),
9791 ("leading_zero", "credentials-provider/v01"),
9792 ("trailing_hyphen", "credentials-provider-/v1"),
9793 ("consecutive_hyphens", "credentials--provider/v1"),
9794 ("uppercase", "Credentials-provider/v1"),
9795 ("missing_v", "credentials-provider/1"),
9796 ("whitespace", "credentials provider/v1"),
9797 ("zero_version", "credentials-provider/v0"),
9798 ("out_of_range_version", "credentials-provider/v4294967296"),
9799 (
9800 "overlength_name",
9801 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9802 ),
9803 ];
9804 let mut cases = invalid_identifiers
9805 .into_iter()
9806 .map(|(name, identifier)| {
9807 (
9808 format!("identifier_{name}"),
9809 "capabilities.provides[0]".to_string(),
9810 identifier.to_string(),
9811 json!({ "provides": [identifier] }),
9812 None,
9813 )
9814 })
9815 .collect::<Vec<_>>();
9816 cases.extend([
9817 (
9818 "unknown_need".to_string(),
9819 "capabilities.requires[0].need".to_string(),
9820 "deferred".to_string(),
9821 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9822 None,
9823 ),
9824 (
9825 "duplicate_provides".to_string(),
9826 "capabilities.provides[1]".to_string(),
9827 "credentials-provider/v1".to_string(),
9828 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9829 None,
9830 ),
9831 (
9832 "duplicate_must_never_reach".to_string(),
9833 "capabilities.must_never_reach[1]".to_string(),
9834 "credentials-provider/v1".to_string(),
9835 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9836 None,
9837 ),
9838 (
9839 "duplicate_requires_same_need".to_string(),
9840 "capabilities.requires[1]".to_string(),
9841 "credentials-provider/v1".to_string(),
9842 json!({ "requires": [
9843 { "capability": "credentials-provider/v1", "need": "required" },
9844 { "capability": "credentials-provider/v1", "need": "required" }
9845 ] }),
9846 None,
9847 ),
9848 (
9849 "duplicate_requires_conflicting_need".to_string(),
9850 "capabilities.requires[1]".to_string(),
9851 "credentials-provider/v1".to_string(),
9852 json!({ "requires": [
9853 { "capability": "credentials-provider/v1", "need": "required" },
9854 { "capability": "credentials-provider/v1", "need": "optional" }
9855 ] }),
9856 None,
9857 ),
9858 (
9859 "capabilities_root_pointer".to_string(),
9860 "runtime_computed[0]".to_string(),
9861 "/capabilities".to_string(),
9862 json!({}),
9863 Some(json!(["/capabilities"])),
9864 ),
9865 (
9866 "capabilities_descendant_pointer".to_string(),
9867 "runtime_computed[0]".to_string(),
9868 "/capabilities/provides".to_string(),
9869 json!({}),
9870 Some(json!(["/capabilities/provides"])),
9871 ),
9872 (
9873 "malformed_pointer_without_leading_slash".to_string(),
9874 "runtime_computed[0]".to_string(),
9875 "capabilities".to_string(),
9876 json!({}),
9877 Some(json!(["capabilities"])),
9878 ),
9879 (
9880 "malformed_pointer_escape".to_string(),
9881 "runtime_computed[0]".to_string(),
9882 "/roles/~2/tools".to_string(),
9883 json!({}),
9884 Some(json!(["/roles/~2/tools"])),
9885 ),
9886 (
9887 "unknown_capabilities_field".to_string(),
9888 "capabilities.future".to_string(),
9889 "<array>".to_string(),
9890 json!({ "future": [] }),
9891 None,
9892 ),
9893 ]);
9894
9895 for (index, (name, field, value, capabilities, runtime_computed)) in
9896 cases.into_iter().enumerate()
9897 {
9898 let registry = Arc::new(Registry::default());
9899 let handler = ControlHandler::new(Arc::clone(®istry));
9900 let response = handler
9901 .handle_control(
9902 ConnectionId::new((index + 1) as u64),
9903 capability_grammar_hello_frame(
9904 capabilities,
9905 runtime_computed,
9906 index as u64 + 1,
9907 ),
9908 )
9909 .expect("invalid HELLO returns a refusal");
9910
9911 assert_eq!(response.len(), 1, "{name} must emit one refusal");
9912 let error = parse_error(&response[0]);
9913 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9914 let message = error["message"]
9915 .as_str()
9916 .expect("error message is a string");
9917 assert!(
9918 message.contains(&field),
9919 "{name}: field missing from {message}"
9920 );
9921 assert!(
9922 message.contains(&value),
9923 "{name}: value missing from {message}"
9924 );
9925 assert_eq!(
9926 registry
9927 .active_registration_count()
9928 .expect("registry reads"),
9929 0,
9930 "{name}: refused HELLO must not create a catalog entry"
9931 );
9932 }
9933 }
9934
9935 #[test]
9936 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9937 let registry = Arc::new(Registry::default());
9938 let handler = ControlHandler::new(Arc::clone(®istry));
9939 let capabilities = json!({
9940 "provides": ["credentials-provider/v1"],
9941 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9942 "must_never_reach": ["federation-transport/v1"]
9943 });
9944 let response = handler
9945 .handle_control(
9946 ConnectionId::new(99),
9947 capability_grammar_hello_frame(
9948 capabilities.clone(),
9949 Some(json!(["/roles/0/tools"])),
9950 99,
9951 ),
9952 )
9953 .expect("valid HELLO registers");
9954 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9955
9956 let request = Frame::build(
9957 FrameType::Request,
9958 control_flags(),
9959 0,
9960 0,
9961 100,
9962 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9963 .expect("catalog request serializes"),
9964 )
9965 .expect("catalog request frame builds");
9966 let response = handler
9967 .handle_catalog_list(request, None)
9968 .expect("catalog list succeeds");
9969 let ClientControlResponse::CatalogList { modules, .. } =
9970 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9971 else {
9972 panic!("catalog request must return catalog.list");
9973 };
9974 assert_eq!(modules.len(), 1);
9975 assert_eq!(
9976 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9977 capabilities
9978 );
9979 }
9980
9981 #[test]
9982 fn catalog_list_mirrors_management_operation_description() {
9983 let registry = Arc::new(Registry::default());
9984 let handler = ControlHandler::new(Arc::clone(®istry));
9985 let description = "List managed records and return their identifiers and metadata.";
9986 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9987 manifest.provides = vec![ProviderRole::ManagementSurface {
9988 operations: vec![ManagementOperation {
9989 name: "records.list".to_string(),
9990 kind: ManagementOperationKind::Query,
9991 description: Some(description.to_string()),
9992 }],
9993 config_schema: json!({"type": "object"}),
9994 observability: vec![ObservabilitySurface {
9995 name: "records.stats".to_string(),
9996 kind: ObservabilityKind::Snapshot,
9997 }],
9998 identity_scope: vec![IdentityScope::Project],
9999 concurrency: Concurrency::ModuleManaged,
10000 }];
10001 registry
10002 .register_with_control_ops(
10003 manifest,
10004 PROTOCOL_VERSION,
10005 ConnectionId::new(99),
10006 Vec::new(),
10007 )
10008 .expect("described management manifest registers");
10009
10010 let request = Frame::build(
10011 FrameType::Request,
10012 control_flags(),
10013 0,
10014 0,
10015 100,
10016 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10017 .expect("catalog request serializes"),
10018 )
10019 .expect("catalog request frame builds");
10020 let response = handler
10021 .handle_catalog_list(request, None)
10022 .expect("catalog list succeeds");
10023 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10024 assert_eq!(
10025 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
10026 "catalog.list must preserve the declared operation description verbatim"
10027 );
10028 }
10029
10030 #[test]
10031 fn catalog_list_mirrors_registered_operation_delivery() {
10032 let registry = Arc::new(Registry::default());
10033 let handler = ControlHandler::new(Arc::clone(®istry));
10034 let delivery =
10035 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
10036 let mut body = serde_json::to_value(ModuleHelloBody {
10037 manifest: manifest("watch-management", PROTOCOL_VERSION),
10038 protocol_ver: PROTOCOL_VERSION,
10039 control_ops: None,
10040 launch_nonce: None,
10041 })
10042 .unwrap();
10043 body["manifest"]["provides"] = json!([{
10044 "role": "management_surface",
10045 "operations": [
10046 {"name": "status", "kind": "query"},
10047 {"name": "future", "kind": "query"},
10048 {"name": "reset", "kind": "mutate"}
10049 ],
10050 "config_schema": {}, "observability": [], "identity_scope": []
10051 }]);
10052 body["manifest"]["operation_delivery"] = delivery.clone();
10053 let hello = Frame::build(
10054 FrameType::Hello,
10055 control_flags(),
10056 0,
10057 0,
10058 99,
10059 serde_json::to_vec(&body).unwrap(),
10060 )
10061 .unwrap();
10062 let response = handler
10063 .handle_control(ConnectionId::new(99), hello)
10064 .unwrap();
10065 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10066
10067 let (_, registrations) = registry.list_modules().unwrap();
10068 let registered = ®istrations[0].manifest;
10069 assert!(registered.is_watch_operation("status"));
10070 assert!(!registered.is_watch_operation("future"));
10071 assert!(!registered.is_watch_operation("reset"));
10072 assert!(!registered.is_watch_operation("missing"));
10073
10074 let request = Frame::build(
10075 FrameType::Request,
10076 control_flags(),
10077 0,
10078 0,
10079 100,
10080 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10081 )
10082 .unwrap();
10083 let response = handler.handle_catalog_list(request, None).unwrap();
10084 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
10085 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
10086 let ClientControlResponse::CatalogList { modules, .. } =
10087 serde_json::from_slice(&response[0].body).unwrap()
10088 else {
10089 panic!("expected catalog.list");
10090 };
10091 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
10092 }
10093
10094 #[test]
10095 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
10096 let registry = Arc::new(Registry::default());
10097 let supervisor = SupervisorHandle::new();
10098 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10099 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10100 let descriptor = json!({
10101 "schema": "ckios.richui.manifest/1",
10102 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10103 "bytes": 4096,
10104 "chunks": 4
10105 });
10106 let connection = ConnectionId::new(201);
10107 let hello = handler
10108 .handle_control(
10109 connection,
10110 hello_frame_with_rich_ui(
10111 "vault",
10112 PROTOCOL_VERSION,
10113 201,
10114 Some("the-real-nonce"),
10115 descriptor.clone(),
10116 ),
10117 )
10118 .unwrap();
10119 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10120
10121 let request =
10122 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
10123 let provides = registry
10124 .get_module_by_connection(connection)
10125 .unwrap()
10126 .unwrap()
10127 .manifest
10128 .provides;
10129 handler
10130 .handle_catalog_update(connection, request, provides, None, Some(false))
10131 .unwrap();
10132
10133 let request = Frame::build(
10134 FrameType::Request,
10135 control_flags(),
10136 0,
10137 0,
10138 203,
10139 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10140 )
10141 .unwrap();
10142 let response = handler.handle_catalog_list(request, None).unwrap();
10143 let ClientControlResponse::CatalogList { modules, .. } =
10144 serde_json::from_slice(&response[0].body).unwrap()
10145 else {
10146 panic!("expected catalog.list");
10147 };
10148 assert_eq!(
10149 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
10150 descriptor
10151 );
10152 assert!(modules[0].nonce_verified);
10153 }
10154
10155 fn hello_with_role_declarations(declarations: Value) -> Frame {
10156 let mut body = serde_json::to_value(ModuleHelloBody {
10157 manifest: manifest("role-test", PROTOCOL_VERSION),
10158 protocol_ver: PROTOCOL_VERSION,
10159 control_ops: None,
10160 launch_nonce: None,
10161 })
10162 .unwrap();
10163 body["manifest"]["role_declarations"] = declarations;
10164 Frame::build(
10165 FrameType::Hello,
10166 control_flags(),
10167 0,
10168 0,
10169 301,
10170 serde_json::to_vec(&body).unwrap(),
10171 )
10172 .unwrap()
10173 }
10174
10175 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
10176 let request = Frame::build(
10177 FrameType::Request,
10178 control_flags(),
10179 0,
10180 0,
10181 302,
10182 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10183 )
10184 .unwrap();
10185 let response = handler.handle_catalog_list(request, None).unwrap();
10186 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
10187 let ClientControlResponse::CatalogList { mut modules, .. } =
10188 serde_json::from_slice(&response[0].body).unwrap()
10189 else {
10190 panic!("expected catalog.list");
10191 };
10192 (raw["modules"][0].clone(), modules.remove(0))
10193 }
10194
10195 #[test]
10196 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
10197 let registry = Arc::new(Registry::default());
10198 let handler = ControlHandler::new(Arc::clone(®istry));
10199 let declarations = json!({
10200 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
10201 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
10202 });
10203 let connection = ConnectionId::new(301);
10204 let response = handler
10205 .handle_control(
10206 connection,
10207 hello_with_role_declarations(declarations.clone()),
10208 )
10209 .unwrap();
10210 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10211 let registered = registry
10212 .get_module_by_connection(connection)
10213 .unwrap()
10214 .unwrap()
10215 .manifest;
10216 assert_eq!(
10220 serde_json::to_value(®istered.role_declarations).unwrap(),
10221 declarations
10222 );
10223 let (raw, entry) = role_declarations_catalog(&handler);
10224 assert_eq!(raw["role_declarations"], declarations);
10225 assert_eq!(entry.role_declarations, registered.role_declarations);
10226
10227 let request =
10228 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
10229 handler
10230 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
10231 .unwrap();
10232 let (raw, entry) = role_declarations_catalog(&handler);
10233 assert_eq!(raw["role_declarations"], declarations);
10234 assert_eq!(entry.role_declarations, registered.role_declarations);
10235 }
10236
10237 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
10238 let registry = Arc::new(Registry::default());
10239 let handler = ControlHandler::new(Arc::clone(®istry));
10240 let capture = EventCapture::default();
10241 let _subscriber =
10242 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10243 let response = handler
10244 .handle_control(
10245 ConnectionId::new(301),
10246 hello_with_role_declarations(declarations),
10247 )
10248 .unwrap();
10249 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10250 assert!(registry
10251 .get_module_by_connection(ConnectionId::new(301))
10252 .unwrap()
10253 .unwrap()
10254 .manifest
10255 .role_declarations
10256 .is_empty());
10257 let (raw, entry) = role_declarations_catalog(&handler);
10258 assert!(raw.get("role_declarations").is_none());
10259 assert!(entry.role_declarations.is_empty());
10260 let warnings: Vec<_> = capture
10261 .events()
10262 .into_iter()
10263 .filter(|event| {
10264 event.fields.get("message").map(String::as_str)
10265 == Some("dropping whole role_declarations map from HELLO")
10266 })
10267 .collect();
10268 assert_eq!(warnings.len(), 1);
10269 assert_eq!(warnings[0].level, tracing::Level::WARN);
10270 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
10271 assert_eq!(warnings[0].fields["reason"], reason);
10272 }
10273
10274 #[test]
10275 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
10276 for value in [
10277 json!(null),
10278 json!(false),
10279 json!(0),
10280 json!("opaque"),
10281 json!([]),
10282 ] {
10283 assert_hello_drops_role_declarations_with_warning(
10284 json!({"routing/v1": {}, "broken/v1": value}),
10285 "role_declarations values must be objects",
10286 );
10287 }
10288 assert_hello_drops_role_declarations_with_warning(
10289 json!([]),
10290 "role_declarations must be an object",
10291 );
10292 }
10293
10294 #[test]
10295 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
10296 assert_hello_drops_role_declarations_with_warning(
10297 json!({"routing/v1": {}, "bad key": {}}),
10298 "role_declarations key must use capability-name grammar",
10299 );
10300 }
10301
10302 #[test]
10303 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
10304 assert_hello_drops_role_declarations_with_warning(
10305 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
10306 "role_declarations exceeds 16384 compact JSON bytes",
10307 );
10308 }
10309
10310 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
10311 let mut body = serde_json::to_value(ModuleHelloBody {
10312 manifest: manifest("flow-ops", PROTOCOL_VERSION),
10313 protocol_ver: PROTOCOL_VERSION,
10314 control_ops: None,
10315 launch_nonce: None,
10316 })
10317 .unwrap();
10318 body["manifest"]["provides"] = json!([{
10319 "role": "management_surface",
10320 "operations": [
10321 {"name": "diagnostics.findings", "kind": "query"},
10322 {"name": "reset", "kind": "mutate"}
10323 ],
10324 "config_schema": {}, "observability": [], "identity_scope": []
10325 }]);
10326 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
10327 if let Some(flow_operations) = flow_operations {
10328 body["manifest"]["flow_operations"] = flow_operations;
10329 }
10330 Frame::build(
10331 FrameType::Hello,
10332 control_flags(),
10333 0,
10334 0,
10335 401,
10336 serde_json::to_vec(&body).unwrap(),
10337 )
10338 .unwrap()
10339 }
10340
10341 #[test]
10342 fn catalog_list_mirrors_flow_operations_unchanged() {
10343 for (capability, flow_operations) in [
10346 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
10347 ("call-modes/v1", json!(["diagnostics.findings"])),
10348 ("flow-scopes/v1", json!([])),
10349 ] {
10350 let registry = Arc::new(Registry::default());
10351 let handler = ControlHandler::new(Arc::clone(®istry));
10352 let connection = ConnectionId::new(401);
10353 let response = handler
10354 .handle_control(
10355 connection,
10356 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
10357 )
10358 .unwrap();
10359 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10360 let registered = registry
10361 .get_module_by_connection(connection)
10362 .unwrap()
10363 .unwrap()
10364 .manifest;
10365 assert_eq!(
10366 serde_json::to_value(®istered.flow_operations).unwrap(),
10367 flow_operations
10368 );
10369 let (raw, entry) = role_declarations_catalog(&handler);
10370 assert_eq!(raw["flow_operations"], flow_operations);
10371 assert_eq!(entry.flow_operations, registered.flow_operations);
10372 }
10373
10374 let registry = Arc::new(Registry::default());
10375 let handler = ControlHandler::new(Arc::clone(®istry));
10376 let response = handler
10377 .handle_control(
10378 ConnectionId::new(401),
10379 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
10380 )
10381 .unwrap();
10382 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10383 let (raw, entry) = role_declarations_catalog(&handler);
10384 assert!(raw.get("flow_operations").is_none());
10385 assert_eq!(entry.flow_operations, None);
10386 }
10387
10388 fn assert_hello_drops_flow_operations_with_warning(
10389 capabilities: Value,
10390 flow_operations: Value,
10391 reason: &str,
10392 ) {
10393 let registry = Arc::new(Registry::default());
10394 let handler = ControlHandler::new(Arc::clone(®istry));
10395 let capture = EventCapture::default();
10396 let _subscriber =
10397 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10398 let response = handler
10399 .handle_control(
10400 ConnectionId::new(401),
10401 hello_with_flow_operations(capabilities, Some(flow_operations)),
10402 )
10403 .unwrap();
10404 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10405 let registered = registry
10406 .get_module_by_connection(ConnectionId::new(401))
10407 .unwrap()
10408 .expect("the module still registers")
10409 .manifest;
10410 assert_eq!(registered.module_id, "flow-ops");
10411 assert_eq!(registered.flow_operations, None);
10412 let (raw, entry) = role_declarations_catalog(&handler);
10413 assert_eq!(raw["module_id"], "flow-ops");
10414 assert!(raw.get("flow_operations").is_none());
10415 assert_eq!(entry.flow_operations, None);
10416 let warnings: Vec<_> = capture
10417 .events()
10418 .into_iter()
10419 .filter(|event| {
10420 event.fields.get("message").map(String::as_str)
10421 == Some("dropping flow_operations from HELLO")
10422 })
10423 .collect();
10424 assert_eq!(warnings.len(), 1);
10425 assert_eq!(warnings[0].level, tracing::Level::WARN);
10426 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
10427 assert_eq!(warnings[0].fields["reason"], reason);
10428 }
10429
10430 #[test]
10431 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
10432 assert_hello_drops_flow_operations_with_warning(
10433 json!(["flow-scopes/v1"]),
10434 json!(["diagnostics.findings", "missing"]),
10435 "flow_operations names 'missing', which is not a declared management operation",
10436 );
10437 }
10438
10439 #[test]
10440 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
10441 assert_hello_drops_flow_operations_with_warning(
10442 json!([]),
10443 json!(["diagnostics.findings"]),
10444 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10445 );
10446 assert_hello_drops_flow_operations_with_warning(
10448 json!(["agent-run-scopes/v1"]),
10449 json!([]),
10450 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10451 );
10452 }
10453
10454 #[test]
10455 fn hello_malformed_flow_operations_is_dropped_with_warning() {
10456 assert_hello_drops_flow_operations_with_warning(
10457 json!(["flow-scopes/v1"]),
10458 json!(["reset", "reset"]),
10459 "flow_operations names 'reset' more than once",
10460 );
10461 assert_hello_drops_flow_operations_with_warning(
10462 json!(["flow-scopes/v1"]),
10463 json!(vec!["reset"; 257]),
10464 "flow_operations lists 257 names, more than 256",
10465 );
10466 assert_hello_drops_flow_operations_with_warning(
10467 json!(["flow-scopes/v1"]),
10468 json!("reset"),
10469 "flow_operations must be an array of operation names",
10470 );
10471 }
10472
10473 #[test]
10474 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
10475 let registry = Arc::new(Registry::default());
10476 let handler = ControlHandler::new(registry.clone());
10477 let descriptor = json!({
10478 "schema": "ckios.richui.manifest/1",
10479 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10480 "bytes": 128,
10481 "chunks": 2
10482 });
10483 let malformed = [
10484 ("schema", json!("")),
10485 ("schema", json!("a".repeat(65))),
10486 ("schema", json!("Bad schema")),
10487 (
10488 "manifest_digest",
10489 json!(format!("sha256:{}", "A".repeat(64))),
10490 ),
10491 ("manifest_digest", json!("sha256:abc")),
10492 ("bytes", json!(0)),
10493 ("bytes", json!(16_777_217)),
10494 ("bytes", json!("128")),
10495 ("chunks", json!(0)),
10496 ("chunks", json!(1025)),
10497 ("chunks", json!("2")),
10498 ];
10499 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
10500 let mut invalid = descriptor.clone();
10501 invalid[field] = invalid_value;
10502 let response = handler
10503 .handle_control(
10504 ConnectionId::new(210 + index as u64),
10505 hello_frame_with_rich_ui(
10506 &format!("bad-rich-ui-{index}"),
10507 PROTOCOL_VERSION,
10508 210 + index as u64,
10509 None,
10510 invalid,
10511 ),
10512 )
10513 .unwrap();
10514 assert_eq!(response[0].header.ty, FrameType::Error);
10515 let error = parse_error(&response[0]);
10516 assert_eq!(error["code"], "invalid_manifest");
10517 assert!(error["message"]
10518 .as_str()
10519 .unwrap()
10520 .contains(&format!("rich_ui.{field}")));
10521 assert!(registry
10522 .get_module(&format!("bad-rich-ui-{index}"))
10523 .unwrap()
10524 .is_none());
10525 }
10526 }
10527
10528 #[test]
10529 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
10530 let registry = Arc::new(Registry::default());
10531 let handler = ControlHandler::new(Arc::clone(®istry));
10532 let connection = ConnectionId::new(219);
10533 registry
10534 .register_with_control_ops(
10535 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
10536 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
10537 "bad schema",
10538 format!("sha256:{}", "a".repeat(64)),
10539 128,
10540 2,
10541 )))
10542 .build(),
10543 PROTOCOL_VERSION,
10544 connection,
10545 Vec::new(),
10546 )
10547 .unwrap();
10548 let request =
10549 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
10550 let response = handler
10551 .handle_catalog_update(connection, request, Vec::new(), None, None)
10552 .unwrap();
10553 assert_eq!(response[0].header.ty, FrameType::Error);
10554 let error = parse_error(&response[0]);
10555 assert_eq!(error["code"], "invalid_manifest");
10556 assert!(error["message"]
10557 .as_str()
10558 .unwrap()
10559 .contains("rich_ui.schema"));
10560 }
10561
10562 #[test]
10563 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
10564 let registry = Arc::new(Registry::default());
10565 let supervisor = SupervisorHandle::new();
10566 supervisor.open_swap("provider", "candidate-nonce".to_string());
10567 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10568 let connection = ConnectionId::new(218);
10569 let hello = handler
10570 .handle_control(
10571 connection,
10572 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
10573 )
10574 .unwrap();
10575 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10576 assert!(registry.list_modules().unwrap().1.is_empty());
10577
10578 registry.promote_candidate("provider").unwrap().unwrap();
10579 let request = Frame::build(
10580 FrameType::Request,
10581 control_flags(),
10582 0,
10583 0,
10584 219,
10585 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10586 )
10587 .unwrap();
10588 let response = handler.handle_catalog_list(request, None).unwrap();
10589 let ClientControlResponse::CatalogList { modules, .. } =
10590 serde_json::from_slice(&response[0].body).unwrap()
10591 else {
10592 panic!("expected catalog.list");
10593 };
10594 assert_eq!(modules.len(), 1);
10595 assert!(modules[0].nonce_verified);
10596 }
10597
10598 #[test]
10599 fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
10600 for provenance_json in [
10601 r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
10602 r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
10603 ] {
10604 let registry = Arc::new(Registry::default());
10605 let handler = ControlHandler::new(registry);
10606 let hello = Frame::build(
10607 FrameType::Hello,
10608 control_flags(),
10609 0,
10610 0,
10611 220,
10612 serde_json::to_vec(&ModuleHelloBody {
10613 manifest: ModuleManifest::builder("built", "0.1.0")
10614 .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
10615 .build(),
10616 protocol_ver: PROTOCOL_VERSION,
10617 control_ops: None,
10618 launch_nonce: None,
10619 })
10620 .unwrap(),
10621 )
10622 .unwrap();
10623 assert_eq!(
10624 handler
10625 .handle_control(ConnectionId::new(220), hello)
10626 .unwrap()[0]
10627 .header
10628 .ty,
10629 FrameType::HelloAck
10630 );
10631 let request = Frame::build(
10632 FrameType::Request,
10633 control_flags(),
10634 0,
10635 0,
10636 221,
10637 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10638 )
10639 .unwrap();
10640 let response = handler.handle_catalog_list(request, None).unwrap();
10641 let ClientControlResponse::CatalogList { modules, .. } =
10642 serde_json::from_slice(&response[0].body).unwrap()
10643 else {
10644 panic!("expected catalog.list");
10645 };
10646 assert_eq!(modules.len(), 1);
10647 assert_eq!(
10648 serde_json::to_vec(&modules[0]).unwrap(),
10649 format!(
10650 r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
10651 )
10652 .as_bytes()
10653 );
10654 }
10655 }
10656
10657 #[test]
10658 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
10659 let registry = Arc::new(Registry::default());
10660 let handler = ControlHandler::new(Arc::clone(®istry));
10661 let hello = Frame::build(
10662 FrameType::Hello,
10663 control_flags(),
10664 0,
10665 0,
10666 220,
10667 serde_json::to_vec(&ModuleHelloBody {
10668 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
10669 protocol_ver: PROTOCOL_VERSION,
10670 control_ops: None,
10671 launch_nonce: Some("unverified".to_string()),
10672 })
10673 .unwrap(),
10674 )
10675 .unwrap();
10676 assert_eq!(
10677 handler
10678 .handle_control(ConnectionId::new(220), hello)
10679 .unwrap()[0]
10680 .header
10681 .ty,
10682 FrameType::HelloAck
10683 );
10684 let request = Frame::build(
10685 FrameType::Request,
10686 control_flags(),
10687 0,
10688 0,
10689 221,
10690 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10691 )
10692 .unwrap();
10693 let response = handler.handle_catalog_list(request, None).unwrap();
10694 let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
10695 assert!(wire["modules"][0].get("provenance").is_none());
10696 let ClientControlResponse::CatalogList { modules, .. } =
10697 serde_json::from_slice(&response[0].body).unwrap()
10698 else {
10699 panic!("expected catalog.list");
10700 };
10701 assert_eq!(
10702 serde_json::to_vec(&modules[0]).unwrap(),
10703 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
10704 );
10705 assert!(!modules[0].nonce_verified);
10706 assert!(modules[0].provenance.is_none());
10707 }
10708
10709 #[test]
10710 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
10711 let registry = Arc::new(Registry::default());
10712 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10713 [("vault".to_string(), true), ("squatter".to_string(), true)],
10714 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10715 );
10716 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
10717 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10718 provides: vec!["credentials-provider/v1".to_string()],
10719 requires: Vec::new(),
10720 must_never_reach: Vec::new(),
10721 });
10722 let frame = Frame::build(
10723 FrameType::Hello,
10724 control_flags(),
10725 0,
10726 0,
10727 77,
10728 serde_json::to_vec(&ModuleHelloBody {
10729 manifest: squatter,
10730 protocol_ver: PROTOCOL_VERSION,
10731 control_ops: None,
10732 launch_nonce: None,
10733 })
10734 .expect("HELLO serializes"),
10735 )
10736 .expect("HELLO frame builds");
10737 let response = handler
10738 .handle_control(ConnectionId::new(77), frame)
10739 .expect("reserved claim receives a typed refusal");
10740 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10741 assert_eq!(
10742 registry
10743 .active_registration_count()
10744 .expect("registry reads"),
10745 0,
10746 "a reserved capability refusal must not leave a catalog entry"
10747 );
10748 }
10749
10750 #[test]
10751 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10752 for settlement in ["timeout", "inconclusive", "drop"] {
10753 let breakers = RouteBindBreakers::default();
10754 let RouteBindAdmission::Admitted {
10755 guard: mut old,
10756 probe: false,
10757 } = breakers.admit("prov")
10758 else {
10759 panic!("ordinary relay admitted")
10760 };
10761 let RouteBindAdmission::Admitted {
10762 guard: mut opener, ..
10763 } = breakers.admit("prov")
10764 else {
10765 panic!("second relay admitted")
10766 };
10767 assert!(
10768 !opener
10769 .record_timeout(1, Duration::ZERO)
10770 .unwrap()
10771 .reopened_after_probe
10772 );
10773 let RouteBindAdmission::Admitted {
10774 guard: mut probe,
10775 probe: true,
10776 } = breakers.admit("prov")
10777 else {
10778 panic!("one cooldown probe admitted")
10779 };
10780 match settlement {
10781 "timeout" => assert!(
10782 !old.record_timeout(1, Duration::ZERO)
10783 .unwrap()
10784 .reopened_after_probe
10785 ),
10786 "inconclusive" => old.record_inconclusive(),
10787 "drop" => drop(old),
10788 _ => unreachable!(),
10789 }
10790 assert!(
10791 matches!(
10792 breakers.admit("prov"),
10793 RouteBindAdmission::Refused {
10794 probe_in_flight: true,
10795 ..
10796 }
10797 ),
10798 "{settlement} of a pre-open relay cannot release the real probe"
10799 );
10800 assert!(
10801 probe
10802 .record_timeout(1, Duration::ZERO)
10803 .unwrap()
10804 .reopened_after_probe
10805 );
10806 assert!(matches!(
10807 breakers.admit("prov"),
10808 RouteBindAdmission::Admitted { probe: true, .. }
10809 ));
10810 }
10811 let breakers = RouteBindBreakers::default();
10812 let admit = || match breakers.admit("prov") {
10813 RouteBindAdmission::Admitted { guard, .. } => guard,
10814 _ => panic!("relay admitted"),
10815 };
10816 admit().record_timeout(1, Duration::ZERO);
10817 let mut old_probe = admit();
10818 breakers.reset_for_new_module_connection("prov");
10819 admit().record_timeout(1, Duration::ZERO);
10820 let _new_probe = admit();
10821 old_probe.record_inconclusive();
10822 assert!(matches!(
10823 breakers.admit("prov"),
10824 RouteBindAdmission::Refused {
10825 probe_in_flight: true,
10826 ..
10827 }
10828 ));
10829 }
10830
10831 #[tokio::test]
10832 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
10833 for candidate in [false, true] {
10834 let registry = Arc::new(Registry::default());
10835 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10836 [("vault".to_string(), true), ("squatter".to_string(), true)],
10837 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10838 );
10839 let conn = ConnectionId::new(77);
10840 let (ctx, mut rx) = route_ctx(conn);
10841 let initial = capability_manifest("squatter", &[], &[]);
10842 if candidate {
10843 registry
10844 .register_candidate_with_control_ops(
10845 initial.clone(),
10846 PROTOCOL_VERSION,
10847 conn,
10848 module_baseline_control_ops(),
10849 )
10850 .unwrap();
10851 handler
10852 .forwarding
10853 .register_candidate_module_connection(
10854 conn,
10855 "squatter".to_string(),
10856 PROTOCOL_VERSION,
10857 manifest_concurrency(&initial),
10858 ctx.egress.clone(),
10859 )
10860 .unwrap();
10861 } else {
10862 hello_via_sink(
10863 &handler,
10864 &ctx,
10865 &mut rx,
10866 hello_frame_with_manifest(initial.clone(), 1),
10867 )
10868 .await;
10869 }
10870 let response = handler
10871 .handle_control_frame(
10872 &ctx,
10873 catalog_update_with_capabilities_frame(
10874 2,
10875 capability_manifest("squatter", &["credentials-provider/v1"], &[])
10876 .capabilities
10877 .unwrap(),
10878 ),
10879 )
10880 .await
10881 .unwrap();
10882 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10883 assert_eq!(
10884 registry
10885 .get_module_by_connection(conn)
10886 .unwrap()
10887 .unwrap()
10888 .manifest,
10889 initial
10890 );
10891 }
10892 }
10893
10894 #[test]
10895 fn server_describe_surfaces_required_capability_verdict_fields() {
10896 let registry = Arc::new(Registry::default());
10897 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10898 [
10899 ("consumer".to_string(), true),
10900 ("provider".to_string(), false),
10901 ],
10902 BTreeMap::new(),
10903 );
10904 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10905 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10906 provides: Vec::new(),
10907 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10908 capability: "credentials-provider/v1".to_string(),
10909 need: subc_protocol::manifest::CapabilityNeed::Required,
10910 }],
10911 must_never_reach: Vec::new(),
10912 });
10913 let hello = Frame::build(
10914 FrameType::Hello,
10915 control_flags(),
10916 0,
10917 0,
10918 78,
10919 serde_json::to_vec(&ModuleHelloBody {
10920 manifest: consumer,
10921 protocol_ver: PROTOCOL_VERSION,
10922 control_ops: None,
10923 launch_nonce: None,
10924 })
10925 .expect("HELLO serializes"),
10926 )
10927 .expect("HELLO frame builds");
10928 handler
10929 .handle_control(ConnectionId::new(78), hello)
10930 .expect("consumer registers");
10931 let describe = Frame::build(
10932 FrameType::Request,
10933 control_flags(),
10934 0,
10935 0,
10936 79,
10937 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10938 .expect("request serializes"),
10939 )
10940 .expect("describe frame builds");
10941 let response = handler
10942 .handle_server_describe(describe)
10943 .expect("server.describe succeeds");
10944 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10945 let requirement = &rendered["capability_requirements"][0];
10946 assert_eq!(requirement["consumer"], "consumer");
10947 assert_eq!(requirement["verdict"], "never_provided");
10948 assert_eq!(requirement["episode_seq"], 1);
10949 assert_eq!(requirement["config_satisfiable"], false);
10950 assert_eq!(requirement["runtime_available"], false);
10951 assert!(requirement["detail"]
10952 .as_str()
10953 .expect("detail string")
10954 .contains("credentials-provider/v1"));
10955 }
10956
10957 fn deregister_required_provider(
10963 daemon_shutdown_begun: bool,
10964 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10965 const CAPABILITY: &str = "project-identity/v1";
10966 let registry = Arc::new(Registry::default());
10967 let daemon_shutdown = DaemonShutdownFlag::default();
10968 let handler = ControlHandler::new(Arc::clone(®istry))
10969 .with_capability_config(
10970 [
10971 ("consumer".to_string(), true),
10972 ("provider".to_string(), true),
10973 ],
10974 BTreeMap::new(),
10975 )
10976 .with_daemon_shutdown_flag(daemon_shutdown.clone());
10977 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10978 consumer.capabilities = Some(CapabilityDeclarations {
10979 provides: Vec::new(),
10980 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10981 capability: CAPABILITY.to_string(),
10982 need: subc_protocol::manifest::CapabilityNeed::Required,
10983 }],
10984 must_never_reach: Vec::new(),
10985 });
10986 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10987 let provider_conn = ConnectionId::new(182);
10988 for (conn, manifest, corr) in [
10989 (ConnectionId::new(181), consumer, 181),
10990 (provider_conn, provider, 182),
10991 ] {
10992 let hello = handler
10993 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10994 .expect("module registers");
10995 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10996 }
10997 assert_eq!(
10998 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10999 Some(CapabilityVerdict::Provided),
11000 "the registered provider must satisfy the requirement first"
11001 );
11002 if daemon_shutdown_begun {
11003 daemon_shutdown.set_for_test();
11004 }
11005
11006 let capture = EventCapture::default();
11007 let _subscriber =
11008 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11009 handler
11010 .cleanup_connection(provider_conn)
11011 .expect("provider deregisters");
11012 let events = capture
11013 .events()
11014 .into_iter()
11015 .filter(|event| {
11016 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
11017 })
11018 .collect();
11019 (
11020 events,
11021 handler.capability_evaluator.verdict("consumer", CAPABILITY),
11022 )
11023 }
11024
11025 #[test]
11026 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
11027 let (events, verdict) = deregister_required_provider(true);
11028 assert_eq!(
11029 verdict,
11030 Some(CapabilityVerdict::NeverProvided),
11031 "shutdown must not change the verdict route.open refusals read"
11032 );
11033 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11034 let event = &events[0];
11035 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
11036 assert_eq!(
11037 event.fields.get("cause").map(String::as_str),
11038 Some("\"daemon_shutdown\""),
11039 "{event:?}"
11040 );
11041 assert_eq!(
11042 event.fields.get("verdict").map(String::as_str),
11043 Some("\"never_provided\""),
11044 "{event:?}"
11045 );
11046 }
11047
11048 #[test]
11049 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
11050 let (events, verdict) = deregister_required_provider(false);
11051 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
11052 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11053 let event = &events[0];
11054 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
11055 assert_eq!(event.fields.get("cause"), None, "{event:?}");
11056 assert_eq!(
11057 event.fields.get("verdict").map(String::as_str),
11058 Some("\"never_provided\""),
11059 "{event:?}"
11060 );
11061 }
11062
11063 #[test]
11064 fn catalog_list_omits_capabilities_for_legacy_manifest() {
11065 let registry = Arc::new(Registry::default());
11066 let handler = ControlHandler::new(Arc::clone(®istry));
11067 let hello = handler
11068 .handle_control(
11069 ConnectionId::new(101),
11070 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
11071 )
11072 .expect("legacy HELLO registers");
11073 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11074
11075 let request = Frame::build(
11076 FrameType::Request,
11077 control_flags(),
11078 0,
11079 0,
11080 102,
11081 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
11082 .expect("catalog request serializes"),
11083 )
11084 .expect("catalog request frame builds");
11085 let response = handler
11086 .handle_catalog_list(request, None)
11087 .expect("catalog list succeeds");
11088 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
11089 assert!(
11090 body["modules"][0].get("capabilities").is_none(),
11091 "legacy manifest must retain an absent capabilities field on catalog.list"
11092 );
11093 }
11094
11095 #[test]
11096 fn hello_ack_omits_storage_when_no_storage_config() {
11097 let registry = Arc::new(Registry::default());
11098 let handler = ControlHandler::new(Arc::clone(®istry));
11099 let responses = handler
11100 .handle_control(
11101 ConnectionId::new(1),
11102 hello_frame("aft", PROTOCOL_VERSION, 7),
11103 )
11104 .unwrap();
11105 let ack = parse_ack(&responses[0]);
11106 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
11107 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
11108 }
11109
11110 #[tokio::test]
11111 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
11112 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
11113 let registry = Arc::new(Registry::default());
11114 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
11115 let responses = handler
11116 .handle_control(
11117 ConnectionId::new(1),
11118 hello_frame("aft", PROTOCOL_VERSION, 7),
11119 )
11120 .unwrap();
11121 let ack = parse_ack(&responses[0]);
11122 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
11123
11124 let described = handler
11125 .handle_control_frame(
11126 &route_ctx(ConnectionId::new(2)).0,
11127 Frame::build(
11128 FrameType::Request,
11129 control_flags(),
11130 0,
11131 0,
11132 9,
11133 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11134 )
11135 .unwrap(),
11136 )
11137 .await
11138 .unwrap();
11139 let ClientControlResponse::ServerDescribe { machine_id, .. } =
11140 serde_json::from_slice(&described[0].body).unwrap()
11141 else {
11142 panic!("server.describe answered with another shape");
11143 };
11144 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
11145 }
11146
11147 #[test]
11148 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
11149 let registry = Arc::new(Registry::default());
11152 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
11153 crate::daemon_config::StorageConfig::Sqlite {
11154 data_home: std::path::PathBuf::from("/data"),
11155 },
11156 ));
11157
11158 let responses = handler
11159 .handle_control(
11160 ConnectionId::new(1),
11161 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
11162 )
11163 .unwrap();
11164 let ack = parse_ack(&responses[0]);
11165 assert_eq!(
11166 ack.storage,
11167 Some(serde_json::json!({
11168 "module_id": "alfonso-routing",
11169 "storage_namespace": "default",
11170 "isolation": { "kind": "module" },
11171 "backend": {
11172 "backend": "sqlite",
11173 "path": "/data/cortexkit/alfonso-routing/store.db"
11174 }
11175 })),
11176 "the delivered descriptor is the module's own sqlite store path"
11177 );
11178 }
11179
11180 #[test]
11181 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
11182 let registry = Arc::new(Registry::default());
11183 let handler = ControlHandler::new(Arc::clone(®istry));
11184 let conn = ConnectionId::new(1);
11185 let responses = handler
11186 .handle_control(
11187 conn,
11188 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11189 )
11190 .unwrap();
11191 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11192 let registration = registry.get_module("aft").unwrap().unwrap();
11193 assert_eq!(registration.control_ops, module_baseline_control_ops());
11194
11195 let frame =
11196 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
11197 assert!(handler
11198 .guard_module_control_op(&frame, "aft", "route.bind")
11199 .unwrap()
11200 .is_none());
11201 let error = handler
11202 .guard_module_control_op(&frame, "aft", "test.synthetic")
11203 .unwrap()
11204 .expect("synthetic ungranted op should be rejected");
11205 assert_eq!(error.header.ty, FrameType::Error);
11206 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
11207 }
11208
11209 #[test]
11210 fn hello_control_ops_some_adds_optional_grants() {
11211 let registry = Arc::new(Registry::default());
11212 let handler = ControlHandler::new(Arc::clone(®istry));
11213 handler
11214 .handle_control(
11215 ConnectionId::new(1),
11216 hello_frame_with_control_ops(
11217 "aft",
11218 PROTOCOL_VERSION,
11219 7,
11220 Some(vec![
11221 "future.synthetic".to_string(),
11222 "route.bind".to_string(),
11223 ]),
11224 ),
11225 )
11226 .unwrap();
11227 let registration = registry.get_module("aft").unwrap().unwrap();
11228 assert_eq!(
11229 registration.control_ops,
11230 vec![
11231 "route.bind".to_string(),
11232 "route.status".to_string(),
11233 "future.synthetic".to_string(),
11234 ]
11235 );
11236 let frame =
11237 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
11238 assert!(handler
11239 .guard_module_control_op(&frame, "aft", "future.synthetic")
11240 .unwrap()
11241 .is_none());
11242 }
11243
11244 #[tokio::test]
11245 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
11246 let registry = Arc::new(Registry::default());
11247 let forwarding = Arc::new(ForwardingTable::default());
11248 let handler =
11249 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11250 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
11251 hello_via_sink(
11252 &handler,
11253 &module_ctx,
11254 &mut module_rx,
11255 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11256 )
11257 .await;
11258
11259 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
11260 let responses = handler
11261 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
11262 .await
11263 .unwrap();
11264 assert_eq!(responses.len(), 1);
11265 assert_eq!(responses[0].header.ty, FrameType::Error);
11266 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
11267 assert!(module_rx.try_recv().is_err());
11268 }
11269
11270 #[tokio::test]
11271 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
11272 let registry = Arc::new(Registry::default());
11273 let forwarding = Arc::new(ForwardingTable::default());
11274 let handler =
11275 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11276 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
11277 hello_via_sink(
11278 &handler,
11279 &module_ctx,
11280 &mut module_rx,
11281 hello_frame_with_control_ops(
11282 "aft",
11283 PROTOCOL_VERSION,
11284 7,
11285 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11286 ),
11287 )
11288 .await;
11289
11290 let project_root = unique_project_root("demux");
11291 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
11292 let route_handler = handler.clone();
11293 let route_task = tokio::spawn(async move {
11294 route_handler
11295 .handle_control_frame(
11296 &route_client_ctx,
11297 route_open_frame(100, "aft", project_root),
11298 )
11299 .await
11300 .unwrap()
11301 });
11302 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11303 .await
11304 .unwrap()
11305 .unwrap();
11306 assert!(matches!(
11307 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
11308 ModuleControlRequest::RouteBind { .. }
11309 ));
11310
11311 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
11312 let health_handler = handler.clone();
11313 let health_task = tokio::spawn(async move {
11314 health_handler
11315 .handle_control_frame(
11316 &health_client_ctx,
11317 supervisor_health_probe_frame(101, "aft"),
11318 )
11319 .await
11320 .unwrap()
11321 });
11322 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11323 .await
11324 .unwrap()
11325 .unwrap();
11326 assert_eq!(
11327 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11328 ModuleControlRequest::HealthCheck {}
11329 );
11330
11331 handler
11332 .handle_control_frame(
11333 &module_ctx,
11334 health_response(health_frame.header.corr, HealthStatus::Degraded),
11335 )
11336 .await
11337 .unwrap();
11338 let health_response = health_task.await.unwrap();
11339 assert_eq!(health_response.len(), 1);
11340 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11341 ClientControlResponse::SupervisorHealthProbe {
11342 module_id,
11343 status,
11344 detail,
11345 metrics,
11346 } => {
11347 assert_eq!(module_id, "aft");
11348 assert_eq!(status, HealthStatus::Degraded);
11349 assert_eq!(detail.as_deref(), Some("warming"));
11350 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
11351 }
11352 other => panic!("unexpected health response: {other:?}"),
11353 }
11354
11355 handler
11356 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11357 .await
11358 .unwrap();
11359 let route_response = route_task.await.unwrap();
11360 assert!(route_response.is_empty());
11361 let published = route_client_rx.recv().await.unwrap();
11362 assert!(matches!(
11363 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11364 ClientControlResponse::RouteOpen { .. }
11365 ));
11366 }
11367
11368 async fn relay_route_open(
11373 handler: &ControlHandler,
11374 client_connection: ConnectionId,
11375 client_egress: &FrameSink,
11376 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11377 corr: u64,
11378 module_id: &str,
11379 project_root_label: &str,
11380 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
11381 let ctx = RouteCtx {
11382 connection_id: client_connection,
11383 egress: client_egress.clone(),
11384 };
11385 let handler = handler.clone();
11386 let project_root = unique_project_root(project_root_label);
11387 let module_id = module_id.to_string();
11388 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
11389 let task = tokio::spawn(async move {
11390 let _guard = tracing::dispatcher::set_default(&dispatch);
11391 handler
11392 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
11393 .await
11394 .unwrap()
11395 });
11396 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
11397 .await
11398 .expect("module receives the relayed route.bind")
11399 .expect("module egress is open");
11400 (task, bind.frame)
11401 }
11402
11403 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
11404 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
11405 ModuleControlRequest::RouteBind {
11406 route_channel,
11407 epoch,
11408 ..
11409 } => (route_channel, epoch),
11410 other => panic!("expected a route.bind request, got {other:?}"),
11411 }
11412 }
11413
11414 fn published_route(frame: &Frame) -> (u16, u32) {
11415 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
11416 ClientControlResponse::RouteOpen {
11417 route_channel,
11418 route_epoch,
11419 ..
11420 } => (route_channel, route_epoch),
11421 other => panic!("expected a route.open response, got {other:?}"),
11422 }
11423 }
11424
11425 #[tokio::test]
11445 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
11446 let registry = Arc::new(Registry::default());
11447 let forwarding = Arc::new(ForwardingTable::default());
11448 let handler =
11449 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11450
11451 let module_connection = ConnectionId::new(30);
11452 let (module_ctx, mut module_rx) = route_ctx(module_connection);
11453 hello_via_sink(
11454 &handler,
11455 &module_ctx,
11456 &mut module_rx,
11457 hello_frame("aft", PROTOCOL_VERSION, 7),
11458 )
11459 .await;
11460
11461 let dying_client = ConnectionId::new(31);
11462 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
11463
11464 let (first_task, first_bind) = relay_route_open(
11467 &handler,
11468 dying_client,
11469 &dying_ctx.egress,
11470 &mut module_rx,
11471 100,
11472 "aft",
11473 "closing-first",
11474 )
11475 .await;
11476 handler
11477 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
11478 .await
11479 .unwrap();
11480 assert!(first_task.await.unwrap().is_empty());
11481 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
11482
11483 let (second_task, second_bind) = relay_route_open(
11485 &handler,
11486 dying_client,
11487 &dying_ctx.egress,
11488 &mut module_rx,
11489 101,
11490 "aft",
11491 "closing-second",
11492 )
11493 .await;
11494 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
11495
11496 assert!(forwarding
11499 .escalate_client_delivery_failure(
11500 dying_client,
11501 first_channel,
11502 first_epoch,
11503 CloseReason::new(
11504 "module_to_client_delivery_failed",
11505 "client egress refused a module frame",
11506 ),
11507 crate::forwarding::UndeliveredFrame {
11508 module_id: None,
11509 sink: &dying_ctx.egress,
11510 },
11511 )
11512 .unwrap());
11513 assert!(!dying_ctx.egress.is_closed());
11514
11515 let ack = handler
11517 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
11518 .await;
11519 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11520 if module_loop_error.is_some() {
11521 handler.cleanup_connection(module_connection).unwrap();
11525 }
11526 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11530 .await
11531 .ok()
11532 .flatten();
11533
11534 assert!(
11536 registry
11537 .get_module_by_connection(module_connection)
11538 .unwrap()
11539 .is_some(),
11540 "one client's closing connection ended the shared module connection: \
11541 {module_loop_error:?}"
11542 );
11543 let cotenant = ConnectionId::new(32);
11545 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11546 let (cotenant_task, cotenant_bind) = relay_route_open(
11547 &handler,
11548 cotenant,
11549 &cotenant_ctx.egress,
11550 &mut module_rx,
11551 102,
11552 "aft",
11553 "closing-cotenant",
11554 )
11555 .await;
11556 handler
11557 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
11558 .await
11559 .unwrap();
11560 assert!(cotenant_task.await.unwrap().is_empty());
11561 let (cotenant_channel, cotenant_epoch) =
11562 published_route(&cotenant_rx.recv().await.unwrap());
11563 assert!(matches!(
11564 forwarding
11565 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11566 .unwrap(),
11567 DataRoute::Client(DataRouteState::Bound(_))
11568 ));
11569
11570 let goodbye = post_ack_module_frame
11573 .expect("module receives a GOODBYE for the abandoned route channel");
11574 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11575 assert_eq!(goodbye.header.channel, abandoned_channel);
11576 assert_eq!(goodbye.header.epoch, abandoned_epoch);
11577
11578 assert!(dying_rx.try_recv().is_err());
11582 let second_response = second_task.await.unwrap();
11583 assert_eq!(second_response.len(), 1);
11584 assert_eq!(
11585 parse_error(&second_response[0])["code"],
11586 "target_unavailable"
11587 );
11588 }
11589
11590 async fn catalog_snapshot(
11593 handler: &ControlHandler,
11594 client_ctx: &RouteCtx,
11595 corr: u64,
11596 ) -> (u64, Vec<(String, bool)>) {
11597 let body =
11598 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
11599 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
11600 let reply = handler
11601 .handle_control_frame(client_ctx, request)
11602 .await
11603 .unwrap()
11604 .pop()
11605 .unwrap();
11606 let ClientControlResponse::CatalogList {
11607 generation,
11608 modules,
11609 ..
11610 } = serde_json::from_slice(&reply.body).unwrap()
11611 else {
11612 panic!("not a catalog.list reply");
11613 };
11614 (
11615 generation,
11616 modules
11617 .into_iter()
11618 .map(|entry| (entry.module_id, entry.ready))
11619 .collect(),
11620 )
11621 }
11622
11623 fn hello_held_before_forwarding() -> (
11629 ControlHandler,
11630 Arc<std::sync::Barrier>,
11631 std::thread::JoinHandle<Vec<Frame>>,
11632 RouteCtx,
11633 mpsc::Receiver<crate::router::OutboundFrame>,
11634 ) {
11635 let hold = Arc::new(std::sync::Barrier::new(2));
11636 let handler = ControlHandler::with_forwarding(
11637 Arc::new(Registry::default()),
11638 Arc::new(ForwardingTable::default()),
11639 )
11640 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
11641 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11642 let hello = {
11643 let handler = handler.clone();
11644 let module_ctx = module_ctx.clone();
11645 std::thread::spawn(move || {
11646 handler
11647 .handle_hello(
11648 module_ctx.connection_id,
11649 Some(module_ctx.egress.clone()),
11650 hello_frame("aft", PROTOCOL_VERSION, 7),
11651 )
11652 .unwrap()
11653 })
11654 };
11655 (handler, hold, hello, module_ctx, module_rx)
11656 }
11657
11658 #[tokio::test]
11667 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
11668 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
11669 let client = ConnectionId::new(41);
11670 let (client_ctx, mut client_rx) = route_ctx(client);
11671
11672 hold.wait();
11673 assert!(handler.registry.get_module("aft").unwrap().is_some());
11675 assert!(!handler
11676 .forwarding
11677 .has_live_module_connection("aft")
11678 .unwrap());
11679 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
11680 assert!(
11681 listed.is_empty(),
11682 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
11683 );
11684 let refused = handler
11685 .handle_control_frame(
11686 &client_ctx,
11687 route_open_frame(301, "aft", unique_project_root("listed-window")),
11688 )
11689 .await
11690 .unwrap();
11691 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
11692 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
11693 let duplicate = handler
11694 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
11695 .await
11696 .unwrap();
11697 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11698
11699 hold.wait();
11700 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11701 let ack = module_rx.try_recv().unwrap().frame;
11702 assert_eq!(ack.header.ty, FrameType::HelloAck);
11703 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11704 assert_eq!(listed, vec![("aft".to_string(), true)]);
11705
11706 let (open_task, bind) = relay_route_open(
11707 &handler,
11708 client,
11709 &client_ctx.egress,
11710 &mut module_rx,
11711 303,
11712 "aft",
11713 "listed-after",
11714 )
11715 .await;
11716 assert!(handler
11717 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11718 .await
11719 .unwrap()
11720 .is_empty());
11721 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
11722 published_route(&client_rx.try_recv().unwrap().frame);
11723 }
11724
11725 #[tokio::test]
11732 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11733 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11734 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11735 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11736
11737 hold.wait();
11738 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11739 assert!(listed.is_empty());
11740 assert_eq!(
11741 during, before,
11742 "the generation moved while the module was not listed"
11743 );
11744
11745 hold.wait();
11746 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11747 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11748 assert_eq!(listed, vec![("aft".to_string(), true)]);
11749 assert_eq!(
11750 after,
11751 before + 1,
11752 "the module appeared without exactly one generation bump"
11753 );
11754 }
11755
11756 #[tokio::test]
11761 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11762 let registry = Arc::new(Registry::default());
11763 let handler = ControlHandler::with_forwarding(
11764 Arc::clone(®istry),
11765 Arc::new(ForwardingTable::default()),
11766 );
11767 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11768 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11769 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11770 drop(module_rx);
11771
11772 let replies = handler
11773 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
11774 .await
11775 .unwrap();
11776 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
11777 assert!(registry.get_module("aft").unwrap().is_none());
11778 assert!(registry
11779 .get_module_by_connection(module_ctx.connection_id)
11780 .unwrap()
11781 .is_none());
11782 assert_eq!(
11783 catalog_snapshot(&handler, &client_ctx, 301).await,
11784 (before, Vec::new())
11785 );
11786 }
11787
11788 async fn route_open_held_after_bind_send() -> (
11792 ControlHandler,
11793 Arc<tokio::sync::Barrier>,
11794 RouteCtx,
11795 mpsc::Receiver<crate::router::OutboundFrame>,
11796 ) {
11797 let hold = Arc::new(tokio::sync::Barrier::new(2));
11798 let handler = ControlHandler::with_forwarding(
11799 Arc::new(Registry::default()),
11800 Arc::new(ForwardingTable::default()),
11801 )
11802 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
11803 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11804 hello_via_sink(
11805 &handler,
11806 &module_ctx,
11807 &mut module_rx,
11808 hello_frame("aft", PROTOCOL_VERSION, 7),
11809 )
11810 .await;
11811 (handler, hold, module_ctx, module_rx)
11812 }
11813
11814 #[tokio::test]
11821 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
11822 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11823 let client = ConnectionId::new(41);
11824 let (client_ctx, mut client_rx) = route_ctx(client);
11825
11826 let (open_task, bind) = relay_route_open(
11827 &handler,
11828 client,
11829 &client_ctx.egress,
11830 &mut module_rx,
11831 200,
11832 "aft",
11833 "acked-before-mark",
11834 )
11835 .await;
11836 let (module_channel, module_epoch) = route_bind_channel(&bind);
11837 hold.wait().await;
11839
11840 assert!(handler
11842 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11843 .await
11844 .unwrap()
11845 .is_empty());
11846 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
11847
11848 hold.wait().await;
11849 assert!(
11850 open_task.await.unwrap().is_empty(),
11851 "route.open answers accepted"
11852 );
11853
11854 match module_rx.try_recv() {
11857 Err(mpsc::error::TryRecvError::Empty) => {}
11858 Ok(outbound) => panic!(
11859 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
11860 outbound.frame.header.ty,
11861 outbound.frame.header.channel,
11862 outbound.frame.header.epoch
11863 ),
11864 Err(err) => panic!("module egress closed: {err}"),
11865 }
11866
11867 let router = Router::with_control_handler(Arc::new(handler.clone()));
11870 let request = Frame::build(
11871 FrameType::Request,
11872 Flags::new(true, Priority::Interactive, false),
11873 client_channel,
11874 client_epoch,
11875 201,
11876 b"first request".to_vec(),
11877 )
11878 .unwrap();
11879 router
11880 .route_for_connection(&client_ctx, request)
11881 .await
11882 .unwrap();
11883 let delivered = module_rx.try_recv().unwrap().frame;
11884 assert_eq!(delivered.header.ty, FrameType::Request);
11885 assert_eq!(
11886 (delivered.header.channel, delivered.header.epoch),
11887 (module_channel, module_epoch)
11888 );
11889 assert_eq!(delivered.body, b"first request");
11890 }
11891
11892 #[tokio::test]
11898 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
11899 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11900 let client = ConnectionId::new(42);
11901 let (client_ctx, _client_rx) = route_ctx(client);
11902
11903 let (open_task, bind) = relay_route_open(
11904 &handler,
11905 client,
11906 &client_ctx.egress,
11907 &mut module_rx,
11908 300,
11909 "aft",
11910 "released-before-mark",
11911 )
11912 .await;
11913 let (module_channel, module_epoch) = route_bind_channel(&bind);
11914 hold.wait().await;
11915
11916 handler.cleanup_connection(client).unwrap();
11917 assert!(
11918 module_rx.try_recv().is_err(),
11919 "client teardown must not GOODBYE a bind route.open is still queueing"
11920 );
11921
11922 hold.wait().await;
11923 let response = open_task.await.unwrap();
11924 assert_eq!(response.len(), 1);
11925 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11926
11927 let goodbye = module_rx
11928 .try_recv()
11929 .expect("module is told to drop the binding the daemon abandoned")
11930 .frame;
11931 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11932 assert_eq!(
11933 (goodbye.header.channel, goodbye.header.epoch),
11934 (module_channel, module_epoch)
11935 );
11936 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11937 }
11938
11939 #[tokio::test]
11944 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11945 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11946 let client = ConnectionId::new(43);
11947 let (client_ctx, client_rx) = route_ctx(client);
11948
11949 let (open_task, bind) = relay_route_open(
11950 &handler,
11951 client,
11952 &client_ctx.egress,
11953 &mut module_rx,
11954 400,
11955 "aft",
11956 "gone-client-before-mark",
11957 )
11958 .await;
11959 let (module_channel, module_epoch) = route_bind_channel(&bind);
11960 hold.wait().await;
11961
11962 drop(client_rx);
11963 assert!(client_ctx.egress.is_closed());
11964 handler
11965 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11966 .await
11967 .unwrap();
11968
11969 hold.wait().await;
11970 let response = open_task.await.unwrap();
11971 assert_eq!(response.len(), 1);
11972 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11973
11974 let goodbye = module_rx
11975 .try_recv()
11976 .expect("module is told to drop the binding it acked")
11977 .frame;
11978 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11979 assert_eq!(
11980 (goodbye.header.channel, goodbye.header.epoch),
11981 (module_channel, module_epoch)
11982 );
11983 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11984 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11985 }
11986
11987 #[test]
11994 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11995 let handler = ControlHandler::default();
11996 let module_connection = ConnectionId::new(30);
11997 let client_connection = ConnectionId::new(31);
11998
11999 handler
12000 .refuse_to_end_module_connection_for_a_client(
12001 module_connection,
12002 77,
12003 ForwardingError::ConnectionClosing {
12004 connection_id: client_connection,
12005 },
12006 )
12007 .expect("a closing client must never end the module connection");
12008
12009 assert!(matches!(
12010 handler.refuse_to_end_module_connection_for_a_client(
12011 module_connection,
12012 78,
12013 ForwardingError::ConnectionClosing {
12014 connection_id: module_connection,
12015 },
12016 ),
12017 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
12018 connection_id
12019 })) if connection_id == module_connection
12020 ));
12021 assert!(matches!(
12022 handler.refuse_to_end_module_connection_for_a_client(
12023 module_connection,
12024 79,
12025 ForwardingError::Poisoned,
12026 ),
12027 Err(RouterError::Forwarding(ForwardingError::Poisoned))
12028 ));
12029 assert!(matches!(
12030 handler.refuse_to_end_module_connection_for_a_client(
12031 module_connection,
12032 80,
12033 ForwardingError::StaleModuleEndpoint,
12034 ),
12035 Err(RouterError::Forwarding(
12036 ForwardingError::StaleModuleEndpoint
12037 ))
12038 ));
12039 }
12040
12041 #[tokio::test]
12054 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
12055 let registry = Arc::new(Registry::default());
12056 let forwarding = Arc::new(ForwardingTable::default());
12057 let supervisor = SupervisorHandle::new();
12058 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12059 let handler =
12060 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12061 .with_supervisor(supervisor);
12062
12063 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12064 hello_via_sink(
12065 &handler,
12066 &target_ctx,
12067 &mut target_rx,
12068 hello_frame("target", PROTOCOL_VERSION, 1),
12069 )
12070 .await;
12071
12072 let wrong_nonce = handler
12076 .handle_control_frame(
12077 &route_ctx(ConnectionId::new(91)).0,
12078 route_open_frame_with_admission_facts(
12079 20,
12080 "target",
12081 unique_project_root("admission-facts"),
12082 Some(subc_control::ConsumerIdentity {
12083 module_id: "fed".to_string(),
12084 launch_nonce: "not-the-real-nonce".to_string(),
12085 }),
12086 None,
12087 ),
12088 )
12089 .await
12090 .unwrap();
12091 assert_eq!(
12092 parse_error(&wrong_nonce[0])["code"],
12093 "bad_consumer_identity",
12094 "a mismatched launch nonce must be refused, not stamped Reserved"
12095 );
12096
12097 let never_spawned = handler
12101 .handle_control_frame(
12102 &route_ctx(ConnectionId::new(92)).0,
12103 route_open_frame_with_admission_facts(
12104 21,
12105 "target",
12106 unique_project_root("admission-facts"),
12107 Some(subc_control::ConsumerIdentity {
12108 module_id: "never-spawned".to_string(),
12109 launch_nonce: "any-nonce".to_string(),
12110 }),
12111 None,
12112 ),
12113 )
12114 .await
12115 .unwrap();
12116 assert_eq!(
12117 parse_error(&never_spawned[0])["code"],
12118 "bad_consumer_identity",
12119 "an unspawned module_id must be refused rather than accepted for lack of a record"
12120 );
12121 }
12122
12123 #[tokio::test]
12146 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
12147 let registry = Arc::new(Registry::default());
12148 let forwarding = Arc::new(ForwardingTable::default());
12149 let supervisor = SupervisorHandle::new();
12150 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12151 let handler =
12152 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12153 .with_supervisor(supervisor);
12154
12155 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
12156 hello_via_sink(
12157 &handler,
12158 &target_ctx,
12159 &mut target_rx,
12160 hello_frame("target", PROTOCOL_VERSION, 1),
12161 )
12162 .await;
12163
12164 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
12165 let route_handler = handler.clone();
12166 let route_task = tokio::spawn(async move {
12167 route_handler
12168 .handle_control_frame(
12169 &client_ctx,
12170 route_open_frame_with_admission_facts(
12171 30,
12172 "target",
12173 unique_project_root("admission-facts"),
12174 Some(subc_control::ConsumerIdentity {
12175 module_id: "fed".to_string(),
12176 launch_nonce: "fed-nonce".to_string(),
12177 }),
12178 None,
12179 ),
12180 )
12181 .await
12182 .unwrap()
12183 });
12184
12185 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12192 .await
12193 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
12194 .expect("module control channel closed before route.bind");
12195 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12196 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12197 panic!("expected route.bind")
12198 };
12199 assert_eq!(
12200 principal,
12201 Some(Principal::Reserved {
12202 module_id: "fed".to_string()
12203 }),
12204 "a correctly attested consumer must be stamped Reserved for its own id"
12205 );
12206
12207 handler
12208 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12209 .await
12210 .unwrap();
12211 assert!(route_task.await.unwrap().is_empty());
12212 assert!(
12213 matches!(
12214 serde_json::from_slice::<ClientControlResponse>(
12215 &client_rx.recv().await.unwrap().body
12216 )
12217 .unwrap(),
12218 ClientControlResponse::RouteOpen { .. }
12219 ),
12220 "the route must actually open, not merely avoid an error"
12221 );
12222 }
12223
12224 #[tokio::test(start_paused = true)]
12225 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
12226 let registry = Arc::new(Registry::default());
12227 let forwarding = Arc::new(ForwardingTable::default());
12228 let supervisor = SupervisorHandle::new();
12229 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12230 let handler =
12231 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12232 .with_supervisor(supervisor);
12233
12234 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
12235 hello_via_sink(
12236 &handler,
12237 &target_ctx,
12238 &mut target_rx,
12239 hello_frame("target", PROTOCOL_VERSION, 1),
12240 )
12241 .await;
12242
12243 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
12244 let direct_handler = handler.clone();
12245 let direct_open = tokio::spawn(async move {
12246 direct_handler
12247 .handle_control_frame(
12248 &direct_ctx,
12249 route_open_frame(2, "target", unique_project_root("route-census-direct")),
12250 )
12251 .await
12252 .unwrap()
12253 });
12254 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12255 .await
12256 .expect("no direct route.bind within 5s")
12257 .expect("target control channel closed before direct route.bind");
12258 handler
12259 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
12260 .await
12261 .unwrap();
12262 assert!(direct_open.await.unwrap().is_empty());
12263 let _ = direct_rx.recv().await.unwrap();
12264
12265 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
12266 let reserved_handler = handler.clone();
12267 let reserved_open = tokio::spawn(async move {
12268 reserved_handler
12269 .handle_control_frame(
12270 &reserved_ctx,
12271 route_open_frame_with_admission_facts(
12272 3,
12273 "target",
12274 unique_project_root("admission-facts"),
12275 Some(ConsumerIdentity {
12276 module_id: "fed".to_string(),
12277 launch_nonce: "fed-nonce".to_string(),
12278 }),
12279 None,
12280 ),
12281 )
12282 .await
12283 .unwrap()
12284 });
12285 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12286 .await
12287 .expect("no reserved route.bind within 5s")
12288 .expect("target control channel closed before reserved route.bind");
12289 handler
12290 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
12291 .await
12292 .unwrap();
12293 assert!(reserved_open.await.unwrap().is_empty());
12294 let _ = reserved_rx.recv().await.unwrap();
12295
12296 forwarding
12297 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
12298 .unwrap();
12299 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
12300 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
12301 module_id: Some("target".to_string()),
12302 })
12303 .unwrap();
12304 let census_frame =
12305 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
12306 let response = handler
12307 .handle_control_frame(&census_ctx, census_frame)
12308 .await
12309 .unwrap()
12310 .pop()
12311 .unwrap();
12312 let actual: Value = serde_json::from_slice(&response.body).unwrap();
12313 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
12314 assert!(matches!(
12315 decoded,
12316 ClientControlResponse::SupervisorRoutes { .. }
12317 ));
12318 let routes = actual["modules"][0]["routes"].as_array().unwrap();
12319 assert_eq!(routes.len(), 2);
12320 assert!(routes.iter().all(|route| route["draining"] == true));
12321 assert!(
12324 routes.iter().all(|route| route["drain_reason"] == "reload"),
12325 "draining routes must name the drain's reason: {routes:?}"
12326 );
12327 assert!(routes.iter().any(|route| {
12328 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
12329 }));
12330 assert!(routes.iter().any(|route| {
12331 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
12332 }));
12333
12334 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12335 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
12336 if std::env::var_os("UPDATE_GOLDEN").is_some() {
12337 std::fs::write(
12338 &golden_path,
12339 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
12340 )
12341 .unwrap();
12342 }
12343 let expected: Value =
12344 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
12345 assert_eq!(actual, expected);
12346 }
12347
12348 async fn query_live_roots(
12349 handler: &ControlHandler,
12350 module_ctx: &RouteCtx,
12351 ) -> ModuleControlResponseToModule {
12352 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
12353 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
12354 let response = handler
12355 .handle_control_frame(module_ctx, frame)
12356 .await
12357 .unwrap()
12358 .pop()
12359 .unwrap();
12360 serde_json::from_slice(&response.body).unwrap()
12361 }
12362
12363 #[tokio::test(start_paused = true)]
12364 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
12365 let registry = Arc::new(Registry::default());
12366 let forwarding = Arc::new(ForwardingTable::default());
12367 let handler = ControlHandler::with_forwarding(registry, forwarding);
12368 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
12369 hello_via_sink(
12370 &handler,
12371 &target_ctx,
12372 &mut target_rx,
12373 hello_frame("target", PROTOCOL_VERSION, 1),
12374 )
12375 .await;
12376 let root = unique_project_root("live-roots-known");
12377 let path = ProjectRootId::from_path_allowing_missing(root.path())
12378 .unwrap()
12379 .as_path()
12380 .to_path_buf();
12381 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
12382 let open_handler = handler.clone();
12383 let opened = tokio::spawn(async move {
12384 open_handler
12385 .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
12386 .await
12387 .unwrap()
12388 });
12389 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12390 .await
12391 .unwrap()
12392 .unwrap();
12393 handler
12394 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
12395 .await
12396 .unwrap();
12397 assert!(opened.await.unwrap().is_empty());
12398 let _ = client_rx.recv().await.unwrap();
12399
12400 let root = unique_project_root("live-roots-pending");
12401 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
12402 .unwrap()
12403 .as_path()
12404 .to_path_buf();
12405 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
12406 let open_handler = handler.clone();
12407 let pending = tokio::spawn(async move {
12408 open_handler
12409 .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
12410 .await
12411 .unwrap()
12412 });
12413 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12414 .await
12415 .unwrap()
12416 .unwrap();
12417 let actual = query_live_roots(&handler, &target_ctx).await;
12418 let ModuleControlResponseToModule::LiveRoots {
12419 roots,
12420 unknown_root_bindings,
12421 total_bindings,
12422 } = actual
12423 else {
12424 panic!("expected live roots")
12425 };
12426 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
12427 assert_eq!(unknown_root_bindings, 0);
12428 assert_eq!(
12429 roots.len(),
12430 2,
12431 "root-known arm must retain each canonical root"
12432 );
12433 assert_eq!(
12434 total_bindings,
12435 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12436 );
12437 let counts = roots
12438 .iter()
12439 .map(|root| (root.project_root.clone(), root.bound, root.pending))
12440 .collect::<Vec<_>>();
12441 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
12442 expected.sort_by(|a, b| a.0.cmp(&b.0));
12443 assert_eq!(
12444 counts, expected,
12445 "roots must sort by path and count pending separately"
12446 );
12447 handler
12448 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
12449 .await
12450 .unwrap();
12451 assert!(pending.await.unwrap().is_empty());
12452 }
12453
12454 #[tokio::test(start_paused = true)]
12455 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
12456 let forwarding = Arc::new(ForwardingTable::default());
12457 let handler =
12458 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12459 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
12460 hello_via_sink(
12461 &handler,
12462 &target_ctx,
12463 &mut target_rx,
12464 hello_frame("target", PROTOCOL_VERSION, 1),
12465 )
12466 .await;
12467 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
12468 let pending = forwarding
12469 .begin_route_bind_relay_for_test(
12470 client_ctx.connection_id,
12471 client_ctx.egress.clone(),
12472 2,
12473 "target",
12474 )
12475 .unwrap();
12476 forwarding
12477 .complete_pending_relay(
12478 target_ctx.connection_id,
12479 pending.corr,
12480 RouteBindRelayOutcome::Accepted,
12481 )
12482 .unwrap();
12483 let actual = query_live_roots(&handler, &target_ctx).await;
12484 let ModuleControlResponseToModule::LiveRoots {
12485 roots,
12486 unknown_root_bindings,
12487 total_bindings,
12488 } = actual
12489 else {
12490 panic!("expected live roots")
12491 };
12492 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
12493 assert_eq!(
12494 unknown_root_bindings, 1,
12495 "unknown-root arm must not read as no bindings"
12496 );
12497 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
12498 assert_eq!(
12499 total_bindings,
12500 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12501 );
12502 }
12503
12504 #[tokio::test(start_paused = true)]
12512 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
12513 let forwarding = Arc::new(ForwardingTable::default());
12514 let handler =
12515 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12516 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
12517 let replies = handler
12518 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
12519 .await
12520 .unwrap();
12521 let queued_by_hello = module_rx.len();
12522
12523 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
12524 let open_handler = handler.clone();
12525 let open = tokio::spawn(async move {
12526 open_handler
12527 .handle_control_frame(
12528 &client_ctx,
12529 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
12530 )
12531 .await
12532 .unwrap()
12533 });
12534 let mut spins = 0;
12536 while module_rx.len() == queued_by_hello {
12537 spins += 1;
12538 assert!(spins < 10_000, "route.open never queued a route.bind");
12539 tokio::task::yield_now().await;
12540 }
12541
12542 for reply in replies {
12544 module_ctx.egress.send(reply).await.unwrap();
12545 }
12546
12547 let first = module_rx.recv().await.unwrap().frame;
12548 assert_eq!(
12549 first.header.ty,
12550 FrameType::HelloAck,
12551 "the first frame a registering module reads must be its HELLO_ACK"
12552 );
12553 assert_eq!(first.header.corr, 7);
12554 let second = module_rx.recv().await.unwrap().frame;
12555 assert_eq!(second.header.ty, FrameType::Request);
12556 assert!(
12557 matches!(
12558 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
12559 ModuleControlRequest::RouteBind { .. }
12560 ),
12561 "the route.bind follows the ack"
12562 );
12563 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
12564
12565 handler
12566 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
12567 .await
12568 .unwrap();
12569 assert!(open.await.unwrap().is_empty());
12570 let _ = client_rx.recv().await.unwrap();
12571 }
12572
12573 #[tokio::test(start_paused = true)]
12574 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
12575 let handler = ControlHandler::with_forwarding(
12576 Arc::new(Registry::default()),
12577 Arc::new(ForwardingTable::default()),
12578 );
12579 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
12580 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
12581 hello_via_sink(
12582 &handler,
12583 &first_ctx,
12584 &mut first_rx,
12585 hello_frame("first", PROTOCOL_VERSION, 1),
12586 )
12587 .await;
12588 hello_via_sink(
12589 &handler,
12590 &second_ctx,
12591 &mut second_rx,
12592 hello_frame("second", PROTOCOL_VERSION, 2),
12593 )
12594 .await;
12595 let root = unique_project_root("second-only");
12596 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
12597 let cloned = handler.clone();
12598 let open = tokio::spawn(async move {
12599 cloned
12600 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
12601 .await
12602 .unwrap()
12603 });
12604 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
12605 .await
12606 .unwrap()
12607 .unwrap();
12608 let first = query_live_roots(&handler, &first_ctx).await;
12609 let second = query_live_roots(&handler, &second_ctx).await;
12610 assert!(
12611 matches!(
12612 first,
12613 ModuleControlResponseToModule::LiveRoots {
12614 total_bindings: 0,
12615 ..
12616 }
12617 ),
12618 "cross-module scope must not expose another module's roots"
12619 );
12620 assert!(
12621 matches!(
12622 second,
12623 ModuleControlResponseToModule::LiveRoots {
12624 total_bindings: 1,
12625 ..
12626 }
12627 ),
12628 "second module must see its pending route"
12629 );
12630 handler
12631 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
12632 .await
12633 .unwrap();
12634 assert!(open.await.unwrap().is_empty());
12635 }
12636
12637 #[tokio::test(start_paused = true)]
12638 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
12639 let handler = ControlHandler::with_forwarding(
12640 Arc::new(Registry::default()),
12641 Arc::new(ForwardingTable::default()),
12642 );
12643 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
12644 hello_via_sink(
12645 &handler,
12646 &target_ctx,
12647 &mut target_rx,
12648 hello_frame("target", PROTOCOL_VERSION, 1),
12649 )
12650 .await;
12651 let actual = query_live_roots(&handler, &target_ctx).await;
12652 let ModuleControlResponseToModule::LiveRoots {
12653 roots,
12654 unknown_root_bindings,
12655 total_bindings,
12656 } = actual
12657 else {
12658 panic!("expected live roots")
12659 };
12660 assert!(roots.is_empty());
12661 assert_eq!(unknown_root_bindings, 0);
12662 assert_eq!(total_bindings, 0);
12663 assert_eq!(
12664 total_bindings,
12665 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12666 );
12667 }
12668
12669 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
12676 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12677 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
12678 let text = std::fs::read_to_string(&path)
12679 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
12680 let vectors: Vec<(String, Value)> = text
12681 .lines()
12682 .filter(|line| !line.trim().is_empty())
12683 .map(|line| {
12684 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
12685 let id = entry["corpus_id"]
12686 .as_str()
12687 .expect("every vector carries a corpus_id")
12688 .to_string();
12689 (id, entry["package"].clone())
12690 })
12691 .collect();
12692 assert_eq!(
12696 vectors.len(),
12697 3,
12698 "vendored fed corpus changed size; re-sync from subc-federation"
12699 );
12700
12701 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
12716 let richest = vectors
12717 .iter()
12718 .filter_map(|(_, package)| package.as_object())
12719 .map(|object| {
12720 object
12721 .keys()
12722 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
12723 .count()
12724 })
12725 .max()
12726 .unwrap_or(0);
12727 assert!(
12728 richest >= 2,
12729 "vendored corpus no longer carries a package with unmodelled fields, \
12730 so the relay test can no longer distinguish a verbatim relay from a lossy one"
12731 );
12732
12733 vectors
12734 }
12735
12736 #[tokio::test]
12744 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12745 for (corpus_id, package) in fed_admission_facts_vectors() {
12746 let registry = Arc::new(Registry::default());
12747 let forwarding = Arc::new(ForwardingTable::default());
12748 let supervisor = SupervisorHandle::new();
12749 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12750 let handler =
12751 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12752 .with_supervisor(supervisor)
12753 .with_admission_facts_config(
12754 Some("fed".to_string()),
12755 Some(vec!["target".to_string()]),
12756 );
12757
12758 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12759 hello_via_sink(
12760 &handler,
12761 &target_ctx,
12762 &mut target_rx,
12763 hello_frame("target", PROTOCOL_VERSION, 1),
12764 )
12765 .await;
12766
12767 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12768 let route_handler = handler.clone();
12769 let expected = package.clone();
12770 let route_task = tokio::spawn(async move {
12771 route_handler
12772 .handle_control_frame(
12773 &client_ctx,
12774 route_open_frame_with_admission_facts(
12775 20,
12776 "target",
12777 unique_project_root("admission-facts"),
12778 Some(subc_control::ConsumerIdentity {
12779 module_id: "fed".to_string(),
12780 launch_nonce: "fed-nonce".to_string(),
12781 }),
12782 Some(package),
12783 ),
12784 )
12785 .await
12786 .unwrap()
12787 });
12788
12789 let bind_frame = target_rx.recv().await.unwrap();
12790 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12791 let ModuleControlRequest::RouteBind {
12792 admission_facts, ..
12793 } = bind
12794 else {
12795 panic!("{corpus_id}: expected route.bind")
12796 };
12797 assert_eq!(
12798 admission_facts,
12799 Some(expected),
12800 "{corpus_id}: relay must not add, drop or reshape any field"
12801 );
12802
12803 handler
12804 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12805 .await
12806 .unwrap();
12807 route_task.await.unwrap();
12808 }
12809 }
12810
12811 #[tokio::test]
12812 async fn route_bind_never_relays_the_openers_build_class() {
12813 let registry = Arc::new(Registry::default());
12814 let forwarding = Arc::new(ForwardingTable::default());
12815 let handler =
12816 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12817 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
12818 hello_via_sink(
12819 &handler,
12820 &target_ctx,
12821 &mut target_rx,
12822 hello_frame("target", PROTOCOL_VERSION, 1),
12823 )
12824 .await;
12825
12826 let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
12827 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
12828 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12829 busy_refusals: None,
12830 target: RouteTarget::ToolProvider {
12831 module_id: "target".to_string(),
12832 },
12833 identity,
12834 consumer_identity: None,
12835 consumer_capabilities: None,
12836 role_versions: None,
12837 admission_facts: None,
12838 scope: None,
12839 })
12840 .unwrap();
12841 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
12842
12843 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
12844 let route_handler = handler.clone();
12845 let route_task = tokio::spawn(async move {
12846 route_handler
12847 .handle_control_frame(&client_ctx, open)
12848 .await
12849 .unwrap()
12850 });
12851 let bind_frame = target_rx.recv().await.unwrap();
12852 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12853 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
12854 panic!("expected route.bind")
12855 };
12856 assert_eq!(
12857 identity.build_class, None,
12858 "route.bind carried the build class"
12859 );
12860 assert!(
12861 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
12862 "the route.bind bytes still name build_class"
12863 );
12864 handler
12865 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12866 .await
12867 .unwrap();
12868 route_task.await.unwrap();
12869 }
12870
12871 #[tokio::test]
12872 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
12873 let registry = Arc::new(Registry::default());
12874 let forwarding = Arc::new(ForwardingTable::default());
12875 let supervisor = SupervisorHandle::new();
12876 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12877 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
12878 let handler =
12879 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12880 .with_supervisor(supervisor)
12881 .with_admission_facts_config(
12882 Some("fed".to_string()),
12883 Some(vec!["target".to_string()]),
12884 );
12885
12886 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
12887 hello_via_sink(
12888 &handler,
12889 &target_ctx,
12890 &mut target_rx,
12891 hello_frame("target", PROTOCOL_VERSION, 1),
12892 )
12893 .await;
12894 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
12895 hello_via_sink(
12896 &handler,
12897 &other_ctx,
12898 &mut other_rx,
12899 hello_frame("other", PROTOCOL_VERSION, 2),
12900 )
12901 .await;
12902
12903 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
12904 let expected_facts = facts.clone();
12905 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
12906 let route_handler = handler.clone();
12907 let route_task = tokio::spawn(async move {
12908 route_handler
12909 .handle_control_frame(
12910 &client_ctx,
12911 route_open_frame_with_admission_facts(
12912 10,
12913 "target",
12914 unique_project_root("admission-facts"),
12915 Some(subc_control::ConsumerIdentity {
12916 module_id: "fed".to_string(),
12917 launch_nonce: "fed-nonce".to_string(),
12918 }),
12919 Some(facts.clone()),
12920 ),
12921 )
12922 .await
12923 .unwrap()
12924 });
12925 let bind_frame = target_rx.recv().await.unwrap();
12926 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12927 let ModuleControlRequest::RouteBind {
12928 admission_facts, ..
12929 } = bind
12930 else {
12931 panic!("expected route.bind")
12932 };
12933 assert_eq!(admission_facts, Some(expected_facts));
12934 handler
12935 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12936 .await
12937 .unwrap();
12938 assert!(route_task.await.unwrap().is_empty());
12939 assert!(matches!(
12940 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
12941 .unwrap(),
12942 ClientControlResponse::RouteOpen { .. }
12943 ));
12944
12945 let direct = handler
12946 .handle_control_frame(
12947 &route_ctx(ConnectionId::new(73)).0,
12948 route_open_frame_with_admission_facts(
12949 11,
12950 "target",
12951 unique_project_root("admission-facts"),
12952 None,
12953 Some(json!({"x": 1})),
12954 ),
12955 )
12956 .await
12957 .unwrap();
12958 assert_eq!(
12959 parse_error(&direct[0])["code"],
12960 "admission_facts_not_permitted"
12961 );
12962
12963 let different_reserved = handler
12964 .handle_control_frame(
12965 &route_ctx(ConnectionId::new(77)).0,
12966 route_open_frame_with_admission_facts(
12967 15,
12968 "target",
12969 unique_project_root("admission-facts"),
12970 Some(subc_control::ConsumerIdentity {
12971 module_id: "other".to_string(),
12972 launch_nonce: "other-nonce".to_string(),
12973 }),
12974 Some(json!({"x": 1})),
12975 ),
12976 )
12977 .await
12978 .unwrap();
12979 assert_eq!(
12980 parse_error(&different_reserved[0])["code"],
12981 "admission_facts_not_permitted"
12982 );
12983
12984 let other_target = handler
12985 .handle_control_frame(
12986 &route_ctx(ConnectionId::new(74)).0,
12987 route_open_frame_with_admission_facts(
12988 12,
12989 "other",
12990 unique_project_root("admission-facts"),
12991 Some(subc_control::ConsumerIdentity {
12992 module_id: "fed".to_string(),
12993 launch_nonce: "fed-nonce".to_string(),
12994 }),
12995 Some(json!({"x": 1})),
12996 ),
12997 )
12998 .await
12999 .unwrap();
13000 assert_eq!(
13001 parse_error(&other_target[0])["code"],
13002 "admission_facts_target_not_allowed"
13003 );
13004
13005 let nonexistent = handler
13006 .handle_control_frame(
13007 &route_ctx(ConnectionId::new(75)).0,
13008 route_open_frame_with_admission_facts(
13009 13,
13010 "missing",
13011 unique_project_root("admission-facts"),
13012 None,
13013 Some(json!({"x": 1})),
13014 ),
13015 )
13016 .await
13017 .unwrap();
13018 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
13019
13020 let described = handler
13021 .handle_control_frame(
13022 &route_ctx(ConnectionId::new(76)).0,
13023 Frame::build(
13024 FrameType::Request,
13025 control_flags(),
13026 0,
13027 0,
13028 14,
13029 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13030 )
13031 .unwrap(),
13032 )
13033 .await
13034 .unwrap();
13035 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13036 serde_json::from_slice(&described[0].body).unwrap()
13037 else {
13038 panic!("expected server.describe response")
13039 };
13040 assert!(capabilities
13041 .iter()
13042 .any(|cap| cap == "admission_facts_relay_v1"));
13043 }
13044
13045 #[tokio::test]
13046 async fn admission_facts_without_configured_carrier_are_rejected() {
13047 let registry = Arc::new(Registry::default());
13048 let forwarding = Arc::new(ForwardingTable::default());
13049 let handler = ControlHandler::with_forwarding(registry, forwarding);
13050 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
13051 hello_via_sink(
13052 &handler,
13053 &target_ctx,
13054 &mut target_rx,
13055 hello_frame("target", PROTOCOL_VERSION, 1),
13056 )
13057 .await;
13058
13059 let responses = handler
13060 .handle_control_frame(
13061 &route_ctx(ConnectionId::new(79)).0,
13062 route_open_frame_with_admission_facts(
13063 16,
13064 "target",
13065 unique_project_root("admission-facts"),
13066 None,
13067 Some(json!({"x": 1})),
13068 ),
13069 )
13070 .await
13071 .unwrap();
13072 assert_eq!(
13073 parse_error(&responses[0])["code"],
13074 "admission_facts_not_permitted"
13075 );
13076 }
13077
13078 #[tokio::test]
13079 async fn route_open_relays_consumer_capabilities_verbatim() {
13080 let registry = Arc::new(Registry::default());
13081 let forwarding = Arc::new(ForwardingTable::default());
13082 let handler =
13083 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13084 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
13085 hello_via_sink(
13086 &handler,
13087 &module_ctx,
13088 &mut module_rx,
13089 hello_frame("aft", PROTOCOL_VERSION, 7),
13090 )
13091 .await;
13092
13093 let expected = vec!["elicitation".to_string(), "roots".to_string()];
13094 let expected_for_request = expected.clone();
13095 let project_root = unique_project_root("consumer-capabilities-present");
13096 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
13097 let route_handler = handler.clone();
13098 let route_task = tokio::spawn(async move {
13099 route_handler
13100 .handle_control_frame(
13101 &client_ctx,
13102 route_open_frame_with_consumer_capabilities(
13103 401,
13104 "aft",
13105 project_root,
13106 Some(expected_for_request),
13107 ),
13108 )
13109 .await
13110 .unwrap()
13111 });
13112 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13113 .await
13114 .unwrap()
13115 .unwrap();
13116 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13117 let ModuleControlRequest::RouteBind {
13118 consumer_capabilities,
13119 ..
13120 } = bind
13121 else {
13122 panic!("expected route.bind request, got {bind:?}");
13123 };
13124 assert_eq!(consumer_capabilities, Some(expected.clone()));
13125
13126 handler
13127 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13128 .await
13129 .unwrap();
13130 let route_response = route_task.await.unwrap();
13131 assert!(route_response.is_empty());
13132 let published = client_rx.recv().await.unwrap();
13133 assert!(matches!(
13134 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13135 ClientControlResponse::RouteOpen { .. }
13136 ));
13137 }
13138
13139 #[tokio::test]
13140 async fn route_open_without_consumer_capabilities_relays_none() {
13141 let registry = Arc::new(Registry::default());
13142 let forwarding = Arc::new(ForwardingTable::default());
13143 let handler =
13144 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13145 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
13146 hello_via_sink(
13147 &handler,
13148 &module_ctx,
13149 &mut module_rx,
13150 hello_frame("aft", PROTOCOL_VERSION, 7),
13151 )
13152 .await;
13153
13154 let project_root = unique_project_root("consumer-capabilities-absent");
13155 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
13156 let route_handler = handler.clone();
13157 let route_task = tokio::spawn(async move {
13158 route_handler
13159 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
13160 .await
13161 .unwrap()
13162 });
13163 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13164 .await
13165 .unwrap()
13166 .unwrap();
13167 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13168 let ModuleControlRequest::RouteBind {
13169 consumer_capabilities,
13170 ..
13171 } = bind
13172 else {
13173 panic!("expected route.bind request, got {bind:?}");
13174 };
13175 assert_eq!(consumer_capabilities, None);
13176
13177 handler
13178 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13179 .await
13180 .unwrap();
13181 let route_response = route_task.await.unwrap();
13182 assert!(route_response.is_empty());
13183 let published = client_rx.recv().await.unwrap();
13184 assert!(matches!(
13185 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13186 ClientControlResponse::RouteOpen { .. }
13187 ));
13188 }
13189
13190 async fn bind_role_versions_for(
13194 sent: Option<BTreeMap<String, String>>,
13195 connection: u64,
13196 ) -> Option<BTreeMap<String, String>> {
13197 let registry = Arc::new(Registry::default());
13198 let forwarding = Arc::new(ForwardingTable::default());
13199 let handler =
13200 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13201 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
13202 hello_via_sink(
13203 &handler,
13204 &module_ctx,
13205 &mut module_rx,
13206 hello_frame("aft", PROTOCOL_VERSION, 7),
13207 )
13208 .await;
13209 let project_root = unique_project_root("role-versions");
13210 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
13211 let route_handler = handler.clone();
13212 let route_task = tokio::spawn(async move {
13213 route_handler
13214 .handle_control_frame(
13215 &client_ctx,
13216 route_open_frame_with_role_versions(403, "aft", project_root, sent),
13217 )
13218 .await
13219 .unwrap()
13220 });
13221 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13222 .await
13223 .expect("a well-formed route.open reaches the module as a bind")
13224 .unwrap();
13225 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13226 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
13227 panic!("expected route.bind request, got {bind:?}");
13228 };
13229 handler
13230 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13231 .await
13232 .unwrap();
13233 assert!(route_task.await.unwrap().is_empty());
13234 let published = client_rx.recv().await.unwrap();
13235 assert!(matches!(
13236 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13237 ClientControlResponse::RouteOpen { .. }
13238 ));
13239 role_versions
13240 }
13241
13242 #[tokio::test]
13243 async fn route_open_relays_role_versions_verbatim() {
13244 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
13245 assert_eq!(
13246 bind_role_versions_for(Some(sent.clone()), 141).await,
13247 Some(sent)
13248 );
13249 }
13250
13251 #[tokio::test]
13254 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
13255 assert_eq!(bind_role_versions_for(None, 143).await, None);
13256 assert_eq!(
13257 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
13258 None
13259 );
13260 }
13261
13262 #[tokio::test]
13263 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
13264 let registry = Arc::new(Registry::default());
13265 let forwarding = Arc::new(ForwardingTable::default());
13266 let handler =
13267 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13268 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
13269 hello_via_sink(
13270 &handler,
13271 &module_ctx,
13272 &mut module_rx,
13273 hello_frame("aft", PROTOCOL_VERSION, 7),
13274 )
13275 .await;
13276
13277 let nine: BTreeMap<String, String> = (0..9)
13278 .map(|index| (format!("role-{index}"), "v1".to_string()))
13279 .collect();
13280 for (label, malformed) in [
13281 (
13282 "invalid role name",
13283 role_versions(&[("Tool_Provider", "v1")]),
13284 ),
13285 ("invalid version", role_versions(&[("tool-provider", "v0")])),
13286 ("nine entries", nine),
13287 ] {
13288 let responses = tokio::time::timeout(
13291 Duration::from_secs(1),
13292 handler.handle_control_frame(
13293 &route_ctx(ConnectionId::new(148)).0,
13294 route_open_frame_with_role_versions(
13295 404,
13296 "aft",
13297 unique_project_root("role-versions-malformed"),
13298 Some(malformed),
13299 ),
13300 ),
13301 )
13302 .await
13303 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
13304 .unwrap();
13305 assert_eq!(responses.len(), 1, "{label}");
13306 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
13307 let error = parse_error(&responses[0]);
13308 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
13309 assert_eq!(
13310 error["detail"]["field"], "role_versions",
13311 "{label}: {error}"
13312 );
13313 assert!(
13314 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
13315 "{label}: a malformed declaration is terminal"
13316 );
13317 assert!(
13318 module_rx.try_recv().is_err(),
13319 "{label}: the module must never see a bind"
13320 );
13321 }
13322 }
13323
13324 #[tokio::test]
13328 async fn route_role_versions_capability_is_advertised() {
13329 let handler = ControlHandler::new(Arc::new(Registry::default()));
13330 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
13331 let ack = hello_via_sink(
13332 &handler,
13333 &ctx,
13334 &mut rx,
13335 hello_frame("m", PROTOCOL_VERSION, 1),
13336 )
13337 .await;
13338 let ack = parse_ack(&ack);
13339 assert!(
13340 ack.subc_capabilities
13341 .iter()
13342 .any(|c| c == "route-role-versions/v1"),
13343 "{:?}",
13344 ack.subc_capabilities
13345 );
13346
13347 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13348 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13349 let reply = handler
13350 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
13351 .await
13352 .unwrap()
13353 .pop()
13354 .unwrap();
13355 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13356 serde_json::from_slice(&reply.body).unwrap()
13357 else {
13358 panic!("not a server.describe reply");
13359 };
13360 assert!(
13361 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
13362 "{capabilities:?}"
13363 );
13364 }
13365
13366 #[tokio::test]
13370 async fn catalog_events_capability_is_advertised() {
13371 let handler = ControlHandler::new(Arc::new(Registry::default()));
13372 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13373 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13374 let reply = handler
13375 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
13376 .await
13377 .unwrap()
13378 .pop()
13379 .unwrap();
13380 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13381 serde_json::from_slice(&reply.body).unwrap()
13382 else {
13383 panic!("not a server.describe reply");
13384 };
13385 assert!(
13386 capabilities
13387 .iter()
13388 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
13389 "{capabilities:?}"
13390 );
13391 }
13392
13393 #[tokio::test]
13394 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
13395 let handler = ControlHandler::new(Arc::new(Registry::default()));
13396 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
13397 let ack = hello_via_sink(
13398 &handler,
13399 &module,
13400 &mut module_rx,
13401 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
13402 )
13403 .await;
13404 let ack = parse_ack(&ack);
13405 assert!(
13406 ack.subc_capabilities
13407 .iter()
13408 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13409 "{:?}",
13410 ack.subc_capabilities
13411 );
13412 assert!(ack
13413 .subc_capabilities
13414 .iter()
13415 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13416
13417 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
13418 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13419 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13420 let reply = handler
13421 .handle_control_frame(&client, frame)
13422 .await
13423 .unwrap()
13424 .pop()
13425 .unwrap();
13426 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13427 serde_json::from_slice(&reply.body).unwrap()
13428 else {
13429 panic!("not a server.describe reply");
13430 };
13431 assert!(
13432 capabilities
13433 .iter()
13434 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13435 "{capabilities:?}"
13436 );
13437 assert!(capabilities
13438 .iter()
13439 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13440 }
13441
13442 #[tokio::test]
13443 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
13444 let handler = ControlHandler::new(Arc::new(Registry::default()));
13445 let (module, mut rx) = route_ctx(ConnectionId::new(156));
13446 let ack = hello_via_sink(
13447 &handler,
13448 &module,
13449 &mut rx,
13450 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
13451 )
13452 .await;
13453 assert!(parse_ack(&ack)
13454 .subc_capabilities
13455 .iter()
13456 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13457 assert!(parse_ack(&ack)
13458 .subc_capabilities
13459 .iter()
13460 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13461 let (client, _rx) = route_ctx(ConnectionId::new(157));
13462 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13463 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13464 let reply = handler
13465 .handle_control_frame(&client, frame)
13466 .await
13467 .unwrap()
13468 .pop()
13469 .unwrap();
13470 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13471 serde_json::from_slice(&reply.body).unwrap()
13472 else {
13473 panic!("server.describe reply")
13474 };
13475 assert!(capabilities
13476 .iter()
13477 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13478 assert!(capabilities
13479 .iter()
13480 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13481 }
13482
13483 #[tokio::test]
13484 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
13485 let handler = ControlHandler::new(Arc::new(Registry::default()));
13486 let (module, mut rx) = route_ctx(ConnectionId::new(158));
13487 let ack = hello_via_sink(
13488 &handler,
13489 &module,
13490 &mut rx,
13491 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
13492 )
13493 .await;
13494 assert!(parse_ack(&ack)
13495 .subc_capabilities
13496 .iter()
13497 .any(|cap| cap == "scope-workdir/v1"));
13498 let (client, _rx) = route_ctx(ConnectionId::new(159));
13499 let frame = Frame::build(
13500 FrameType::Request,
13501 control_flags(),
13502 0,
13503 0,
13504 5,
13505 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13506 )
13507 .unwrap();
13508 let reply = handler
13509 .handle_control_frame(&client, frame)
13510 .await
13511 .unwrap()
13512 .remove(0);
13513 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13514 serde_json::from_slice(&reply.body).unwrap()
13515 else {
13516 panic!("server.describe reply")
13517 };
13518 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
13519 }
13520
13521 #[tokio::test]
13522 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
13523 let handler = ControlHandler::new(Arc::new(Registry::default()));
13524 let (module, mut rx) = route_ctx(ConnectionId::new(160));
13525 let ack = hello_via_sink(
13526 &handler,
13527 &module,
13528 &mut rx,
13529 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
13530 )
13531 .await;
13532 assert!(parse_ack(&ack)
13533 .subc_capabilities
13534 .iter()
13535 .any(|cap| cap == "worker-runs/v1"));
13536 let (client, _rx) = route_ctx(ConnectionId::new(161));
13537 let frame = Frame::build(
13538 FrameType::Request,
13539 control_flags(),
13540 0,
13541 0,
13542 5,
13543 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13544 )
13545 .unwrap();
13546 let reply = handler
13547 .handle_control_frame(&client, frame)
13548 .await
13549 .unwrap()
13550 .remove(0);
13551 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13552 serde_json::from_slice(&reply.body).unwrap()
13553 else {
13554 panic!("server.describe reply")
13555 };
13556 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
13557 }
13558
13559 #[tokio::test]
13560 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
13561 let handler = ControlHandler::new(Arc::new(Registry::default()));
13562 let (module, mut rx) = route_ctx(ConnectionId::new(154));
13563 let ack = hello_via_sink(
13564 &handler,
13565 &module,
13566 &mut rx,
13567 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
13568 )
13569 .await;
13570 assert!(parse_ack(&ack)
13571 .subc_capabilities
13572 .iter()
13573 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13574 let (client, _rx) = route_ctx(ConnectionId::new(155));
13575 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13576 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13577 let reply = handler
13578 .handle_control_frame(&client, frame)
13579 .await
13580 .unwrap()
13581 .pop()
13582 .unwrap();
13583 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13584 serde_json::from_slice(&reply.body).unwrap()
13585 else {
13586 panic!("not a server.describe reply")
13587 };
13588 assert!(capabilities
13589 .iter()
13590 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13591 }
13592
13593 #[tokio::test]
13594 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
13595 let registry = Arc::new(Registry::default());
13596 let forwarding = Arc::new(ForwardingTable::default());
13597 let handler =
13598 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13599 .with_health_probe_timeout(Duration::from_secs(5));
13600 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
13601 hello_via_sink(
13602 &handler,
13603 &module_ctx,
13604 &mut module_rx,
13605 non_routable_hello_frame_with_control_ops(
13606 "mcp",
13607 300,
13608 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13609 ),
13610 )
13611 .await;
13612 assert!(registry
13613 .get_module("mcp")
13614 .unwrap()
13615 .unwrap()
13616 .manifest
13617 .provides
13618 .is_empty());
13619
13620 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
13621 let route_response = handler
13622 .handle_control_frame(
13623 &route_client_ctx,
13624 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
13625 )
13626 .await
13627 .unwrap();
13628 assert_eq!(route_response[0].header.ty, FrameType::Error);
13629 assert_eq!(
13630 parse_error(&route_response[0])["code"],
13631 "target_unavailable"
13632 );
13633 assert!(parse_error(&route_response[0])["message"]
13634 .as_str()
13635 .unwrap()
13636 .contains("does not provide the requested target"));
13637 assert!(module_rx.try_recv().is_err());
13638
13639 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
13640 let health_handler = handler.clone();
13641 let health_task = tokio::spawn(async move {
13642 health_handler
13643 .handle_control_frame(
13644 &health_client_ctx,
13645 supervisor_health_probe_frame(302, "mcp"),
13646 )
13647 .await
13648 .unwrap()
13649 });
13650 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13651 .await
13652 .unwrap()
13653 .unwrap();
13654 assert_eq!(
13655 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13656 ModuleControlRequest::HealthCheck {}
13657 );
13658 handler
13659 .handle_control_frame(
13660 &module_ctx,
13661 health_response(health_frame.header.corr, HealthStatus::Ok),
13662 )
13663 .await
13664 .unwrap();
13665 let health_response = health_task.await.unwrap();
13666 assert_eq!(health_response[0].header.ty, FrameType::Response);
13667 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13668 ClientControlResponse::SupervisorHealthProbe {
13669 module_id, status, ..
13670 } => {
13671 assert_eq!(module_id, "mcp");
13672 assert_eq!(status, HealthStatus::Ok);
13673 }
13674 other => panic!("unexpected health response: {other:?}"),
13675 }
13676
13677 forwarding
13682 .cleanup_connection(module_ctx.connection_id)
13683 .unwrap();
13684 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
13685 let cleanup_response = tokio::time::timeout(
13686 Duration::from_millis(200),
13687 handler.handle_control_frame(
13688 &cleanup_probe_ctx,
13689 supervisor_health_probe_frame(303, "mcp"),
13690 ),
13691 )
13692 .await
13693 .expect("probe should fail immediately when the control lane is gone")
13694 .unwrap();
13695 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
13696 assert_eq!(
13697 parse_error(&cleanup_response[0])["code"],
13698 "target_unavailable"
13699 );
13700 assert!(parse_error(&cleanup_response[0])["message"]
13701 .as_str()
13702 .unwrap()
13703 .contains("no module connection"));
13704
13705 handler
13706 .cleanup_connection(module_ctx.connection_id)
13707 .unwrap();
13708 }
13709
13710 #[tokio::test]
13711 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
13712 let registry = Arc::new(Registry::default());
13713 let supervisor_handle = SupervisorHandle::new();
13714 let supervisor =
13715 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13716 .with_handle(supervisor_handle.clone())
13717 .with_connection_file_path(
13718 std::env::temp_dir()
13719 .join(format!("subc-route-open-warming-{}", std::process::id())),
13720 );
13721 let module = supervisor
13722 .supervise_configured(
13723 ModuleSpec {
13724 carry_containment: false,
13725 module_id: "warming".to_string(),
13726 program: fake_aft_stub_path(),
13727 args: Vec::new(),
13728 env: Vec::new(),
13729 reserved: false,
13730 reserved_prefixes: Vec::new(),
13731 protocol: ModuleProtocol::Subc,
13732 overlap: Default::default(),
13733 },
13734 true,
13735 )
13736 .unwrap();
13737 assert_eq!(module.state().unwrap(), ModuleState::Running);
13738
13739 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13740 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13741 let response = handler
13742 .handle_control_frame(
13743 &ctx,
13744 route_open_frame(304, "warming", unique_project_root("warming")),
13745 )
13746 .await
13747 .unwrap();
13748 module.stop().await.unwrap();
13749
13750 assert_eq!(response[0].header.ty, FrameType::Error);
13751 let error = parse_error(&response[0]);
13752 assert_eq!(error["code"], "module_warming");
13753 assert!(error["message"]
13754 .as_str()
13755 .unwrap()
13756 .contains("state=running, enabled=true, live=false"));
13757 }
13758
13759 #[test]
13760 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13761 let handler = ControlHandler::new(Arc::new(Registry::default()));
13762 let capture = EventCapture::default();
13763 let _subscriber =
13764 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13765 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13766 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
13767 let pending = (0..limit).collect::<Vec<_>>();
13768 let response = handler
13769 .route_open_capacity_refusal(
13770 &ctx,
13771 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
13772 "busy",
13773 pending.len(),
13774 limit,
13775 )
13776 .unwrap();
13777 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13778 let event = capture
13779 .events()
13780 .into_iter()
13781 .find(|event| {
13782 event.target == "control"
13783 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
13784 })
13785 .expect("connection admission refusal event");
13786 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13787 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13788 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13791 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13792 }
13793
13794 #[test]
13795 fn route_open_target_cap_logs_admission_reason_and_capacity() {
13796 let handler = ControlHandler::new(Arc::new(Registry::default()));
13797 let capture = EventCapture::default();
13798 let _subscriber =
13799 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13800 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
13801 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
13802 let guards = (0..limit)
13803 .map(|_| {
13804 handler
13805 .route_bind_concurrency
13806 .try_admit("busy", limit)
13807 .unwrap()
13808 })
13809 .collect::<Vec<_>>();
13810 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
13811 Err(in_flight) => in_flight,
13812 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
13813 };
13814 let response = handler
13815 .route_open_target_capacity_refusal(
13816 &ctx,
13817 &RouteOpenCaller::unknown(),
13818 &route_open_frame(397, "busy", unique_project_root("target-cap")),
13819 "busy",
13820 in_flight,
13821 )
13822 .unwrap();
13823 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13824 let event = capture
13825 .events()
13826 .into_iter()
13827 .find(|event| {
13828 event.target == "control"
13829 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
13830 })
13831 .expect("target admission refusal event");
13832 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13833 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13834 drop(guards);
13835 }
13836
13837 #[cfg(unix)]
13838 #[tokio::test(start_paused = true)]
13839 async fn supervisor_list_shows_a_failed_modules_next_retry() {
13840 let registry = Arc::new(Registry::default());
13841 let handle = SupervisorHandle::new();
13842 let supervisor =
13843 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
13844 .with_handle(handle.clone());
13845 let module = supervisor
13846 .spawn(crate::ModuleSpec {
13847 carry_containment: false,
13848 module_id: "retry-list".into(),
13849 program: PathBuf::from("/bin/sh"),
13850 args: vec!["-c".into(), "exit 17".into()],
13851 env: vec![],
13852 reserved: false,
13853 reserved_prefixes: vec![],
13854 protocol: ModuleProtocol::None,
13855 overlap: Default::default(),
13856 })
13857 .unwrap();
13858 let deadline = std::time::Instant::now() + Duration::from_secs(10);
13859 while module.state().unwrap() != crate::ModuleState::Failed {
13860 assert!(
13861 std::time::Instant::now() < deadline,
13862 "waiting for budget failure"
13863 );
13864 tokio::task::yield_now().await;
13865 }
13866 let handler = ControlHandler::new(registry).with_supervisor(handle);
13867 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13868 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13869 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13870 assert_eq!(body["modules"][0]["state"], "failed");
13871 let retry = body["modules"][0]["next_retry_at_ms"]
13872 .as_u64()
13873 .expect("scheduled retry on the wire");
13874 let now = std::time::SystemTime::now()
13875 .duration_since(std::time::UNIX_EPOCH)
13876 .unwrap()
13877 .as_millis() as u64;
13878 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
13879 module.set_enabled(false).await.unwrap();
13880 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13881 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13882 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13883 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
13884 }
13885
13886 #[tokio::test]
13887 async fn slow_step_before_pid_read_does_not_spend_its_budget() {
13888 let _guard = crate::subtree_resources::fault_read_guard().await;
13889 let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
13890 let supervisor =
13891 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13892 let module = supervisor
13893 .spawn(crate::ModuleSpec {
13894 carry_containment: false,
13895 module_id: "pid-read-budget".into(),
13896 program: std::env::current_exe().unwrap(),
13897 args: vec![
13898 "--exact".into(),
13899 "control::tests::pid_resource_read_fixture".into(),
13900 ],
13901 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13902 .into_iter()
13903 .map(|key| (key.into(), dir.display().to_string()))
13904 .chain(std::iter::once((
13905 "SUBC_SUBTREE_PID_FIXTURE".into(),
13906 "1".into(),
13907 )))
13908 .collect(),
13909 reserved: false,
13910 reserved_prefixes: Vec::new(),
13911 protocol: ModuleProtocol::None,
13912 overlap: Default::default(),
13913 })
13914 .unwrap();
13915 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
13916 while !matches!(
13917 module.child_resource_usage(),
13918 subc_control::ChildResourceUsage::Measured(_)
13919 ) {
13920 assert!(
13921 tokio::time::Instant::now() < ready_deadline,
13922 "fixture process was not measurable"
13923 );
13924 tokio::time::sleep(Duration::from_millis(10)).await;
13925 }
13926 tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
13927 let reading = crate::subtree_resources::read_process(module.clone()).await;
13928 module.stop().await.unwrap();
13929 assert!(
13930 matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
13931 "{reading:?}"
13932 );
13933 }
13934
13935 #[test]
13936 fn pid_resource_read_fixture() {
13937 if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
13938 loop {
13939 std::thread::park_timeout(Duration::from_secs(600));
13940 }
13941 }
13942 }
13943
13944 #[tokio::test]
13945 async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
13946 let _guard = crate::subtree_resources::fault_read_guard().await;
13947 let registry = Arc::new(Registry::default());
13948 let handle = SupervisorHandle::new();
13949 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
13950 .with_handle(handle.clone());
13951 let module = supervisor
13952 .supervise_configured(
13953 crate::ModuleSpec {
13954 carry_containment: false,
13955 module_id: "memory-read".into(),
13956 program: PathBuf::from("disabled-module"),
13957 args: Vec::new(),
13958 env: Vec::new(),
13959 reserved: false,
13960 reserved_prefixes: Vec::new(),
13961 protocol: ModuleProtocol::Subc,
13962 overlap: Default::default(),
13963 },
13964 false,
13965 )
13966 .unwrap();
13967 let (target, release, began) = crate::subtree_resources::held();
13968 module.set_subtree_target_for_test(target);
13969 let handler = ControlHandler::new(registry).with_supervisor(handle);
13970 let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
13971 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13972 let replies = handler
13973 .handle_client_control_request(
13974 &ctx,
13975 frame,
13976 ClientControlRequest::SupervisorList {},
13977 &mut None,
13978 )
13979 .await
13980 .unwrap();
13981 assert!(
13982 replies.is_empty(),
13983 "list reply must be detached from the connection reader"
13984 );
13985 began.await.unwrap();
13986 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13987 let replies = handler
13988 .handle_client_control_request(
13989 &ctx,
13990 frame,
13991 ClientControlRequest::SupervisorSpawnSnapshot {},
13992 &mut None,
13993 )
13994 .await
13995 .unwrap();
13996 assert_eq!(
13997 replies.len(),
13998 1,
13999 "another request must complete while the subtree read stalls"
14000 );
14001 let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
14002 .await
14003 .unwrap()
14004 .unwrap();
14005 release.send(()).unwrap();
14006 drop(release);
14007 let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
14008 assert_eq!(body["modules"][0]["subtree"]["source"], "unavailable");
14009 assert_eq!(body["modules"][0]["subtree"]["reason"], "timeout");
14010 assert_eq!(
14011 body["modules"][0]["subtree"]["process_count"], 1,
14012 "a count obtained before the stalled read must survive timeout"
14013 );
14014 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14015 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14016 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14017 assert_eq!(
14018 body["modules"][0]["subtree"]["source"], "process_sum",
14019 "the next request must read afresh"
14020 );
14021 }
14022
14023 #[tokio::test]
14024 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
14025 let registry = Arc::new(Registry::default());
14026 let handle = SupervisorHandle::new();
14027 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14028 .with_handle(handle.clone());
14029 supervisor
14030 .supervise_configured(
14031 crate::ModuleSpec {
14032 carry_containment: false,
14033 module_id: "drift".into(),
14034 program: PathBuf::from("disabled-module"),
14035 args: Vec::new(),
14036 env: Vec::new(),
14037 reserved: false,
14038 reserved_prefixes: Vec::new(),
14039 protocol: ModuleProtocol::Subc,
14040 overlap: Default::default(),
14041 },
14042 false,
14043 )
14044 .unwrap();
14045 let handler = ControlHandler::new(registry).with_supervisor(handle);
14046 for configured in [None, Some(false), Some(true)] {
14047 handler.capability_evaluator.configure(
14048 configured.map(|enabled| ("drift".to_string(), enabled)),
14049 BTreeMap::new(),
14050 );
14051 let frame =
14052 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14053 let replies = handler
14054 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
14055 .await
14056 .unwrap();
14057 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14058 assert!(body.get("configured_enabled").is_none());
14059 assert_eq!(body["op"], "supervisor.ack");
14060 let frame =
14061 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14062 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14063 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14064 assert_eq!(
14065 body["modules"][0].get("configured_enabled"),
14066 configured.map(Value::Bool).as_ref()
14067 );
14068 assert_eq!(body["modules"][0]["subtree"]["source"], "unavailable");
14069 assert_eq!(body["modules"][0]["subtree"]["reason"], "not_running");
14070 }
14071 }
14072
14073 #[tokio::test(flavor = "current_thread")]
14074 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
14075 let handler = ControlHandler::new(Arc::new(Registry::default()));
14076 let capture = EventCapture::default();
14077 let _subscriber =
14078 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14079 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
14080 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14081
14082 let _ = handler
14083 .handle_client_control_request(
14084 &ctx,
14085 frame,
14086 ClientControlRequest::SupervisorSetEnabled {
14087 module_id: "broca".to_string(),
14088 enabled: false,
14089 },
14090 &mut None,
14091 )
14092 .await
14093 .unwrap();
14094
14095 let events = capture
14096 .events()
14097 .into_iter()
14098 .filter(|event| {
14099 event.target == "control"
14100 && event.fields.get("message").map(String::as_str)
14101 == Some("supervisor request received")
14102 })
14103 .collect::<Vec<_>>();
14104 assert_eq!(events.len(), 1, "one supervisor request log line");
14105 let event = &events[0];
14106 assert_eq!(event.level, tracing::Level::INFO);
14107 assert_eq!(
14108 event.fields.get("op"),
14109 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
14110 );
14111 assert_eq!(
14112 event.fields.get("module_id"),
14113 Some(&"\"broca\"".to_string())
14114 );
14115 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
14116 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
14117 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
14118 }
14119
14120 #[tokio::test(flavor = "current_thread")]
14121 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
14122 let registry = Arc::new(Registry::default());
14123 let module_connection = ConnectionId::new(303);
14124 registry
14125 .register_with_control_ops(
14126 manifest("aft", PROTOCOL_VERSION),
14127 PROTOCOL_VERSION,
14128 module_connection,
14129 module_baseline_control_ops(),
14130 )
14131 .unwrap();
14132 let handler = ControlHandler::new(registry);
14133 let capture = EventCapture::default();
14134 let _subscriber =
14135 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14136 let (ctx, _rx) = route_ctx(module_connection);
14137 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14138
14139 let _ = handler
14140 .handle_client_control_request(
14141 &ctx,
14142 frame,
14143 ClientControlRequest::SupervisorSetEnabled {
14144 module_id: "broca".to_string(),
14145 enabled: false,
14146 },
14147 &mut None,
14148 )
14149 .await
14150 .unwrap();
14151
14152 let callers = capture
14153 .events()
14154 .into_iter()
14155 .filter(|event| {
14156 event.fields.get("message").map(String::as_str)
14157 == Some("supervisor request received")
14158 })
14159 .map(|event| event.fields.get("caller").cloned())
14160 .collect::<Vec<_>>();
14161 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
14162 }
14163
14164 #[tokio::test(flavor = "current_thread")]
14165 async fn supervisor_list_does_not_log_request_received() {
14166 let handler = ControlHandler::new(Arc::new(Registry::default()));
14167 let capture = EventCapture::default();
14168 let _subscriber =
14169 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14170 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
14171 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14172
14173 handler
14174 .handle_client_control_request(
14175 &ctx,
14176 frame,
14177 ClientControlRequest::SupervisorList {},
14178 &mut None,
14179 )
14180 .await
14181 .unwrap();
14182
14183 assert!(capture.events().into_iter().all(|event| {
14184 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14185 }));
14186 }
14187
14188 #[tokio::test(flavor = "current_thread")]
14189 async fn supervisor_rescan_preview_does_not_log_request_received() {
14190 let handler = ControlHandler::new(Arc::new(Registry::default()));
14191 let capture = EventCapture::default();
14192 let _subscriber =
14193 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14194 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
14195 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14196
14197 handler
14198 .handle_client_control_request(
14199 &ctx,
14200 frame,
14201 ClientControlRequest::SupervisorRescan { preview: true },
14202 &mut None,
14203 )
14204 .await
14205 .unwrap();
14206
14207 assert!(capture.events().into_iter().all(|event| {
14208 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14209 }));
14210 }
14211
14212 struct ApproveOperator;
14213 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
14214 fn prompt(
14215 &self,
14216 _: &str,
14217 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
14218 ) -> crate::operator_confirm::ProviderResult {
14219 crate::operator_confirm::ProviderResult::Approved
14220 }
14221 }
14222
14223 #[tokio::test]
14224 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
14225 assert_eq!(
14226 ControlHandler::default().privileged_operator_prompt,
14227 PrivilegedOperatorPrompt::Log
14228 );
14229 for (initial, changed, running) in [
14230 ("log", "enforce", PrivilegedOperatorPrompt::Log),
14231 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
14232 ] {
14233 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
14234 let path = dir.join("subc.jsonc");
14235 std::fs::write(
14236 &path,
14237 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
14238 )
14239 .unwrap();
14240 let config = crate::daemon_config::load(&path).unwrap().unwrap();
14241 let registry = Arc::new(Registry::default());
14242 let handle = SupervisorHandle::new();
14243 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
14244 .with_handle(handle.clone());
14245 let handler = ControlHandler::new(registry)
14246 .with_supervisor(handle)
14247 .with_privileged_operator_prompt(config.privileged_operator_prompt)
14248 .with_supervisor_rescan(supervisor, &path, config.port);
14249 handler.forwarding.operator_confirms().configure(
14250 Arc::new(ApproveOperator),
14251 crate::operator_confirm::OperatorLimits::default(),
14252 );
14253 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
14254
14255 for (disk_mode, needs_restart) in [
14256 (Some(initial), false),
14257 (Some(changed), true),
14258 (Some(changed), true),
14259 (None, running == PrivilegedOperatorPrompt::Enforce),
14260 (Some(initial), false),
14261 ] {
14262 let mut doc = json!({"version":1});
14263 if let Some(mode) = disk_mode {
14264 doc["privileged_operator_prompt"] = json!(mode);
14265 }
14266 std::fs::write(&path, doc.to_string()).unwrap();
14267 for preview in [true, false] {
14268 let request = ClientControlRequest::SupervisorRescan { preview };
14269 let frame = Frame::build(
14270 FrameType::Request,
14271 control_flags(),
14272 0,
14273 0,
14274 1,
14275 serde_json::to_vec(&request).unwrap(),
14276 )
14277 .unwrap();
14278 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14279 assert_eq!(frames.len(), 1);
14280 let ClientControlResponse::SupervisorRescan { result } =
14281 serde_json::from_slice(&frames[0].body).unwrap()
14282 else {
14283 panic!("expected a rescan response");
14284 };
14285 let expected = if needs_restart {
14286 vec!["privileged_operator_prompt"]
14287 } else {
14288 Vec::new()
14289 };
14290 assert_eq!(result.restart_required, expected);
14291 assert_eq!(handler.privileged_operator_prompt, running);
14292 assert_eq!(
14293 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
14294 running
14295 );
14296 }
14297 }
14298 }
14299 }
14300
14301 #[tokio::test]
14302 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
14303 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
14304 let path = dir.join("subc.jsonc");
14305 let registry = Arc::new(Registry::default());
14306 let handle = SupervisorHandle::new();
14307 let supervisor =
14308 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
14309 let handler = ControlHandler::new(registry)
14310 .with_supervisor(handle)
14311 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
14312 .with_supervisor_rescan(supervisor, &path, None);
14313 handler.forwarding.operator_confirms().configure(
14314 Arc::new(ApproveOperator),
14315 crate::operator_confirm::OperatorLimits::default(),
14316 );
14317 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
14318 for preview in [true, false] {
14319 std::fs::write(
14320 &path,
14321 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
14322 )
14323 .unwrap();
14324 let request = ClientControlRequest::SupervisorRescan { preview };
14325 let frame = Frame::build(
14326 FrameType::Request,
14327 control_flags(),
14328 0,
14329 0,
14330 1,
14331 serde_json::to_vec(&request).unwrap(),
14332 )
14333 .unwrap();
14334 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14335 assert_eq!(frames.len(), 1);
14336 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
14337 assert_eq!(
14338 handler.privileged_operator_prompt,
14339 PrivilegedOperatorPrompt::Enforce
14340 );
14341 }
14342 }
14343
14344 #[tokio::test]
14349 async fn route_open_refusal_names_the_check_that_refused() {
14350 let handler = ControlHandler::new(Arc::new(Registry::default()));
14351 let capture = EventCapture::default();
14352 let _subscriber =
14353 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14354 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14355 let response = handler
14356 .handle_control_frame(
14357 &ctx,
14358 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
14359 )
14360 .await
14361 .unwrap();
14362
14363 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14364 let event = capture
14365 .events()
14366 .into_iter()
14367 .find(|event| {
14368 event.target == "control"
14369 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14370 })
14371 .expect("route.open refusal event");
14372 assert_eq!(
14373 event.fields.get("reason"),
14374 Some(&"\"not_registered\"".to_string())
14375 );
14376 }
14377
14378 #[tokio::test]
14379 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
14380 let registry = Arc::new(Registry::default());
14381 let supervisor_handle = SupervisorHandle::new();
14382 let supervisor =
14383 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
14384 .with_handle(supervisor_handle.clone())
14385 .with_connection_file_path(std::env::temp_dir().join(format!(
14386 "subc-route-open-refusal-info-{}",
14387 std::process::id()
14388 )));
14389 let module = supervisor
14390 .supervise_configured(
14391 ModuleSpec {
14392 carry_containment: false,
14393 module_id: "warming".to_string(),
14394 program: fake_aft_stub_path(),
14395 args: Vec::new(),
14396 env: Vec::new(),
14397 reserved: false,
14398 reserved_prefixes: Vec::new(),
14399 protocol: ModuleProtocol::Subc,
14400 overlap: Default::default(),
14401 },
14402 true,
14403 )
14404 .unwrap();
14405 assert_eq!(module.state().unwrap(), ModuleState::Running);
14406
14407 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14408 assert!(handler
14409 .counters()
14410 .snapshot()
14411 .get("route_open_refused_by_code")
14412 .is_none());
14413 let capture = EventCapture::default();
14414 let _subscriber =
14415 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14416 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
14417 let response = handler
14418 .handle_control_frame(
14419 &ctx,
14420 route_open_frame(394, "warming", unique_project_root("refusal-info")),
14421 )
14422 .await
14423 .unwrap();
14424 module.stop().await.unwrap();
14425
14426 assert_eq!(parse_error(&response[0])["code"], "module_warming");
14427 let event = capture
14428 .events()
14429 .into_iter()
14430 .find(|event| {
14431 event.target == "control"
14432 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
14433 })
14434 .expect("route.open refusal event");
14435 assert_eq!(
14436 event.fields.get("module_id"),
14437 Some(&"\"warming\"".to_string())
14438 );
14439 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
14440 assert_eq!(
14441 event.fields.get("reason"),
14442 Some(&"\"supervised_not_registered\"".to_string())
14443 );
14444 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
14445 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
14446 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
14447 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14448 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14449 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
14450 assert_eq!(
14451 handler.counters().snapshot()["route_open_refused_by_code"],
14452 json!({ "module_warming": 1 })
14453 );
14454 }
14455
14456 const OUTAGE_START: &str = "route.open refusing module: not serving";
14457 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
14458
14459 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
14460 capture
14461 .events()
14462 .into_iter()
14463 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
14464 .collect()
14465 }
14466
14467 fn supervise_stub(
14468 registry: &Arc<Registry>,
14469 module_id: &str,
14470 enabled: bool,
14471 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
14472 let supervisor_handle = SupervisorHandle::new();
14473 let supervisor =
14474 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
14475 .with_handle(supervisor_handle.clone())
14476 .with_connection_file_path(std::env::temp_dir().join(format!(
14477 "subc-route-outage-{module_id}-{}",
14478 std::process::id()
14479 )));
14480 let module = supervisor
14481 .supervise_configured(
14482 ModuleSpec {
14483 carry_containment: false,
14484 module_id: module_id.to_string(),
14485 program: fake_aft_stub_path(),
14486 args: Vec::new(),
14487 env: Vec::new(),
14488 reserved: false,
14489 reserved_prefixes: Vec::new(),
14490 protocol: ModuleProtocol::Subc,
14491 overlap: Default::default(),
14492 },
14493 enabled,
14494 )
14495 .unwrap();
14496 (supervisor_handle, module)
14497 }
14498
14499 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
14500 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
14501 module_id: module_id.to_string(),
14502 drain_timeout_ms: Some(50),
14503 })
14504 .unwrap();
14505 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
14506 }
14507
14508 #[test]
14512 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
14513 let registry = Arc::new(Registry::default());
14514 let forwarding = Arc::new(ForwardingTable::default());
14515 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14516 let second = ControlHandler::with_forwarding(registry, forwarding);
14517 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
14518 }
14519
14520 #[tokio::test(flavor = "current_thread")]
14524 async fn route_open_unknown_module_refusals_add_no_outage_state() {
14525 let handler = ControlHandler::new(Arc::new(Registry::default()));
14526 let capture = EventCapture::default();
14527 let _subscriber =
14528 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14529 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
14530 for corr in 0..8 {
14531 let response = handler
14532 .handle_control_frame(
14533 &ctx,
14534 route_open_frame(
14535 380 + corr,
14536 &format!("nobody-{corr}"),
14537 unique_project_root("outage-unknown"),
14538 ),
14539 )
14540 .await
14541 .unwrap();
14542 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14543 }
14544
14545 assert_eq!(handler.route_outages.tracked_module_count(), 0);
14546 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
14547 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
14548 }
14549
14550 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14555 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
14556 let registry = Arc::new(Registry::default());
14557 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
14558 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14559 let capture = EventCapture::default();
14560 let _subscriber =
14561 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14562 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
14563 handler.route_outages.record_accepted("outage-restart");
14566
14567 let response = handler
14568 .handle_control_frame(
14569 &ctx,
14570 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
14571 )
14572 .await
14573 .unwrap();
14574 assert_eq!(response[0].header.ty, FrameType::Error);
14575 let starts = outage_lines(&capture, OUTAGE_START);
14576 assert_eq!(starts.len(), 1, "{starts:?}");
14577 assert_eq!(starts[0].level, tracing::Level::WARN);
14578 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
14579 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
14580 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
14581 handler.route_outages.record_accepted("outage-restart");
14582 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
14583
14584 let restart = handler
14585 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
14586 .await
14587 .unwrap();
14588 assert_eq!(
14589 restart[0].header.ty,
14590 FrameType::Response,
14591 "{:?}",
14592 parse_error(&restart[0])
14593 );
14594 handler
14595 .handle_control_frame(
14596 &ctx,
14597 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
14598 )
14599 .await
14600 .unwrap();
14601 module.stop().await.unwrap();
14602
14603 let starts = outage_lines(&capture, OUTAGE_START);
14604 assert_eq!(starts.len(), 2, "{starts:?}");
14605 assert_eq!(starts[1].level, tracing::Level::INFO);
14606 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
14607 }
14608
14609 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14613 async fn failed_operator_restart_leaves_no_operator_mark() {
14614 let registry = Arc::new(Registry::default());
14615 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
14616 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14617 let capture = EventCapture::default();
14618 let _subscriber =
14619 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14620 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
14621 handler.route_outages.record_accepted("outage-disabled");
14622
14623 let restart = handler
14624 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
14625 .await
14626 .unwrap();
14627 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
14628 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
14629
14630 handler
14631 .handle_control_frame(
14632 &ctx,
14633 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
14634 )
14635 .await
14636 .unwrap();
14637 let starts = outage_lines(&capture, OUTAGE_START);
14638 assert_eq!(starts.len(), 1, "{starts:?}");
14639 assert_eq!(starts[0].level, tracing::Level::WARN);
14640 }
14641
14642 #[tokio::test(flavor = "current_thread")]
14643 async fn route_open_unknown_module_escapes_target_module_id() {
14644 let handler = ControlHandler::new(Arc::new(Registry::default()));
14645 let capture = EventCapture::default();
14646 let _subscriber =
14647 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14648 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
14649 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14650 let response = handler
14651 .handle_control_frame(
14652 &ctx,
14653 route_open_frame(
14654 395,
14655 hostile_module_id,
14656 unique_project_root("hostile-target-module-id"),
14657 ),
14658 )
14659 .await
14660 .unwrap();
14661
14662 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14663 let event = capture
14664 .events()
14665 .into_iter()
14666 .find(|event| {
14667 event.target == "control"
14668 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14669 })
14670 .expect("route.open unknown-module refusal event");
14671 let logged = event.fields.get("module_id").expect("module_id field");
14672 assert!(!logged.bytes().any(|byte| byte < 0x20));
14673 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14674 }
14675
14676 #[tokio::test(flavor = "current_thread")]
14679 async fn route_open_refusal_carries_daemon_origin() {
14680 let handler = ControlHandler::new(Arc::new(Registry::default()));
14681 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14682 let response = handler
14683 .handle_control_frame(
14684 &ctx,
14685 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
14686 )
14687 .await
14688 .unwrap();
14689
14690 assert_eq!(response.len(), 1);
14691 assert_eq!(response[0].header.ty, FrameType::Error);
14692 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14693 assert!(
14694 response[0].header.flags.is_daemon_origin(),
14695 "route.open refusal must carry daemon origin: {:?}",
14696 response[0].header.flags
14697 );
14698 }
14699
14700 #[tokio::test(flavor = "current_thread")]
14704 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
14705 let registry = Arc::new(Registry::default());
14706 let forwarding = Arc::new(ForwardingTable::default());
14707 let handler =
14708 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14709 let module_connection = ConnectionId::new(95);
14710 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14711 hello_via_sink(
14712 &handler,
14713 &module_ctx,
14714 &mut module_rx,
14715 hello_frame("aft", PROTOCOL_VERSION, 395),
14716 )
14717 .await;
14718 forwarding
14719 .begin_module_drain("aft", RouteCloseReason::Restart)
14720 .unwrap()
14721 .expect("the module endpoint drains");
14722
14723 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
14724 let response = handler
14725 .handle_control_frame(
14726 &client_ctx,
14727 route_open_frame(396, "aft", unique_project_root("drain-origin")),
14728 )
14729 .await
14730 .unwrap();
14731
14732 assert_eq!(response.len(), 1);
14733 assert_eq!(response[0].header.ty, FrameType::Error);
14734 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14735 assert!(
14736 response[0].header.flags.is_daemon_origin(),
14737 "drain module_reloading must carry daemon origin: {:?}",
14738 response[0].header.flags
14739 );
14740 }
14741
14742 #[tokio::test(flavor = "current_thread")]
14746 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
14747 let registry = Arc::new(Registry::default());
14748 let forwarding = Arc::new(ForwardingTable::default());
14749 let handler =
14750 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14751 let module_connection = ConnectionId::new(95);
14752 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14753 hello_via_sink(
14754 &handler,
14755 &module_ctx,
14756 &mut module_rx,
14757 hello_frame("aft", PROTOCOL_VERSION, 395),
14758 )
14759 .await;
14760
14761 let client_connection = ConnectionId::new(96);
14762 let (client_ctx, _client_rx) = route_ctx(client_connection);
14763 let (route_task, bind) = relay_route_open(
14764 &handler,
14765 client_connection,
14766 &client_ctx.egress,
14767 &mut module_rx,
14768 396,
14769 "aft",
14770 "spoofed-reloading",
14771 )
14772 .await;
14773 let rejection = Frame::build(
14776 FrameType::Error,
14777 crate::router::daemon_error_flags(),
14778 0,
14779 0,
14780 bind.header.corr,
14781 serde_json::to_vec(&ErrorBody::new(
14782 "module_reloading",
14783 "module refused route.bind",
14784 ))
14785 .unwrap(),
14786 )
14787 .unwrap();
14788 handler
14789 .handle_control_frame(&module_ctx, rejection)
14790 .await
14791 .unwrap();
14792
14793 let response = route_task.await.unwrap();
14794 assert_eq!(response.len(), 1);
14795 assert_eq!(response[0].header.ty, FrameType::Error);
14796 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14797 assert!(
14798 !response[0].header.flags.is_daemon_origin(),
14799 "a module's own rejection must not carry daemon origin: {:?}",
14800 response[0].header.flags
14801 );
14802 }
14803
14804 #[tokio::test(flavor = "current_thread")]
14805 async fn route_open_module_rejection_uses_daemon_counter_key() {
14806 let registry = Arc::new(Registry::default());
14807 let forwarding = Arc::new(ForwardingTable::default());
14808 let handler =
14809 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14810 let module_connection = ConnectionId::new(95);
14811 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14812 hello_via_sink(
14813 &handler,
14814 &module_ctx,
14815 &mut module_rx,
14816 hello_frame("aft", PROTOCOL_VERSION, 395),
14817 )
14818 .await;
14819
14820 let client_connection = ConnectionId::new(96);
14821 let (client_ctx, _client_rx) = route_ctx(client_connection);
14822 let capture = EventCapture::default();
14823 let _subscriber =
14824 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14825 let (route_task, bind) = relay_route_open(
14826 &handler,
14827 client_connection,
14828 &client_ctx.egress,
14829 &mut module_rx,
14830 396,
14831 "aft",
14832 "hostile-module-code",
14833 )
14834 .await;
14835 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
14836 let rejection = Frame::build(
14837 FrameType::Error,
14838 control_flags(),
14839 0,
14840 0,
14841 bind.header.corr,
14842 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
14843 )
14844 .unwrap();
14845 handler
14846 .handle_control_frame(&module_ctx, rejection)
14847 .await
14848 .unwrap();
14849
14850 let response = route_task.await.unwrap();
14851 assert_eq!(parse_error(&response[0])["code"], hostile_code);
14852 let counters = handler.counters().snapshot();
14853 assert_eq!(
14854 counters["route_open_refused_by_code"],
14855 json!({ "module_rejected": 1 })
14856 );
14857 assert!(counters["route_open_refused_by_code"]
14858 .get(hostile_code)
14859 .is_none());
14860
14861 let event = capture
14862 .events()
14863 .into_iter()
14864 .find(|event| {
14865 event.target == "control"
14866 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
14867 })
14868 .expect("route.open module-rejection refusal event");
14869 let logged = event.fields.get("module_code").expect("module_code field");
14870 assert!(!logged.bytes().any(|byte| byte < 0x20));
14871 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14872 }
14873
14874 #[tokio::test]
14875 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
14876 let registry = Arc::new(Registry::default());
14877 let supervisor_handle = SupervisorHandle::new();
14878 let missing_program = std::env::temp_dir().join(format!(
14879 "subc-route-open-missing-program-{}",
14880 std::process::id()
14881 ));
14882 let supervisor =
14883 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
14884 .with_handle(supervisor_handle.clone());
14885 let module = supervisor
14886 .supervise_configured(
14887 ModuleSpec {
14888 carry_containment: false,
14889 module_id: "failed".to_string(),
14890 program: missing_program,
14891 args: Vec::new(),
14892 env: Vec::new(),
14893 reserved: false,
14894 reserved_prefixes: Vec::new(),
14895 protocol: ModuleProtocol::Subc,
14896 overlap: Default::default(),
14897 },
14898 true,
14899 )
14900 .unwrap();
14901 assert_eq!(module.state().unwrap(), ModuleState::Failed);
14902
14903 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14904 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
14905 let response = handler
14906 .handle_control_frame(
14907 &ctx,
14908 route_open_frame(305, "failed", unique_project_root("failed")),
14909 )
14910 .await
14911 .unwrap();
14912
14913 assert_eq!(response[0].header.ty, FrameType::Error);
14914 let error = parse_error(&response[0]);
14915 assert_eq!(error["code"], "target_unavailable");
14916 assert!(error["message"]
14917 .as_str()
14918 .unwrap()
14919 .contains("state=failed, enabled=true, live=false"));
14920 }
14921
14922 #[tokio::test]
14923 async fn route_open_role_mismatch_remains_target_unavailable() {
14924 let registry = Arc::new(Registry::default());
14925 let handler = ControlHandler::new(Arc::clone(®istry));
14926 handler
14927 .handle_control(
14928 ConnectionId::new(41),
14929 non_routable_hello_frame_with_control_ops("health-only", 306, None),
14930 )
14931 .unwrap();
14932
14933 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
14934 let response = handler
14935 .handle_control_frame(
14936 &ctx,
14937 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
14938 )
14939 .await
14940 .unwrap();
14941
14942 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14943 assert!(parse_error(&response[0])["message"]
14944 .as_str()
14945 .unwrap()
14946 .contains("does not provide the requested target"));
14947 }
14948
14949 #[tokio::test]
14950 async fn route_open_inactive_registration_remains_target_unavailable() {
14951 let registry = Arc::new(Registry::default());
14952 let handler = ControlHandler::new(Arc::clone(®istry));
14953 handler
14954 .handle_control(
14955 ConnectionId::new(43),
14956 hello_frame("inactive", PROTOCOL_VERSION, 308),
14957 )
14958 .unwrap();
14959 assert!(registry
14960 .set_module_state_for_test("inactive", ChannelState::Closed)
14961 .unwrap());
14962
14963 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
14964 let response = handler
14965 .handle_control_frame(
14966 &ctx,
14967 route_open_frame(309, "inactive", unique_project_root("inactive")),
14968 )
14969 .await
14970 .unwrap();
14971
14972 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14973 assert!(parse_error(&response[0])["message"]
14974 .as_str()
14975 .unwrap()
14976 .contains("is not active"));
14977 }
14978
14979 #[tokio::test]
14980 async fn late_health_reply_is_recorded_through_the_module_response_path() {
14981 let registry = Arc::new(Registry::default());
14982 let forwarding = Arc::new(ForwardingTable::default());
14983 let supervisor_handle = SupervisorHandle::new();
14984 let supervisor =
14985 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
14986 .with_forwarding(Arc::clone(&forwarding))
14987 .with_handle(supervisor_handle.clone());
14988 let module = supervisor
14989 .supervise_configured(
14990 crate::ModuleSpec {
14991 carry_containment: false,
14992 module_id: "late-health-response".to_string(),
14993 program: PathBuf::from("disabled-module"),
14994 args: Vec::new(),
14995 env: Vec::new(),
14996 reserved: false,
14997 reserved_prefixes: Vec::new(),
14998 protocol: ModuleProtocol::Subc,
14999 overlap: Default::default(),
15000 },
15001 false,
15002 )
15003 .unwrap();
15004 let handler =
15005 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15006 .with_supervisor(supervisor_handle);
15007 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
15008 handler
15009 .handle_control_frame(
15010 &module_ctx,
15011 hello_frame_with_control_ops(
15012 "late-health-response",
15013 PROTOCOL_VERSION,
15014 7,
15015 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15016 ),
15017 )
15018 .await
15019 .unwrap();
15020 let probe_started_at = Instant::now() - Duration::from_millis(80);
15021 let pending = forwarding
15022 .begin_health_probe_rpc_for(
15023 "late-health-response",
15024 MODULE_CONTROL_OP_HEALTH_CHECK,
15025 probe_started_at,
15026 Instant::now() - Duration::from_millis(1),
15027 )
15028 .unwrap();
15029 assert!(forwarding
15030 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
15031 .unwrap());
15032
15033 let responses = handler
15034 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
15035 .await
15036 .unwrap();
15037
15038 assert!(responses.is_empty());
15039 let health = module.status().unwrap().health;
15040 assert_eq!(health.late_answer_count, 1);
15041 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
15042 }
15043
15044 #[tokio::test]
15045 async fn health_probe_timeout_and_module_death_are_typed() {
15046 let registry = Arc::new(Registry::default());
15047 let forwarding = Arc::new(ForwardingTable::default());
15048 let handler =
15049 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15050 .with_health_probe_timeout(Duration::from_millis(50));
15051 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
15052 hello_via_sink(
15053 &handler,
15054 &module_ctx,
15055 &mut module_rx,
15056 hello_frame_with_control_ops(
15057 "aft",
15058 PROTOCOL_VERSION,
15059 7,
15060 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15061 ),
15062 )
15063 .await;
15064
15065 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
15066 let responses = handler
15067 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
15068 .await
15069 .unwrap();
15070 assert_eq!(responses[0].header.ty, FrameType::Error);
15071 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
15072 let _ = module_rx.try_recv();
15073
15074 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
15075 let health_handler = handler.clone();
15076 let death_task = tokio::spawn(async move {
15077 health_handler
15078 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
15079 .await
15080 .unwrap()
15081 });
15082 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15083 .await
15084 .unwrap()
15085 .unwrap();
15086 handler
15087 .cleanup_connection(module_ctx.connection_id)
15088 .unwrap();
15089 let responses = death_task.await.unwrap();
15090 assert_eq!(responses[0].header.ty, FrameType::Error);
15091 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
15092 }
15093
15094 #[test]
15095 fn hello_requires_exact_protocol_version() {
15096 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
15097 let registry = Arc::new(Registry::default());
15098 let handler = ControlHandler::new(Arc::clone(®istry));
15099 let responses = handler
15100 .handle_control(
15101 ConnectionId::new(connection),
15102 hello_frame("aft", offered, 9),
15103 )
15104 .unwrap();
15105
15106 assert_eq!(responses.len(), 1);
15107 assert_eq!(responses[0].header.ty, FrameType::Error);
15108 let error = parse_error(&responses[0]);
15109 assert_eq!(error["code"], "version_unsupported");
15110 assert!(registry.get_module("aft").unwrap().is_none());
15111 assert_eq!(registry.active_registration_count().unwrap(), 0);
15112 }
15113 }
15114
15115 #[test]
15116 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
15117 let registry = Arc::new(Registry::default());
15118 let forwarding = Arc::new(ForwardingTable::default());
15119 let handler =
15120 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15121 let module_connection = ConnectionId::new(301);
15122 let registration = registry
15123 .register_with_control_ops(
15124 manifest("aft-push", PROTOCOL_VERSION),
15125 PROTOCOL_VERSION,
15126 module_connection,
15127 module_baseline_control_ops(),
15128 )
15129 .unwrap();
15130 let (module_tx, _module_rx) = mpsc::channel(8);
15131 let endpoint = forwarding
15132 .register_module_connection(
15133 module_connection,
15134 "aft-push".to_string(),
15135 PROTOCOL_VERSION,
15136 manifest_concurrency(®istration.manifest),
15137 FrameSink::new(module_tx),
15138 )
15139 .unwrap();
15140
15141 let unknown = Frame::build(
15143 FrameType::Push,
15144 control_flags(),
15145 0,
15146 0,
15147 5,
15148 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
15149 )
15150 .unwrap();
15151 let out = handler.handle_status_update(endpoint, unknown).unwrap();
15152 assert!(
15153 out.is_empty(),
15154 "unknown push op must be ignored, got {out:?}"
15155 );
15156
15157 let malformed = Frame::build(
15159 FrameType::Push,
15160 control_flags(),
15161 0,
15162 0,
15163 6,
15164 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
15165 )
15166 .unwrap();
15167 let out = handler.handle_status_update(endpoint, malformed).unwrap();
15168 assert_eq!(out.len(), 1);
15169 assert_eq!(out[0].header.ty, FrameType::Error);
15170 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
15171 }
15172
15173 #[test]
15174 fn hello_rejected_when_connection_already_owns_client_routes() {
15175 let registry = Arc::new(Registry::default());
15176 let forwarding = Arc::new(ForwardingTable::default());
15177 let handler =
15178 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15179 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
15181 let client_connection = ConnectionId::new(202);
15182
15183 let responses = handler
15186 .handle_control(
15187 client_connection,
15188 hello_frame("aft-second", PROTOCOL_VERSION, 9),
15189 )
15190 .unwrap();
15191 assert_eq!(responses[0].header.ty, FrameType::Error);
15192 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
15193 assert!(registry.get_module("aft-second").unwrap().is_none());
15194 }
15195
15196 #[tokio::test]
15197 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
15198 let registry = Arc::new(Registry::default());
15199 let forwarding = Arc::new(ForwardingTable::default());
15200 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
15201 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
15202 hello_via_sink(
15203 &handler,
15204 &module_ctx,
15205 &mut module_rx,
15206 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
15207 )
15208 .await;
15209 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
15210 let pending = forwarding
15211 .begin_route_bind_relay_for_test(
15212 client_ctx.connection_id,
15213 client_ctx.egress.clone(),
15214 2,
15215 "alpha",
15216 )
15217 .unwrap();
15218 forwarding
15219 .complete_pending_relay(
15220 module_ctx.connection_id,
15221 pending.corr,
15222 RouteBindRelayOutcome::Accepted,
15223 )
15224 .unwrap();
15225 client_rx.try_recv().unwrap();
15226 for module_id in ["beta", "alpha"] {
15227 let replies = handler
15228 .handle_control_frame(
15229 &module_ctx,
15230 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
15231 )
15232 .await
15233 .unwrap();
15234 assert_eq!(replies.len(), 1, "second HELLO must be refused");
15235 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
15236 }
15237 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
15238 assert!(registry.get_module("beta").unwrap().is_none());
15239 assert!(matches!(
15240 forwarding
15241 .lookup_data_route(
15242 client_ctx.connection_id,
15243 pending.client_channel,
15244 pending.client_epoch,
15245 )
15246 .unwrap(),
15247 DataRoute::Client(DataRouteState::Bound(_))
15248 ));
15249 assert!(handler
15250 .hello_launch_nonces
15251 .lock()
15252 .unwrap()
15253 .presented(module_ctx.connection_id, Some("alpha-nonce")));
15254 assert!(module_rx.try_recv().is_err());
15255 }
15256
15257 #[test]
15258 fn reserved_module_hello_requires_matching_launch_nonce() {
15259 let registry = Arc::new(Registry::default());
15260 let supervisor = SupervisorHandle::new();
15261 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
15264 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15265
15266 let no_nonce = handler
15268 .handle_control(
15269 ConnectionId::new(1),
15270 hello_frame("vault", PROTOCOL_VERSION, 1),
15271 )
15272 .unwrap();
15273 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
15274 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
15275 assert!(registry.get_module("vault").unwrap().is_none());
15276
15277 let wrong = handler
15279 .handle_control(
15280 ConnectionId::new(2),
15281 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
15282 )
15283 .unwrap();
15284 assert_eq!(wrong[0].header.ty, FrameType::Error);
15285 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
15286 assert!(registry.get_module("vault").unwrap().is_none());
15287
15288 let ok = handler
15290 .handle_control(
15291 ConnectionId::new(3),
15292 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
15293 )
15294 .unwrap();
15295 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
15296 assert!(registry.get_module("vault").unwrap().is_some());
15297 }
15298
15299 #[test]
15300 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
15301 let registry = Arc::new(Registry::default());
15302 let supervisor = SupervisorHandle::new();
15303 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15304 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15305 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15306
15307 let squat = handler
15308 .handle_control(
15309 ConnectionId::new(1),
15310 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
15311 )
15312 .unwrap();
15313 assert_eq!(squat[0].header.ty, FrameType::Error);
15314 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
15315 assert!(parse_error(&squat[0])["message"]
15316 .as_str()
15317 .unwrap()
15318 .contains("fed:"));
15319
15320 let accepted_peer = handler
15321 .handle_control(
15322 ConnectionId::new(2),
15323 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
15324 )
15325 .unwrap();
15326 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
15327
15328 let accepted_short = handler
15329 .handle_control(
15330 ConnectionId::new(3),
15331 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
15332 )
15333 .unwrap();
15334 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
15335
15336 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
15337 let response = handler
15338 .handle_control(
15339 ConnectionId::new(conn),
15340 hello_frame(module_id, PROTOCOL_VERSION, conn),
15341 )
15342 .unwrap();
15343 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
15344 }
15345 }
15346
15347 #[test]
15348 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
15349 let registry = Arc::new(Registry::default());
15350 let supervisor = SupervisorHandle::new();
15351 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15352 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15353 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
15354 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15355
15356 let owner_nonce = handler
15357 .handle_control(
15358 ConnectionId::new(1),
15359 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
15360 )
15361 .unwrap();
15362 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
15363 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
15364 assert!(registry.get_module("fed:special").unwrap().is_none());
15365
15366 let exact_nonce = handler
15367 .handle_control(
15368 ConnectionId::new(2),
15369 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
15370 )
15371 .unwrap();
15372 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
15373 assert!(registry.get_module("fed:special").unwrap().is_some());
15374 }
15375
15376 #[test]
15377 fn non_reserved_module_ignores_launch_nonce() {
15378 let registry = Arc::new(Registry::default());
15379 let handler = ControlHandler::new(Arc::clone(®istry));
15382 let no_nonce = handler
15383 .handle_control(
15384 ConnectionId::new(1),
15385 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
15386 )
15387 .unwrap();
15388 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
15389 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
15390
15391 let echoed_nonce = handler
15392 .handle_control(
15393 ConnectionId::new(2),
15394 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
15395 )
15396 .unwrap();
15397 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
15398 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
15399 }
15400
15401 #[test]
15402 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
15403 let handler = ControlHandler::default();
15404 let conn = ConnectionId::new(1);
15405 let malformed = Frame::build(
15406 FrameType::Hello,
15407 control_flags(),
15408 0,
15409 0,
15410 3,
15411 b"{not json".to_vec(),
15412 )
15413 .unwrap();
15414
15415 let error = handler.handle_control(conn, malformed).unwrap();
15416 assert_eq!(error[0].header.ty, FrameType::Error);
15417 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
15418
15419 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15420 let pong = handler.handle_control(conn, ping).unwrap();
15421 assert_eq!(pong[0].header.ty, FrameType::Pong);
15422 assert_eq!(pong[0].header.corr, 4);
15423 }
15424
15425 #[test]
15426 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
15427 let registry = Arc::new(Registry::default());
15428 let handler = ControlHandler::new(Arc::clone(®istry));
15429
15430 handler
15431 .handle_control(
15432 ConnectionId::new(1),
15433 hello_frame("aft", PROTOCOL_VERSION, 1),
15434 )
15435 .unwrap();
15436 let duplicate = handler
15437 .handle_control(
15438 ConnectionId::new(2),
15439 hello_frame("aft", PROTOCOL_VERSION, 2),
15440 )
15441 .unwrap();
15442
15443 assert_eq!(duplicate[0].header.ty, FrameType::Error);
15444 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15445 let registration = registry.get_module("aft").unwrap().unwrap();
15446 assert_eq!(registration.connection_id, ConnectionId::new(1));
15447 }
15448
15449 #[test]
15450 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
15451 let registry = Arc::new(Registry::default());
15452 let forwarding = Arc::new(ForwardingTable::default());
15453 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
15454 let handler =
15455 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15456 .with_process_liveness(process_liveness);
15457 let (ctx, route_channel, route_epoch) =
15458 bind_liveness_route(®istry, &forwarding, "aft-dead");
15459 let responses = handler
15460 .handle_route_poll(
15461 &ctx,
15462 route_poll_frame(41, PollKind::Liveness, route_channel),
15463 route_channel,
15464 route_epoch,
15465 PollKind::Liveness,
15466 )
15467 .unwrap();
15468
15469 assert_eq!(responses.len(), 1);
15470 assert_eq!(responses[0].header.ty, FrameType::Response);
15471 assert_route_poll_liveness(&responses[0], false);
15472 }
15473
15474 #[test]
15475 fn liveness_poll_without_process_source_uses_bound_route() {
15476 let registry = Arc::new(Registry::default());
15477 let forwarding = Arc::new(ForwardingTable::default());
15478 let handler =
15479 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15480 let (ctx, route_channel, route_epoch) =
15481 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
15482 let responses = handler
15483 .handle_route_poll(
15484 &ctx,
15485 route_poll_frame(42, PollKind::Liveness, route_channel),
15486 route_channel,
15487 route_epoch,
15488 PollKind::Liveness,
15489 )
15490 .unwrap();
15491
15492 assert_route_poll_liveness(&responses[0], true);
15493 }
15494
15495 #[test]
15496 fn liveness_poll_untracked_process_source_uses_bound_route() {
15497 let registry = Arc::new(Registry::default());
15498 let forwarding = Arc::new(ForwardingTable::default());
15499 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
15500 let handler =
15501 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15502 .with_process_liveness(process_liveness);
15503 let (ctx, route_channel, route_epoch) =
15504 bind_liveness_route(®istry, &forwarding, "aft-untracked");
15505 let responses = handler
15506 .handle_route_poll(
15507 &ctx,
15508 route_poll_frame(43, PollKind::Liveness, route_channel),
15509 route_channel,
15510 route_epoch,
15511 PollKind::Liveness,
15512 )
15513 .unwrap();
15514
15515 assert_route_poll_liveness(&responses[0], true);
15516 }
15517
15518 #[tokio::test]
15519 async fn unknown_op_returns_unknown_control_op() {
15520 let handler = ControlHandler::default();
15521 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
15522 let request = Frame::build(
15523 FrameType::Request,
15524 control_flags(),
15525 0,
15526 0,
15527 55,
15528 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
15529 )
15530 .unwrap();
15531
15532 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15533
15534 assert_eq!(response.len(), 1);
15535 assert_eq!(response[0].header.ty, FrameType::Error);
15536 assert_eq!(response[0].header.corr, 55);
15537 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
15538 }
15539
15540 #[tokio::test]
15541 async fn supervisor_provenance_rejects_unknown_exact_module() {
15542 let handler = ControlHandler::default();
15543 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
15544 let request = Frame::build(
15545 FrameType::Request,
15546 control_flags(),
15547 0,
15548 0,
15549 57,
15550 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
15551 )
15552 .unwrap();
15553
15554 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15555
15556 assert_eq!(response.len(), 1);
15557 assert_eq!(response[0].header.ty, FrameType::Error);
15558 assert_eq!(response[0].header.corr, 57);
15559 let error = parse_error(&response[0]);
15560 assert_eq!(error["code"], "unknown_module");
15561 assert_eq!(error["message"], "module_id 'missing' is not supervised");
15562 }
15563
15564 #[test]
15565 fn provenance_probe_override_keeps_handler_tests_deterministic() {
15566 let expected = subc_control::RunningImageAgreement::Unavailable {
15567 reason: subc_control::RunningImageUnavailableReason::HashFailed,
15568 };
15569 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
15570 assert_eq!(handler.provenance_probe_override, Some(expected));
15571 }
15572
15573 #[test]
15574 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
15575 let verdict = reload_verdict(
15576 std::path::Path::new("/bin/new"),
15577 Some(std::path::Path::new("/bin/old")),
15578 subc_control::RunningImageAgreement::Unavailable {
15579 reason: subc_control::RunningImageUnavailableReason::HashFailed,
15580 },
15581 );
15582 assert!(matches!(
15583 verdict.path,
15584 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
15585 if configured == std::path::Path::new("/bin/new")
15586 && spawned_from == std::path::Path::new("/bin/old")
15587 ));
15588 }
15589
15590 #[test]
15591 fn launch_spec_without_running_digest_is_unavailable_not_match() {
15592 use subc_control::{
15593 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
15594 };
15595 let configured = crate::supervise::ModuleSpec {
15596 carry_containment: false,
15597 module_id: "legacy".into(),
15598 program: "/module".into(),
15599 args: Vec::new(),
15600 env: Vec::new(),
15601 reserved: false,
15602 reserved_prefixes: Vec::new(),
15603 protocol: subc_control::ModuleProtocol::Subc,
15604 overlap: Default::default(),
15605 };
15606 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
15607 assert_eq!(
15608 launch_spec_agreement(&configured, false, Some(&digest)),
15609 Agreement::Unavailable {
15610 reason: Reason::NotRunning
15611 }
15612 );
15613 assert_eq!(
15614 launch_spec_agreement(&configured, true, None),
15615 Agreement::Unavailable {
15616 reason: Reason::SpawnedSpecUnavailable
15617 }
15618 );
15619 assert_eq!(
15620 launch_spec_agreement(&configured, true, Some(&digest)),
15621 Agreement::Match
15622 );
15623 }
15624
15625 #[cfg(windows)]
15626 #[test]
15627 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
15628 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
15629 let configured = dir.join("module.exe");
15630 std::fs::write(&configured, b"executable path fixture").unwrap();
15631 let spawned = std::fs::canonicalize(&configured).unwrap();
15632 assert_ne!(
15633 configured, spawned,
15634 "fixture must exercise different path spellings"
15635 );
15636 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
15637 let lower_case = configured.to_string_lossy().to_lowercase();
15638 for spelling in [
15639 &configured,
15640 Path::new(&forward_slashes),
15641 Path::new(&lower_case),
15642 ] {
15643 let verdict = reload_verdict(
15644 spelling,
15645 Some(&spawned),
15646 subc_control::RunningImageAgreement::Unavailable {
15647 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15648 },
15649 );
15650 assert_eq!(
15651 verdict.path,
15652 ReloadPathAgreement::Match,
15653 "configured {spelling:?}, spawned {spawned:?}"
15654 );
15655 }
15656 }
15657
15658 #[cfg(windows)]
15659 #[test]
15660 fn windows_reload_path_keeps_different_existing_program_pending() {
15661 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
15662 let old = dir.join("old.exe");
15663 let new = dir.join("new.exe");
15664 std::fs::write(&old, b"same executable content").unwrap();
15665 std::fs::copy(&old, &new).unwrap();
15666 let spawned = std::fs::canonicalize(&old).unwrap();
15667 let image = subc_control::RunningImageAgreement::Unavailable {
15668 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15669 };
15670 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
15671 assert!(
15672 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
15673 if configured == new && spawned_from == spawned)
15674 );
15675 let missing = dir.join("missing.exe");
15676 assert!(matches!(
15677 reload_verdict(&missing, Some(&spawned), image).path,
15678 ReloadPathAgreement::Mismatch { .. }
15679 ));
15680 }
15681
15682 #[test]
15683 fn reload_verdict_detects_replaced_image_at_same_path() {
15684 let image = subc_control::RunningImageAgreement::Mismatch {
15685 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
15686 digest: "old".into(),
15687 },
15688 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
15689 digest: "new".into(),
15690 },
15691 };
15692 let verdict = reload_verdict(
15693 std::path::Path::new("/bin/same"),
15694 Some(std::path::Path::new("/bin/same")),
15695 image.clone(),
15696 );
15697 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15698 assert_eq!(verdict.image, image);
15699 }
15700
15701 #[test]
15702 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
15703 let image = subc_control::RunningImageAgreement::Unavailable {
15704 reason: subc_control::RunningImageUnavailableReason::NotRunning,
15705 };
15706 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
15707 assert_eq!(
15708 verdict.path,
15709 subc_control::ReloadPathAgreement::Unavailable {
15710 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
15711 }
15712 );
15713 assert_eq!(verdict.image, image);
15714
15715 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
15716 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15717 };
15718 let verdict = reload_verdict(
15719 std::path::Path::new("/bin/same"),
15720 Some(std::path::Path::new("/bin/same")),
15721 unconfirmed.clone(),
15722 );
15723 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15724 assert_eq!(verdict.image, unconfirmed);
15725 }
15726
15727 #[test]
15728 fn reload_verdict_preserves_each_image_unavailability_reason() {
15729 use subc_control::RunningImageUnavailableReason as Reason;
15730
15731 for reason in [
15732 Reason::NotRunning,
15733 Reason::UnsupportedPlatform,
15734 Reason::RunningExecutableUnreadable,
15735 Reason::SpawnedPathUnreadable,
15736 Reason::HashFailed,
15737 Reason::ProcessIdentityUnconfirmed,
15738 Reason::Unknown("future_probe_reason".to_string()),
15739 ] {
15740 let image = subc_control::RunningImageAgreement::Unavailable {
15741 reason: reason.clone(),
15742 };
15743 let verdict = reload_verdict(
15744 std::path::Path::new("/bin/same"),
15745 Some(std::path::Path::new("/bin/same")),
15746 image.clone(),
15747 );
15748 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15749 assert_eq!(verdict.image, image, "{reason:?}");
15750 }
15751 }
15752
15753 #[tokio::test]
15754 async fn malformed_control_bodies_return_invalid_control_body() {
15755 let handler = ControlHandler::default();
15756 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
15757
15758 for (corr, body) in [
15759 (56, br#"{"route_channel":1}"#.as_slice()),
15760 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
15761 (
15762 58,
15763 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
15764 ),
15765 ] {
15766 let request = Frame::build(
15767 FrameType::Request,
15768 control_flags(),
15769 0,
15770 0,
15771 corr,
15772 body.to_vec(),
15773 )
15774 .unwrap();
15775 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15776
15777 assert_eq!(response.len(), 1);
15778 assert_eq!(response[0].header.ty, FrameType::Error);
15779 assert_eq!(response[0].header.corr, corr);
15780 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
15781 }
15782 }
15783
15784 #[tokio::test]
15785 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
15786 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15787 let registry = Arc::new(Registry::default());
15788 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
15789 let router = Router::with_control_handler(Arc::clone(&control));
15790 let connection = router.begin_connection();
15791 let (ctx, mut rx) = route_ctx(connection.id());
15792
15793 router
15794 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
15795 .await
15796 .unwrap();
15797 let response = rx.recv().await.unwrap();
15798 let ack = parse_ack(&response);
15799 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15800 let channel = 1;
15801
15802 let goodbye =
15803 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
15804 router.route_for_connection(&ctx, goodbye).await.unwrap();
15805 assert!(rx.try_recv().is_err());
15806 assert!(registry.get_module("aft").unwrap().is_none());
15807
15808 router
15809 .route_for_connection(&ctx, channel_request(channel, 13))
15810 .await
15811 .unwrap();
15812 let error_frame = rx.recv().await.unwrap();
15813 assert_eq!(error_frame.header.ty, FrameType::Error);
15814 assert_eq!(error_frame.header.channel, channel);
15815 let captured = crate::router::test_log::captured_logs(&logs);
15816 assert_eq!(
15817 captured
15818 .lines()
15819 .filter(|line| {
15820 line.contains("module_id=aft")
15821 && line.contains("reason=explicit_goodbye")
15822 && line.contains("module registration ended")
15823 })
15824 .count(),
15825 1,
15826 "unexpected GOODBYE registry log: {captured}"
15827 );
15828 }
15829
15830 #[tokio::test]
15831 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
15832 let registry = Arc::new(Registry::default());
15833 let handler = ControlHandler::new(registry).with_capability_config(
15834 [("prov".to_string(), true), ("cons".to_string(), true)],
15835 BTreeMap::new(),
15836 );
15837 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
15838 register_capability_manifest(
15839 &handler,
15840 &provider_ctx,
15841 &mut provider_rx,
15842 capability_manifest("prov", &["thing/v1"], &[]),
15843 1,
15844 )
15845 .await;
15846 let mut consumer = capability_manifest("cons", &[], &[]);
15847 consumer.capabilities.as_mut().unwrap().requires.push(
15848 subc_protocol::manifest::CapabilityRequirement {
15849 capability: "thing/v1".to_string(),
15850 need: subc_protocol::manifest::CapabilityNeed::Required,
15851 },
15852 );
15853 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
15854 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
15855 assert_eq!(
15856 handler.capability_evaluator.verdict("cons", "thing/v1"),
15857 Some(CapabilityVerdict::Provided)
15858 );
15859 let (mut client_rx, _) = open_route_for_capability_test(
15860 &handler,
15861 &provider_ctx,
15862 &mut provider_rx,
15863 703,
15864 3,
15865 "prov",
15866 None,
15867 )
15868 .await;
15869 let goodbye =
15870 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15871 handler
15872 .handle_control_frame(&provider_ctx, goodbye)
15873 .await
15874 .unwrap();
15875 assert_eq!(
15876 handler.capability_evaluator.verdict("cons", "thing/v1"),
15877 Some(CapabilityVerdict::NeverProvided)
15878 );
15879 let closed = client_rx
15880 .try_recv()
15881 .expect("GOODBYE pushes route.closed before route GOODBYE");
15882 assert!(
15883 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
15884 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
15885 );
15886 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
15887 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
15888 }
15889
15890 struct RouteClosedFixture {
15891 handler: ControlHandler,
15892 #[cfg(unix)]
15893 supervisor: Supervisor,
15894 module_ctx: RouteCtx,
15895 #[cfg(unix)]
15896 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15897 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15898 route: Arc<crate::forwarding::RouteBinding>,
15899 }
15900
15901 async fn route_closed_fixture() -> RouteClosedFixture {
15902 let registry = Arc::new(Registry::default());
15903 let handler = ControlHandler::new(Arc::clone(®istry));
15904 #[cfg(unix)]
15905 let supervisor = Supervisor::new(registry, RestartPolicy::default())
15906 .with_forwarding(Arc::clone(&handler.forwarding));
15907 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
15908 register_capability_manifest(
15909 &handler,
15910 &module_ctx,
15911 &mut module_rx,
15912 capability_manifest("prov", &[], &[]),
15913 1,
15914 )
15915 .await;
15916 let (client_rx, bind) = open_route_for_capability_test(
15917 &handler,
15918 &module_ctx,
15919 &mut module_rx,
15920 702,
15921 2,
15922 "prov",
15923 None,
15924 )
15925 .await;
15926 let ModuleControlRequest::RouteBind {
15927 route_channel: channel,
15928 epoch: route_epoch,
15929 ..
15930 } = bind
15931 else {
15932 panic!("expected route.bind");
15933 };
15934 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
15935 handler
15936 .forwarding
15937 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
15938 .unwrap()
15939 else {
15940 panic!("the acknowledged route must be bound");
15941 };
15942 RouteClosedFixture {
15943 handler,
15944 #[cfg(unix)]
15945 supervisor,
15946 module_ctx,
15947 #[cfg(unix)]
15948 module_rx,
15949 client_rx,
15950 route,
15951 }
15952 }
15953
15954 impl RouteClosedFixture {
15955 #[cfg(unix)]
15956 async fn cut(&mut self) {
15957 let notices = async {
15958 let mut draining = self.module_rx.recv().await.unwrap();
15959 assert!(matches!(
15960 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
15961 &draining.body
15962 )
15963 .unwrap(),
15964 subc_protocol::session::ModuleControlCommand::Draining {
15965 reason: RouteCloseReason::Disable,
15966 ..
15967 }
15968 ));
15969 draining.flushed.take().unwrap().send(()).unwrap();
15970 let mut closing = self.client_rx.recv().await.unwrap();
15971 assert!(matches!(
15972 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
15973 ClientControlPush::RouteClosing {
15974 reason: RouteCloseReason::Restart,
15975 ..
15976 }
15977 ));
15978 closing.flushed.take().unwrap().send(()).unwrap();
15979 };
15980 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
15981 result.unwrap();
15982 }
15983
15984 fn disconnect(&mut self) -> ClientControlPush {
15985 self.handler
15986 .cleanup_connection(self.module_ctx.connection_id)
15987 .unwrap();
15988 let closed = self.client_rx.try_recv().unwrap();
15989 assert_eq!(closed.header.ty, FrameType::Push);
15990 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
15991 assert!(matches!(
15992 &push,
15993 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
15994 if channels == &[self.route.client_channel]
15995 ));
15996 assert_eq!(
15997 self.client_rx.try_recv().unwrap().header.ty,
15998 FrameType::Goodbye
15999 );
16000 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
16001 push
16002 }
16003 }
16004
16005 #[cfg(unix)]
16006 #[tokio::test(start_paused = true)]
16007 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
16008 let mut fixture = route_closed_fixture().await;
16009 fixture.cut().await;
16010 assert!(matches!(
16011 fixture.disconnect(),
16012 ClientControlPush::RouteClosed {
16013 reason: RouteCloseReason::Restart,
16014 drained: true,
16015 ..
16016 }
16017 ));
16018 }
16019
16020 #[cfg(unix)]
16021 #[tokio::test(start_paused = true)]
16022 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
16023 let mut fixture = route_closed_fixture().await;
16024 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
16025 let started = tokio::time::Instant::now();
16026 fixture.cut().await;
16027 assert!(started.elapsed() >= Duration::from_secs(2));
16028 assert_eq!(fixture.route.flow.in_flight(), 1);
16029 assert!(matches!(
16030 fixture.disconnect(),
16031 ClientControlPush::RouteClosed {
16032 reason: RouteCloseReason::Restart,
16033 drained: false,
16034 ..
16035 }
16036 ));
16037 }
16038
16039 #[tokio::test]
16040 async fn module_connection_crash_reports_crash_and_not_drained() {
16041 let mut fixture = route_closed_fixture().await;
16042 assert!(matches!(
16046 fixture.disconnect(),
16047 ClientControlPush::RouteClosed {
16048 reason: RouteCloseReason::Crash,
16049 drained: false,
16050 ..
16051 }
16052 ));
16053 }
16054
16055 #[tokio::test]
16056 async fn dropping_router_connection_releases_registration() {
16057 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16058 let registry = Arc::new(Registry::default());
16059 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
16060 let router = Router::with_control_handler(Arc::clone(&control));
16061 let connection = router.begin_connection();
16062 let connection_id = connection.id();
16063 let (ctx, mut rx) = route_ctx(connection_id);
16064
16065 router
16066 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
16067 .await
16068 .unwrap();
16069 let response = rx.recv().await.unwrap();
16070 let ack = parse_ack(&response);
16071 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16072 assert!(registry.get_module("aft").unwrap().is_some());
16073
16074 drop(connection);
16075
16076 assert!(registry.get_module("aft").unwrap().is_none());
16077 assert_eq!(registry.active_registration_count().unwrap(), 0);
16078
16079 control
16080 .cleanup_connection(ConnectionId::new(u64::MAX))
16081 .unwrap();
16082 let captured = crate::router::test_log::captured_logs(&logs);
16083 println!("captured registry lifecycle logs:\n{captured}");
16084 let events: Vec<_> = captured
16085 .lines()
16086 .filter(|line| {
16087 line.contains("module registered module_id=aft ")
16088 || line.contains("module registration ended")
16089 })
16090 .collect();
16091 assert_eq!(
16092 events.len(),
16093 2,
16094 "unexpected registry lifecycle logs: {captured}"
16095 );
16096 assert!(events[0].contains("module registered module_id=aft "));
16097 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
16098 assert!(events[1].contains(&format!(
16099 "module_id=aft connection_id={}",
16100 connection_id.get()
16101 )));
16102 assert!(events[1].contains("reason=connection_closed"));
16103 assert!(events[1].contains("module registration ended"));
16104 assert_eq!(
16105 captured
16106 .lines()
16107 .filter(|line| line.contains("module registered module_id=aft "))
16108 .count(),
16109 1,
16110 "legacy registration admission line must appear once: {captured}"
16111 );
16112 }
16113
16114 #[tokio::test]
16115 async fn hello_registration_keeps_legacy_module_registered_line_once() {
16116 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16117 let registry = Arc::new(Registry::default());
16118 let control = ControlHandler::new(Arc::clone(®istry));
16119 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
16120 hello_via_sink(
16121 &control,
16122 &ctx,
16123 &mut rx,
16124 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
16125 )
16126 .await;
16127
16128 let captured = crate::router::test_log::captured_logs(&logs);
16129 let admissions: Vec<_> = captured
16130 .lines()
16131 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
16132 .collect();
16133 assert_eq!(
16134 admissions.len(),
16135 1,
16136 "legacy admission line must remain exactly once: {captured}"
16137 );
16138 assert!(admissions[0].contains("routable_provider=true"));
16139 assert!(admissions[0].contains("connection_id=777"));
16140 }
16141
16142 fn capability_manifest(
16143 module_id: &str,
16144 provides: &[&str],
16145 must_never_reach: &[&str],
16146 ) -> ModuleManifest {
16147 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
16148 manifest.capabilities = Some(CapabilityDeclarations {
16149 provides: provides
16150 .iter()
16151 .map(|capability| (*capability).to_string())
16152 .collect(),
16153 requires: Vec::new(),
16154 must_never_reach: must_never_reach
16155 .iter()
16156 .map(|capability| (*capability).to_string())
16157 .collect(),
16158 });
16159 manifest
16160 }
16161
16162 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
16163 Frame::build(
16164 FrameType::Hello,
16165 control_flags(),
16166 0,
16167 0,
16168 corr,
16169 serde_json::to_vec(&ModuleHelloBody {
16170 protocol_ver: manifest.protocol_ver,
16171 manifest,
16172 control_ops: None,
16173 launch_nonce: None,
16174 })
16175 .expect("capability test HELLO serializes"),
16176 )
16177 .expect("capability test HELLO frame builds")
16178 }
16179
16180 fn catalog_update_with_capabilities_frame(
16181 corr: u64,
16182 capabilities: CapabilityDeclarations,
16183 ) -> Frame {
16184 Frame::build(
16185 FrameType::Request,
16186 control_flags(),
16187 0,
16188 0,
16189 corr,
16190 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16191 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
16192 capabilities: Some(capabilities),
16193 ready: None,
16194 })
16195 .expect("capability catalog.update serializes"),
16196 )
16197 .expect("capability catalog.update frame builds")
16198 }
16199
16200 async fn register_capability_manifest(
16201 handler: &ControlHandler,
16202 ctx: &RouteCtx,
16203 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16204 manifest: ModuleManifest,
16205 corr: u64,
16206 ) {
16207 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
16208 }
16209
16210 async fn open_route_for_capability_test(
16211 handler: &ControlHandler,
16212 target_ctx: &RouteCtx,
16213 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16214 client_connection_id: u64,
16215 corr: u64,
16216 target_module_id: &str,
16217 consumer_identity: Option<ConsumerIdentity>,
16218 ) -> (
16219 mpsc::Receiver<crate::router::OutboundFrame>,
16220 ModuleControlRequest,
16221 ) {
16222 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
16223 let route_handler = handler.clone();
16224 let target_module_id = target_module_id.to_string();
16225 let route_task = tokio::spawn(async move {
16226 route_handler
16227 .handle_control_frame(
16228 &client_ctx,
16229 route_open_frame_with_admission_facts(
16230 corr,
16231 &target_module_id,
16232 unique_project_root("admission-facts"),
16233 consumer_identity,
16234 None,
16235 ),
16236 )
16237 .await
16238 .expect("capability test route.open succeeds")
16239 });
16240 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
16241 .await
16242 .expect("capability test route.open must reach route.bind")
16243 .expect("target control receiver stays open");
16244 let bind_request: ModuleControlRequest =
16245 serde_json::from_slice(&bind.body).expect("route.bind decodes");
16246 handler
16247 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
16248 .await
16249 .expect("capability test route.bind ACK succeeds");
16250 assert!(route_task.await.expect("route.open task joins").is_empty());
16251 let opened = client_rx
16252 .recv()
16253 .await
16254 .expect("successful route.open publishes a response");
16255 assert!(matches!(
16256 serde_json::from_slice::<ClientControlResponse>(&opened.body),
16257 Ok(ClientControlResponse::RouteOpen { .. })
16258 ));
16259 (client_rx, bind_request)
16260 }
16261
16262 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
16263 assert_eq!(frame.header.ty, FrameType::Push);
16264 assert_eq!(frame.header.channel, 0);
16265 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
16266 .expect("route.closed control push decodes");
16267 let ClientControlPush::RouteClosed { channels, .. } = &push else {
16268 panic!("expected route.closed");
16269 };
16270 assert_eq!(channels.len(), 1, "exactly one violating route closed");
16271 let closed_channel = channels[0];
16272 let channels = channels.clone();
16273 assert_eq!(
16274 push,
16275 ClientControlPush::RouteClosed {
16276 module_id: target_module_id.to_string(),
16277 channels,
16278 reason: RouteCloseReason::CapabilityDenied,
16279 drained: false,
16280 abandoned: 0,
16281 excluded_subscriptions: 0,
16282 terminal: Some(false),
16283 }
16284 );
16285 closed_channel
16286 }
16287
16288 #[tokio::test]
16289 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
16290 let registry = Arc::new(Registry::default());
16291 let forwarding = Arc::new(ForwardingTable::default());
16292 let supervisor = SupervisorHandle::new();
16293 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16294 let handler =
16295 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16296 .with_supervisor(supervisor);
16297 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
16298 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
16299 register_capability_manifest(
16300 &handler,
16301 &target_ctx,
16302 &mut target_rx,
16303 capability_manifest("target", &["credentials-provider/v1"], &[]),
16304 1,
16305 )
16306 .await;
16307 register_capability_manifest(
16308 &handler,
16309 &opener_ctx,
16310 &mut opener_rx,
16311 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16312 2,
16313 )
16314 .await;
16315
16316 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
16317 let replies = handler
16318 .handle_control_frame(
16319 &client_ctx,
16320 route_open_frame_with_admission_facts(
16321 3,
16322 "target",
16323 unique_project_root("admission-facts"),
16324 Some(ConsumerIdentity {
16325 module_id: "opener".to_string(),
16326 launch_nonce: "opener-nonce".to_string(),
16327 }),
16328 None,
16329 ),
16330 )
16331 .await
16332 .expect("denied route.open returns a typed frame");
16333 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16334 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16335 assert!(
16336 target_rx.try_recv().is_err(),
16337 "forbidden route.open must not relay route.bind"
16338 );
16339 }
16340
16341 #[tokio::test]
16342 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
16343 let registry = Arc::new(Registry::default());
16344 let forwarding = Arc::new(ForwardingTable::default());
16345 let supervisor = SupervisorHandle::new();
16346 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16347 let handler =
16348 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16349 .with_supervisor(supervisor);
16350 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
16351 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
16352 register_capability_manifest(
16353 &handler,
16354 &target_ctx,
16355 &mut target_rx,
16356 capability_manifest("target", &["credentials-provider/v1"], &[]),
16357 1,
16358 )
16359 .await;
16360 register_capability_manifest(
16361 &handler,
16362 &old_opener_ctx,
16363 &mut old_opener_rx,
16364 capability_manifest("opener", &[], &[]),
16365 2,
16366 )
16367 .await;
16368 let (mut client_rx, _) = open_route_for_capability_test(
16369 &handler,
16370 &target_ctx,
16371 &mut target_rx,
16372 712,
16373 3,
16374 "target",
16375 Some(ConsumerIdentity {
16376 module_id: "opener".to_string(),
16377 launch_nonce: "opener-nonce".to_string(),
16378 }),
16379 )
16380 .await;
16381 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16382
16383 handler
16384 .cleanup_connection(old_opener_ctx.connection_id)
16385 .expect("old opener registration cleans up");
16386 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
16387 register_capability_manifest(
16388 &handler,
16389 &new_opener_ctx,
16390 &mut new_opener_rx,
16391 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16392 4,
16393 )
16394 .await;
16395
16396 let closed_channel = assert_capability_denied_push(
16397 client_rx
16398 .try_recv()
16399 .expect("HELLO deny addition must emit route.closed")
16400 .frame,
16401 "target",
16402 );
16403 let goodbye = client_rx
16404 .try_recv()
16405 .expect("HELLO deny addition must send the client a route GOODBYE")
16406 .frame;
16407 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16408 assert_eq!(goodbye.header.channel, closed_channel);
16409 assert_ne!(goodbye.header.channel, 0);
16410 assert_ne!(goodbye.header.epoch, 0);
16411 assert!(client_rx.try_recv().is_err());
16412 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16413 assert!(matches!(
16414 target_rx.try_recv(),
16415 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16416 ));
16417 }
16418
16419 #[tokio::test]
16420 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
16421 let registry = Arc::new(Registry::default());
16422 let forwarding = Arc::new(ForwardingTable::default());
16423 let supervisor = SupervisorHandle::new();
16424 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16425 let handler =
16426 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16427 .with_supervisor(supervisor);
16428 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
16429 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
16430 register_capability_manifest(
16431 &handler,
16432 &target_ctx,
16433 &mut target_rx,
16434 capability_manifest("target", &[], &[]),
16435 1,
16436 )
16437 .await;
16438 register_capability_manifest(
16439 &handler,
16440 &opener_ctx,
16441 &mut opener_rx,
16442 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16443 2,
16444 )
16445 .await;
16446 let (mut client_rx, _) = open_route_for_capability_test(
16447 &handler,
16448 &target_ctx,
16449 &mut target_rx,
16450 722,
16451 3,
16452 "target",
16453 Some(ConsumerIdentity {
16454 module_id: "opener".to_string(),
16455 launch_nonce: "opener-nonce".to_string(),
16456 }),
16457 )
16458 .await;
16459 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16460
16461 let replies = handler
16462 .handle_control_frame(
16463 &target_ctx,
16464 catalog_update_with_capabilities_frame(
16465 4,
16466 CapabilityDeclarations {
16467 provides: vec!["credentials-provider/v1".to_string()],
16468 requires: Vec::new(),
16469 must_never_reach: Vec::new(),
16470 },
16471 ),
16472 )
16473 .await
16474 .expect("claim catalog.update succeeds");
16475 assert!(matches!(
16476 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
16477 Ok(ModuleControlResponseToModule::CatalogUpdate {})
16478 ));
16479 let closed_channel = assert_capability_denied_push(
16480 client_rx
16481 .try_recv()
16482 .expect("claim addition must emit route.closed")
16483 .frame,
16484 "target",
16485 );
16486 let goodbye = client_rx
16487 .try_recv()
16488 .expect("claim addition must send the client a route GOODBYE")
16489 .frame;
16490 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16491 assert_eq!(goodbye.header.channel, closed_channel);
16492 assert_ne!(goodbye.header.channel, 0);
16493 assert_ne!(goodbye.header.epoch, 0);
16494 assert!(client_rx.try_recv().is_err());
16495 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16496 assert!(matches!(
16497 target_rx.try_recv(),
16498 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16499 ));
16500 }
16501
16502 #[tokio::test]
16503 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
16504 let registry = Arc::new(Registry::default());
16505 let forwarding = Arc::new(ForwardingTable::default());
16506 let supervisor = SupervisorHandle::new();
16507 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16508 let handler =
16509 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16510 .with_supervisor(supervisor);
16511 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
16512 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
16513 register_capability_manifest(
16514 &handler,
16515 &target_ctx,
16516 &mut target_rx,
16517 capability_manifest("target", &["credentials-provider/v1"], &[]),
16518 1,
16519 )
16520 .await;
16521 register_capability_manifest(
16522 &handler,
16523 &opener_ctx,
16524 &mut opener_rx,
16525 capability_manifest("opener", &[], &[]),
16526 2,
16527 )
16528 .await;
16529 let (mut client_rx, _) = open_route_for_capability_test(
16530 &handler,
16531 &target_ctx,
16532 &mut target_rx,
16533 732,
16534 3,
16535 "target",
16536 Some(ConsumerIdentity {
16537 module_id: "opener".to_string(),
16538 launch_nonce: "opener-nonce".to_string(),
16539 }),
16540 )
16541 .await;
16542 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16543
16544 handler
16545 .handle_control_frame(
16546 &target_ctx,
16547 catalog_update_with_capabilities_frame(
16548 4,
16549 CapabilityDeclarations {
16550 provides: Vec::new(),
16551 requires: Vec::new(),
16552 must_never_reach: Vec::new(),
16553 },
16554 ),
16555 )
16556 .await
16557 .expect("claim removal catalog.update succeeds");
16558 assert_eq!(
16559 forwarding.active_binding_count().unwrap(),
16560 1,
16561 "removing an attested target claim must leave the route census unchanged"
16562 );
16563 assert!(
16564 client_rx.try_recv().is_err(),
16565 "claim removal must not emit route.closed or a client route GOODBYE"
16566 );
16567 assert!(
16568 target_rx.try_recv().is_err(),
16569 "claim removal must not send the target a route GOODBYE"
16570 );
16571 }
16572
16573 #[tokio::test]
16576 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
16577 let registry = Arc::new(Registry::default());
16578 let forwarding = Arc::new(ForwardingTable::default());
16579 let supervisor = SupervisorHandle::new();
16580 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16581 let handler =
16582 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16583 .with_supervisor(supervisor);
16584 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
16585 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
16586 register_capability_manifest(
16587 &handler,
16588 &target_ctx,
16589 &mut target_rx,
16590 capability_manifest("target", &["credentials-provider/v1"], &[]),
16591 1,
16592 )
16593 .await;
16594 register_capability_manifest(
16595 &handler,
16596 &opener_ctx,
16597 &mut opener_rx,
16598 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16599 2,
16600 )
16601 .await;
16602
16603 let (_client_rx, bind) = open_route_for_capability_test(
16604 &handler,
16605 &target_ctx,
16606 &mut target_rx,
16607 742,
16608 3,
16609 "target",
16610 None,
16611 )
16612 .await;
16613 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
16614 panic!("direct scope-honesty route must bind");
16615 };
16616 assert_eq!(principal, Some(Principal::Direct));
16617 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16618 }
16619
16620 #[tokio::test]
16623 async fn must_never_reach_self_route_is_capability_forbidden() {
16624 let registry = Arc::new(Registry::default());
16625 let forwarding = Arc::new(ForwardingTable::default());
16626 let supervisor = SupervisorHandle::new();
16627 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
16628 let handler =
16629 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16630 .with_supervisor(supervisor);
16631 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
16632 register_capability_manifest(
16633 &handler,
16634 &self_ctx,
16635 &mut self_rx,
16636 capability_manifest(
16637 "self-provider",
16638 &["credentials-provider/v1"],
16639 &["credentials-provider/v1"],
16640 ),
16641 1,
16642 )
16643 .await;
16644
16645 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
16646 let replies = handler
16647 .handle_control_frame(
16648 &client_ctx,
16649 route_open_frame_with_admission_facts(
16650 2,
16651 "self-provider",
16652 unique_project_root("admission-facts"),
16653 Some(ConsumerIdentity {
16654 module_id: "self-provider".to_string(),
16655 launch_nonce: "self-nonce".to_string(),
16656 }),
16657 None,
16658 ),
16659 )
16660 .await
16661 .expect("self-route refusal returns a typed frame");
16662 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16663 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16664 assert!(
16665 self_rx.try_recv().is_err(),
16666 "self denial must not relay route.bind"
16667 );
16668 }
16669
16670 #[test]
16671 fn unsupported_channel_zero_frame_returns_error() {
16672 let handler = ControlHandler::default();
16673 let request = Frame::build(
16674 FrameType::Request,
16675 control_flags(),
16676 0,
16677 0,
16678 21,
16679 b"opaque".to_vec(),
16680 )
16681 .unwrap();
16682
16683 let response = handler
16684 .handle_control(ConnectionId::new(1), request)
16685 .unwrap();
16686
16687 assert_eq!(response[0].header.ty, FrameType::Error);
16688 assert_eq!(
16689 parse_error(&response[0])["code"],
16690 "unsupported_control_frame"
16691 );
16692 }
16693
16694 mod swap {
16699 use super::*;
16700
16701 const INCUMBENT: ConnectionId = ConnectionId::new(30);
16702 const CANDIDATE: ConnectionId = ConnectionId::new(40);
16703
16704 struct Swap {
16705 registry: Arc<Registry>,
16706 forwarding: Arc<ForwardingTable>,
16707 handler: ControlHandler,
16708 incumbent_ctx: RouteCtx,
16709 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16710 candidate_ctx: RouteCtx,
16711 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16712 }
16713
16714 async fn swap_with_incumbent() -> Swap {
16715 let registry = Arc::new(Registry::default());
16716 let forwarding = Arc::new(ForwardingTable::default());
16717 let handler =
16718 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
16719 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
16720 hello_via_sink(
16721 &handler,
16722 &incumbent_ctx,
16723 &mut incumbent_rx,
16724 hello_frame("aft", PROTOCOL_VERSION, 7),
16725 )
16726 .await;
16727 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
16728 Swap {
16729 registry,
16730 forwarding,
16731 handler,
16732 incumbent_ctx,
16733 incumbent_rx,
16734 candidate_ctx,
16735 candidate_rx,
16736 }
16737 }
16738
16739 fn register_candidate(swap: &Swap, ready: Option<bool>) {
16740 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
16741 candidate_manifest.ready = ready;
16742 let registration = swap
16743 .registry
16744 .register_candidate_with_control_ops(
16745 candidate_manifest,
16746 PROTOCOL_VERSION,
16747 CANDIDATE,
16748 module_baseline_control_ops(),
16749 )
16750 .unwrap();
16751 swap.forwarding
16752 .register_candidate_module_connection(
16753 CANDIDATE,
16754 "aft".to_string(),
16755 PROTOCOL_VERSION,
16756 manifest_concurrency(®istration.manifest),
16757 swap.candidate_ctx.egress.clone(),
16758 )
16759 .unwrap();
16760 }
16761
16762 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
16763 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
16764 swap.registry.promote_candidate("aft").unwrap().unwrap();
16765 cutover.incumbent.unwrap()
16766 }
16767
16768 #[tokio::test]
16777 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
16778 let mut swap = swap_with_incumbent().await;
16779 let hold = Arc::new(std::sync::Barrier::new(2));
16780 let handler = swap
16781 .handler
16782 .clone()
16783 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
16784 handler
16785 .supervisor
16786 .open_swap("aft", "candidate-nonce".to_string());
16787 let hello = {
16788 let handler = handler.clone();
16789 let candidate_ctx = swap.candidate_ctx.clone();
16790 std::thread::spawn(move || {
16791 handler
16792 .handle_hello(
16793 candidate_ctx.connection_id,
16794 Some(candidate_ctx.egress.clone()),
16795 hello_frame_with_nonce(
16796 "aft",
16797 PROTOCOL_VERSION,
16798 8,
16799 Some("candidate-nonce"),
16800 ),
16801 )
16802 .unwrap()
16803 })
16804 };
16805
16806 hold.wait();
16807 assert!(swap
16810 .registry
16811 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
16812 .unwrap()
16813 .is_some());
16814 assert!(swap
16815 .registry
16816 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16817 .unwrap()
16818 .is_none());
16819 assert!(swap
16820 .forwarding
16821 .module_endpoint_for_connection(CANDIDATE)
16822 .unwrap()
16823 .is_none());
16824
16825 hold.wait();
16826 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
16827 assert_eq!(
16828 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
16829 FrameType::HelloAck
16830 );
16831 let published = swap
16832 .registry
16833 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16834 .unwrap()
16835 .expect("candidate is published once its HELLO completes");
16836 assert_eq!(published.connection_id, CANDIDATE);
16837 assert!(swap
16838 .forwarding
16839 .module_endpoint_for_connection(CANDIDATE)
16840 .unwrap()
16841 .is_some());
16842 }
16843
16844 fn keyed_total(counters: &Value, key: &str) -> u64 {
16845 counters[key]
16846 .as_object()
16847 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
16848 .unwrap_or(0)
16849 }
16850
16851 #[tokio::test]
16852 async fn replacement_logs_old_end_and_new_admission_once() {
16853 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16854 let mut swap = swap_with_incumbent().await;
16855 swap.handler
16856 .supervisor
16857 .open_swap("aft", "candidate-nonce".to_string());
16858 hello_via_sink(
16859 &swap.handler,
16860 &swap.candidate_ctx,
16861 &mut swap.candidate_rx,
16862 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
16863 )
16864 .await;
16865 cutover(&swap);
16866 swap.handler.cleanup_connection(INCUMBENT).unwrap();
16867
16868 let captured = crate::router::test_log::captured_logs(&logs);
16869 println!("captured replacement registry logs:\n{captured}");
16870 let new_admissions: Vec<_> = captured
16871 .lines()
16872 .filter(|line| {
16873 line.contains("connection_id=40")
16874 && line.contains("swap candidate registered; not routable until cutover")
16875 })
16876 .collect();
16877 assert_eq!(
16878 new_admissions.len(),
16879 1,
16880 "unexpected admission logs: {captured}"
16881 );
16882 assert!(new_admissions[0].contains("module_id=aft"));
16883 assert!(new_admissions[0].contains("connection_id=40"));
16884 assert!(
16885 new_admissions[0].contains("swap candidate registered; not routable until cutover")
16886 );
16887 assert!(new_admissions[0].contains("ready=true"));
16888
16889 let old_admissions: Vec<_> = captured
16890 .lines()
16891 .filter(|line| {
16892 line.contains("module registered module_id=aft ")
16893 && line.contains("connection_id=30")
16894 })
16895 .collect();
16896 assert_eq!(
16897 old_admissions.len(),
16898 1,
16899 "unexpected incumbent admission: {captured}"
16900 );
16901
16902 let old_ends: Vec<_> = captured
16903 .lines()
16904 .filter(|line| {
16905 line.contains("connection_id=30") && line.contains("module registration ended")
16906 })
16907 .collect();
16908 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
16909 assert!(old_ends[0].contains("module_id=aft"));
16910 assert!(old_ends[0].contains("connection_id=30"));
16911 assert!(old_ends[0].contains("reason=replaced"));
16912 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
16913 assert_eq!(
16914 captured
16915 .lines()
16916 .filter(|line| line.contains("module registration promoted"))
16917 .count(),
16918 1,
16919 "unexpected promotion logs: {captured}"
16920 );
16921 }
16922
16923 #[tokio::test]
16929 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
16930 let mut swap = swap_with_incumbent().await;
16931 let handler = swap.handler.clone();
16932
16933 let cotenant = ConnectionId::new(31);
16935 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
16936 let (cotenant_task, cotenant_bind) = relay_route_open(
16937 &handler,
16938 cotenant,
16939 &cotenant_ctx.egress,
16940 &mut swap.incumbent_rx,
16941 100,
16942 "aft",
16943 "swap-cotenant",
16944 )
16945 .await;
16946 handler
16947 .handle_control_frame(
16948 &swap.incumbent_ctx,
16949 route_bind_ack(cotenant_bind.header.corr),
16950 )
16951 .await
16952 .unwrap();
16953 assert!(cotenant_task.await.unwrap().is_empty());
16954 let (cotenant_channel, cotenant_epoch) =
16955 published_route(&cotenant_rx.recv().await.unwrap());
16956
16957 let caller = ConnectionId::new(32);
16959 let (caller_ctx, mut caller_rx) = route_ctx(caller);
16960 let (caller_task, caller_bind) = relay_route_open(
16961 &handler,
16962 caller,
16963 &caller_ctx.egress,
16964 &mut swap.incumbent_rx,
16965 101,
16966 "aft",
16967 "swap-caller",
16968 )
16969 .await;
16970 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
16971
16972 register_candidate(&swap, None);
16973 cutover(&swap);
16974
16975 let ack = handler
16977 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
16978 .await;
16979 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
16980 if module_loop_error.is_some() {
16981 handler.cleanup_connection(INCUMBENT).unwrap();
16984 }
16985
16986 assert!(
16988 cotenant_rx.try_recv().is_err(),
16989 "the co-tenant route on the incumbent was torn down by one late ack: \
16990 {module_loop_error:?}"
16991 );
16992 assert!(matches!(
16993 swap.forwarding
16994 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
16995 .unwrap(),
16996 DataRoute::Client(DataRouteState::Bound(_))
16997 ));
16998 assert_eq!(module_loop_error, None);
16999 assert!(swap
17000 .registry
17001 .get_module_by_connection(INCUMBENT)
17002 .unwrap()
17003 .is_some());
17004
17005 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
17007 .await
17008 .expect("the incumbent is told to drop the abandoned binding")
17009 .unwrap()
17010 .frame;
17011 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17012 assert_eq!(goodbye.header.channel, abandoned_channel);
17013 assert_eq!(goodbye.header.epoch, abandoned_epoch);
17014 assert!(swap.incumbent_rx.try_recv().is_err());
17015
17016 let response = caller_task.await.unwrap();
17018 assert_eq!(response.len(), 1);
17019 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17020 assert!(caller_rx.try_recv().is_err());
17021
17022 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
17026 let counters = handler.counters().snapshot();
17027 assert_eq!(
17028 keyed_total(&counters, "route_open_accepted_by_principal"),
17029 1
17030 );
17031 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
17032 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
17033 }
17034
17035 #[tokio::test]
17039 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
17040 let mut swap = swap_with_incumbent().await;
17041 register_candidate(&swap, None);
17042 let incumbent = cutover(&swap);
17043 swap.forwarding
17044 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
17045 .unwrap()
17046 .expect("the incumbent is still registered");
17047
17048 let client = ConnectionId::new(33);
17049 let (client_ctx, mut client_rx) = route_ctx(client);
17050 let route_handler = swap.handler.clone();
17051 let open_ctx = RouteCtx {
17052 connection_id: client,
17053 egress: client_ctx.egress.clone(),
17054 };
17055 let mut route_task = tokio::spawn(async move {
17056 route_handler
17057 .handle_control_frame(
17058 &open_ctx,
17059 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
17060 )
17061 .await
17062 .unwrap()
17063 });
17064 let bind = tokio::select! {
17065 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
17066 response = &mut route_task => {
17067 let response = response.unwrap();
17068 panic!(
17069 "post-cutover route.open was refused instead of relayed to the candidate: {}",
17070 parse_error(&response[0])["code"]
17071 );
17072 }
17073 };
17074 swap.handler
17075 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
17076 .await
17077 .unwrap();
17078 assert!(route_task.await.unwrap().is_empty());
17079 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
17080 match swap
17081 .forwarding
17082 .lookup_data_route(client, channel, epoch)
17083 .unwrap()
17084 {
17085 DataRoute::Client(DataRouteState::Bound(route)) => {
17086 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
17087 }
17088 other => panic!("expected a bound route on the candidate, got {other:?}"),
17089 }
17090 assert!(swap.incumbent_rx.try_recv().is_err());
17091 }
17092
17093 #[tokio::test]
17098 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
17099 let swap = swap_with_incumbent().await;
17100 register_candidate(&swap, Some(false));
17101 let update = Frame::build(
17102 FrameType::Request,
17103 control_flags(),
17104 0,
17105 0,
17106 55,
17107 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
17108 provides: manifest("aft", PROTOCOL_VERSION).provides,
17109 capabilities: None,
17110 ready: Some(true),
17111 })
17112 .unwrap(),
17113 )
17114 .unwrap();
17115
17116 let replies = swap
17117 .handler
17118 .handle_control_frame(&swap.candidate_ctx, update)
17119 .await
17120 .unwrap();
17121
17122 assert_eq!(replies.len(), 1);
17123 assert_eq!(
17124 replies[0].header.ty,
17125 FrameType::Response,
17126 "candidate catalog.update was refused: {:?}",
17127 serde_json::from_slice::<Value>(&replies[0].body).ok()
17128 );
17129 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
17130 assert_eq!(
17131 swap.registry
17132 .get_module("aft")
17133 .unwrap()
17134 .unwrap()
17135 .connection_id,
17136 INCUMBENT
17137 );
17138 }
17139 }
17140
17141 mod swap_admission {
17145 use super::*;
17146
17147 const INCUMBENT_NONCE: &str = "incumbent-nonce";
17148 const CANDIDATE_NONCE: &str = "candidate-nonce";
17149
17150 fn handler_with_incumbent(
17151 module_id: &str,
17152 reserved: bool,
17153 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
17154 let registry = Arc::new(Registry::default());
17155 let supervisor = SupervisorHandle::new();
17156 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
17157 if reserved {
17158 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
17159 }
17160 let handler =
17161 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
17162 let incumbent = handler
17163 .handle_control(
17164 ConnectionId::new(1),
17165 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
17166 )
17167 .unwrap();
17168 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
17169 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
17170 (registry, supervisor, handler)
17171 }
17172
17173 #[test]
17179 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
17180 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
17181
17182 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
17183 let replies = handler
17184 .handle_control(
17185 ConnectionId::new(connection),
17186 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
17187 )
17188 .unwrap();
17189 assert_eq!(replies[0].header.ty, FrameType::Error);
17190 assert_eq!(
17191 parse_error(&replies[0])["code"],
17192 "swap_token_invalid",
17193 "nonce {nonce:?}"
17194 );
17195 }
17196 assert!(registry.get_candidate("aft").unwrap().is_none());
17197 assert_eq!(
17198 registry.get_module("aft").unwrap().unwrap().connection_id,
17199 ConnectionId::new(1)
17200 );
17201
17202 let admitted = handler
17205 .handle_control(
17206 ConnectionId::new(4),
17207 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
17208 )
17209 .unwrap();
17210 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
17211 assert_eq!(
17212 registry
17213 .get_candidate("aft")
17214 .unwrap()
17215 .unwrap()
17216 .connection_id,
17217 ConnectionId::new(4)
17218 );
17219 assert_eq!(
17220 registry.get_module("aft").unwrap().unwrap().connection_id,
17221 ConnectionId::new(1),
17222 "the candidate must not take the active slot"
17223 );
17224 let replayed = handler
17225 .handle_control(
17226 ConnectionId::new(5),
17227 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
17228 )
17229 .unwrap();
17230 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
17231
17232 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
17236 let squatter = handler
17237 .handle_control(
17238 ConnectionId::new(6),
17239 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
17240 )
17241 .unwrap();
17242 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
17243 assert!(
17244 registry.get_module("aft").unwrap().is_none(),
17245 "a squatter took the active slot of an id being swapped"
17246 );
17247 }
17248
17249 #[test]
17254 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
17255 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
17256
17257 let replies = handler
17258 .handle_control(
17259 ConnectionId::new(2),
17260 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17261 )
17262 .unwrap();
17263
17264 assert_eq!(
17265 replies[0].header.ty,
17266 FrameType::HelloAck,
17267 "reserved candidate refused: {:?}",
17268 serde_json::from_slice::<Value>(&replies[0].body).ok()
17269 );
17270 assert_eq!(
17271 registry
17272 .get_candidate("vault")
17273 .unwrap()
17274 .unwrap()
17275 .connection_id,
17276 ConnectionId::new(2)
17277 );
17278 }
17279
17280 #[test]
17283 fn without_an_open_swap_the_ordinary_gates_decide() {
17284 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
17285 supervisor.close_swap("vault");
17286
17287 let candidate = handler
17288 .handle_control(
17289 ConnectionId::new(2),
17290 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17291 )
17292 .unwrap();
17293 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
17294 let duplicate = handler
17295 .handle_control(
17296 ConnectionId::new(3),
17297 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
17298 )
17299 .unwrap();
17300 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
17301 assert!(registry.get_candidate("vault").unwrap().is_none());
17302 }
17303 }
17304
17305 mod scopes {
17309 mod idle_tests {
17310 use super::*;
17311 include!("scope_idle_tests.rs");
17312 }
17313 mod route_open_timings {
17314 use super::*;
17315 include!("route_open_timing_tests.rs");
17316 }
17317 mod route_open_refusal_log {
17318 use super::*;
17319 include!("route_open_refusal_log_tests.rs");
17320 }
17321 mod activity {
17322 include!("activity_scope_tests.rs");
17323 }
17324 mod delta {
17325 include!("scope_handler_delta_tests.rs");
17326 }
17327 mod call_modes {
17328 include!("scope_call_mode_tests.rs");
17329 }
17330 #[tokio::test]
17331 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
17332 let rig = rig().await;
17333 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
17334 "s".into(),
17335 1,
17336 "tool".into(),
17337 ))
17338 .unwrap();
17339 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
17340 let (direct, _rx) = wide_ctx(999);
17341 for (ctx, expected) in [
17342 (&rig.owner, FrameType::Response),
17343 (&direct, FrameType::Error),
17344 ] {
17345 let frame =
17346 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
17347 .unwrap();
17348 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
17349 assert_eq!(replies[0].header.ty, expected);
17350 if expected == FrameType::Response {
17351 let reply: subc_control::ScopeServedByReply =
17352 serde_json::from_slice(&replies[0].body).unwrap();
17353 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
17354 } else {
17355 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
17356 }
17357 }
17358 let reply = call(
17359 &rig.handler,
17360 &rig.owner,
17361 &ModuleControlRequestFromModule::ScopeDescribe {
17362 owner: Principal::Reserved {
17363 module_id: OWNER.into(),
17364 },
17365 scope_ref: "s".into(),
17366 },
17367 )
17368 .await;
17369 assert!(matches!(
17370 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
17371 ModuleControlResponseToModule::ScopeDescribe { .. }
17372 ));
17373 }
17374
17375 use subc_protocol::scope::{
17376 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
17377 ScopeStatus,
17378 };
17379
17380 use super::*;
17381
17382 const OWNER: &str = "prefrontal-core";
17383
17384 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
17385 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
17386 }
17387
17388 async fn call(
17389 handler: &ControlHandler,
17390 ctx: &RouteCtx,
17391 request: &ModuleControlRequestFromModule,
17392 ) -> Frame {
17393 let body = serde_json::to_vec(request).unwrap();
17394 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
17395 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
17396 assert_eq!(replies.len(), 1, "{replies:?}");
17397 replies.pop().unwrap()
17398 }
17399
17400 async fn sync(
17401 handler: &ControlHandler,
17402 ctx: &RouteCtx,
17403 generation: u64,
17404 scopes: Vec<ScopeRecord>,
17405 ) -> Result<ModuleControlResponseToModule, String> {
17406 let reply = call(
17407 handler,
17408 ctx,
17409 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
17410 )
17411 .await;
17412 match reply.header.ty {
17413 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
17414 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
17415 }
17416 }
17417
17418 async fn describe(
17419 handler: &ControlHandler,
17420 ctx: &RouteCtx,
17421 owner: &str,
17422 scope_ref: &str,
17423 ) -> ModuleControlResponseToModule {
17424 let reply = call(
17425 handler,
17426 ctx,
17427 &ModuleControlRequestFromModule::ScopeDescribe {
17428 owner: Principal::Reserved {
17429 module_id: owner.to_string(),
17430 },
17431 scope_ref: scope_ref.to_string(),
17432 },
17433 )
17434 .await;
17435 assert_eq!(
17436 reply.header.ty,
17437 FrameType::Response,
17438 "{:?}",
17439 parse_error(&reply)
17440 );
17441 serde_json::from_slice(&reply.body).unwrap()
17442 }
17443
17444 #[tokio::test]
17445 async fn scope_describe_never_echoes_bind_proof_or_hash() {
17446 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17447 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17448 let mut rig = rig().await;
17449 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17450 .await;
17451 let _bound = rig
17452 .bound(
17453 Some(AFT),
17454 PLEXUS,
17455 Some(
17456 rig.selector("s", Some(1))
17457 .with_bind_proof(Some(PROOF.into())),
17458 ),
17459 )
17460 .await;
17461 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
17462 let output = serde_json::to_string(&reply).unwrap();
17463 assert!(output.contains("\"status\":\"live\""));
17464 assert!(!output.contains(PROOF));
17465 assert!(!output.contains(HASH));
17466 }
17467
17468 async fn module(
17470 handler: &ControlHandler,
17471 connection: u64,
17472 module_id: &str,
17473 nonce: Option<&str>,
17474 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17475 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
17476 hello_via_sink(
17477 handler,
17478 &ctx,
17479 &mut rx,
17480 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
17481 )
17482 .await;
17483 (ctx, rx)
17484 }
17485
17486 #[tokio::test]
17489 async fn a_client_connection_cannot_sync_or_describe() {
17490 let handler = ControlHandler::new(Arc::new(Registry::default()));
17491 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
17492 for request in [
17493 ModuleControlRequestFromModule::ScopeSync {
17494 generation: 1,
17495 scopes: vec![head("s", 1)],
17496 },
17497 ModuleControlRequestFromModule::ScopeDescribe {
17498 owner: Principal::Direct,
17499 scope_ref: "s".to_string(),
17500 },
17501 ] {
17502 let reply = call(&handler, &ctx, &request).await;
17503 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
17504 }
17505 assert!(
17506 !handler
17507 .scopes
17508 .read()
17509 .unwrap()
17510 .describe(
17511 &Principal::Reserved {
17512 module_id: OWNER.to_string()
17513 },
17514 "s"
17515 )
17516 .owner_synced
17517 );
17518 }
17519
17520 #[tokio::test]
17523 async fn a_module_without_a_supervised_launch_cannot_sync() {
17524 let handler = ControlHandler::new(Arc::new(Registry::default()));
17525 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
17526 assert_eq!(
17527 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
17528 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17529 );
17530 }
17531
17532 #[tokio::test]
17533 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
17534 let supervisor = SupervisorHandle::new();
17535 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17536 let handler = ControlHandler::new(Arc::new(Registry::default()))
17537 .with_supervisor(supervisor.clone());
17538 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17539 sync(&handler, &incumbent, 1, vec![head("s", 1)])
17540 .await
17541 .expect("the current launch syncs");
17542
17543 supervisor.open_swap(OWNER, "n2".to_string());
17546 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
17547 assert_eq!(
17548 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17549 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17550 );
17551 sync(&handler, &incumbent, 2, vec![head("s", 1)])
17552 .await
17553 .expect("the serving owner syncs during the swap");
17554
17555 supervisor.close_swap(OWNER);
17558 assert_eq!(
17559 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17560 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17561 );
17562 sync(&handler, &incumbent, 3, vec![head("s", 1)])
17563 .await
17564 .expect("the serving owner syncs after the rollback");
17565 handler.cleanup_connection(candidate.connection_id).unwrap();
17566
17567 supervisor.open_swap(OWNER, "n3".to_string());
17572 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
17573 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
17574 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
17575 .await
17576 .expect("the promoted launch takes authority");
17577 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17578 panic!("unexpected reply {reply:?}");
17579 };
17580 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
17581 assert_eq!(
17582 sync(&handler, &incumbent, 4, Vec::new()).await,
17583 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17584 );
17585 }
17586
17587 #[tokio::test]
17590 async fn closing_the_authority_connection_frees_sync_authority() {
17591 let supervisor = SupervisorHandle::new();
17592 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17593 let handler = ControlHandler::new(Arc::new(Registry::default()))
17594 .with_supervisor(supervisor.clone());
17595 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17596 sync(&handler, &first, 10, vec![head("s", 1)])
17597 .await
17598 .unwrap();
17599 handler.cleanup_connection(first.connection_id).unwrap();
17600
17601 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
17602 sync(&handler, &second, 1, vec![head("s", 1)])
17603 .await
17604 .expect("the next connection takes the released authority");
17605 }
17606
17607 #[tokio::test]
17608 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
17609 let supervisor = SupervisorHandle::new();
17610 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17611 let handler =
17612 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
17613 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
17614 sync(&handler, &first, 10, vec![head("s", 1)])
17615 .await
17616 .unwrap();
17617 handler
17618 .handle_control_frame(
17619 &first,
17620 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
17621 )
17622 .await
17623 .unwrap();
17624 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
17625 sync(&handler, &second, 1, vec![head("s", 1)])
17626 .await
17627 .expect("GOODBYE releases authority even if the old socket remains open");
17628 }
17629
17630 #[tokio::test]
17631 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
17632 let registry = Arc::new(Registry::default());
17633 let supervisor_handle = SupervisorHandle::new();
17634 let supervisor =
17635 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
17636 .with_handle(supervisor_handle.clone())
17637 .with_daemon_incarnation("incarnation-7".to_string());
17638 supervisor
17641 .supervise_configured(
17642 ModuleSpec {
17643 carry_containment: false,
17644 module_id: OWNER.to_string(),
17645 program: PathBuf::from("/nonexistent/prefrontal-core"),
17646 args: Vec::new(),
17647 env: Vec::new(),
17648 reserved: false,
17649 reserved_prefixes: Vec::new(),
17650 protocol: ModuleProtocol::Subc,
17651 overlap: Default::default(),
17652 },
17653 false,
17654 )
17655 .unwrap();
17656 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
17657 let handler =
17658 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17659 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
17660
17661 let ModuleControlResponseToModule::ScopeDescribe {
17663 status,
17664 daemon_incarnation,
17665 owner_synced,
17666 owner_configured,
17667 scope,
17668 ..
17669 } = describe(&handler, &reader, OWNER, "s").await
17670 else {
17671 panic!("not a describe reply");
17672 };
17673 assert_eq!(status, ScopeStatus::NotLive);
17674 assert_eq!(daemon_incarnation, "incarnation-7");
17675 assert!(!owner_synced);
17676 assert!(owner_configured);
17677 assert!(scope.is_none());
17678
17679 let ModuleControlResponseToModule::ScopeDescribe {
17682 status,
17683 owner_configured,
17684 ..
17685 } = describe(&handler, &reader, "ghost", "s").await
17686 else {
17687 panic!("not a describe reply");
17688 };
17689 assert_eq!(status, ScopeStatus::NotLive);
17690 assert!(!owner_configured);
17691
17692 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
17694 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
17695 let ModuleControlResponseToModule::ScopeDescribe {
17696 status,
17697 scope_epoch,
17698 owner_synced,
17699 scope,
17700 ..
17701 } = describe(&handler, &reader, OWNER, "s").await
17702 else {
17703 panic!("not a describe reply");
17704 };
17705 assert_eq!(status, ScopeStatus::Live);
17706 assert_eq!(scope_epoch, Some(4));
17707 assert!(owner_synced);
17708 let stamp = scope.expect("a live scope carries its stamp");
17709 assert!(
17710 stamp.owner_authorized,
17711 "prefrontal-core is the default authority"
17712 );
17713 assert_eq!(stamp.kind, ScopeKind::Head);
17714 }
17715
17716 #[tokio::test]
17717 async fn scope_authority_owners_decides_owner_authorized() {
17718 let supervisor = SupervisorHandle::new();
17719 supervisor.set_spawn_nonce("broca", "b1".to_string());
17720 let handler = ControlHandler::new(Arc::new(Registry::default()))
17721 .with_supervisor(supervisor)
17722 .with_scope_authority_owners(vec!["broca".to_string()]);
17723 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
17724 let mut gated = head("s", 1);
17725 gated.attributes.agent_id = Some("agent".to_string());
17726 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
17727 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
17728 describe(&handler, &broca, "broca", "s").await
17729 else {
17730 panic!("not a describe reply");
17731 };
17732 assert!(scope.unwrap().owner_authorized);
17733 }
17734
17735 #[tokio::test]
17739 async fn scope_ops_and_the_scopes_capability_are_advertised() {
17740 let handler = ControlHandler::new(Arc::new(Registry::default()));
17741 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
17742 let ack = hello_via_sink(
17743 &handler,
17744 &ctx,
17745 &mut rx,
17746 hello_frame("m", PROTOCOL_VERSION, 1),
17747 )
17748 .await;
17749 let ack = parse_ack(&ack);
17750 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
17751 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
17752 }
17753 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
17754
17755 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
17756 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
17757 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
17758 let reply = handler
17759 .handle_control_frame(&client, frame)
17760 .await
17761 .unwrap()
17762 .pop()
17763 .unwrap();
17764 let ClientControlResponse::ServerDescribe { capabilities, .. } =
17765 serde_json::from_slice(&reply.body).unwrap()
17766 else {
17767 panic!("not a server.describe reply");
17768 };
17769 assert!(
17770 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
17771 "{capabilities:?}"
17772 );
17773 }
17774
17775 const PLEXUS: &str = "plexus";
17778 const OTHER: &str = "other";
17779 const AFT: &str = "aft";
17780 const BROCA: &str = "broca";
17781 const MAGIC: &str = "magic-context";
17782
17783 fn nonce(module_id: &str) -> String {
17784 format!("nonce-{module_id}")
17785 }
17786
17787 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17788 let (tx, rx) = mpsc::channel(64);
17789 (
17790 RouteCtx {
17791 connection_id: ConnectionId::new(connection),
17792 egress: FrameSink::new(tx),
17793 },
17794 rx,
17795 )
17796 }
17797
17798 struct Rig {
17802 handler: ControlHandler,
17803 forwarding: Arc<ForwardingTable>,
17804 owner: RouteCtx,
17805 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17806 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
17807 generation: u64,
17808 next_connection: u64,
17809 _supervisor: Supervisor,
17810 }
17811
17812 async fn rig() -> Rig {
17813 rig_with_flow_support(true).await
17814 }
17815
17816 async fn rig_with_flow_support(flow_support: bool) -> Rig {
17817 rig_with_scope_capabilities(flow_support, false).await
17818 }
17819
17820 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
17821 let mut provides = Vec::new();
17822 if flow_support {
17823 provides.push("flow-scopes/v1");
17824 }
17825 if run_support {
17826 provides.push("agent-run-scopes/v1");
17827 }
17828 rig_with_target_capabilities(&provides).await
17829 }
17830
17831 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
17832 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
17833 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17834 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17837 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
17838 Frame::build(
17839 FrameType::Hello,
17840 control_flags(),
17841 0,
17842 0,
17843 connection,
17844 serde_json::to_vec(&body).unwrap(),
17845 )
17846 .unwrap()
17847 }
17848
17849 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
17850 let registry = Arc::new(Registry::default());
17851 let forwarding = Arc::new(ForwardingTable::default());
17852 let supervisor_handle = SupervisorHandle::new();
17853 let supervisor =
17854 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
17855 .with_handle(supervisor_handle.clone());
17856 supervisor
17857 .supervise_configured(
17858 ModuleSpec {
17859 carry_containment: false,
17860 module_id: OWNER.to_string(),
17861 program: PathBuf::from("/nonexistent/prefrontal-core"),
17862 args: Vec::new(),
17863 env: Vec::new(),
17864 reserved: false,
17865 reserved_prefixes: Vec::new(),
17866 protocol: ModuleProtocol::Subc,
17867 overlap: Default::default(),
17868 },
17869 false,
17870 )
17871 .unwrap();
17872 for module_id in [OWNER, AFT, BROCA, MAGIC] {
17873 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
17874 }
17875 let handler =
17876 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17877 .with_supervisor(supervisor_handle);
17878 let (owner, mut owner_rx) = wide_ctx(1);
17879 hello_via_sink(
17880 &handler,
17881 &owner,
17882 &mut owner_rx,
17883 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
17884 )
17885 .await;
17886 let mut modules = BTreeMap::new();
17887 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
17888 let (ctx, mut rx) = wide_ctx(connection);
17889 let hello = hello_with_capabilities(module_id, connection, provides);
17890 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
17891 modules.insert(module_id.to_string(), (ctx, rx));
17892 }
17893 Rig {
17894 handler,
17895 forwarding,
17896 owner,
17897 _owner_rx: owner_rx,
17898 modules,
17899 generation: 0,
17900 next_connection: 100,
17901 _supervisor: supervisor,
17902 }
17903 }
17904
17905 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
17906 ScopeCarrier::new(Principal::Reserved {
17907 module_id: module_id.to_string(),
17908 })
17909 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
17910 }
17911
17912 fn session(scope_epoch: u64) -> ScopeRecord {
17915 let mut record = head("s", scope_epoch);
17916 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
17917 record.attributes.agent_id = Some("agent-1".to_string());
17918 record.attributes.delegates = true;
17919 record
17920 }
17921
17922 impl Rig {
17923 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
17924 self.generation += 1;
17925 sync(&self.handler, &self.owner, self.generation, scopes)
17926 .await
17927 .expect("the owner's sync is accepted");
17928 }
17929
17930 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17931 ScopeSelector::new(
17932 Principal::Reserved {
17933 module_id: OWNER.to_string(),
17934 },
17935 scope_ref.to_string(),
17936 scope_epoch,
17937 )
17938 }
17939
17940 fn open_frame(
17941 &mut self,
17942 opener: Option<&str>,
17943 target: &str,
17944 scope: Option<ScopeSelector>,
17945 ) -> (
17946 RouteCtx,
17947 mpsc::Receiver<crate::router::OutboundFrame>,
17948 Frame,
17949 ) {
17950 self.next_connection += 1;
17951 let (ctx, rx) = wide_ctx(self.next_connection);
17952 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
17953 busy_refusals: None,
17954 target: RouteTarget::ToolProvider {
17955 module_id: target.to_string(),
17956 },
17957 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
17958 consumer_identity: opener.map(|module_id| ConsumerIdentity {
17959 module_id: module_id.to_string(),
17960 launch_nonce: nonce(module_id),
17961 }),
17962 consumer_capabilities: None,
17963 role_versions: None,
17964 admission_facts: None,
17965 scope,
17966 })
17967 .unwrap();
17968 let frame = Frame::build(
17969 FrameType::Request,
17970 control_flags(),
17971 0,
17972 0,
17973 self.next_connection,
17974 body,
17975 )
17976 .unwrap();
17977 (ctx, rx, frame)
17978 }
17979
17980 async fn refused(
17982 &mut self,
17983 opener: Option<&str>,
17984 target: &str,
17985 scope: Option<ScopeSelector>,
17986 ) -> String {
17987 self.refusal_body(opener, target, scope).await["code"]
17988 .as_str()
17989 .unwrap()
17990 .to_string()
17991 }
17992
17993 async fn refusal_body(
17994 &mut self,
17995 opener: Option<&str>,
17996 target: &str,
17997 scope: Option<ScopeSelector>,
17998 ) -> Value {
17999 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
18000 let replies = tokio::time::timeout(
18001 Duration::from_secs(2),
18002 self.handler.handle_control_frame(&ctx, frame),
18003 )
18004 .await
18005 .expect("the open must be refused before waiting for a bind ack")
18006 .unwrap();
18007 assert_eq!(replies.len(), 1, "{replies:?}");
18008 assert_eq!(replies[0].header.ty, FrameType::Error);
18009 let (_, module_rx) = self.modules.get_mut(target).unwrap();
18010 assert!(
18011 module_rx.try_recv().is_err(),
18012 "a refused open relays nothing"
18013 );
18014 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
18015 parse_error(&replies[0])
18016 }
18017
18018 async fn relayed(
18020 &mut self,
18021 opener: Option<&str>,
18022 target: &str,
18023 scope: Option<ScopeSelector>,
18024 ) -> Relayed {
18025 self.relayed_with_legacy_root(opener, target, scope, None)
18026 .await
18027 }
18028
18029 async fn relayed_with_legacy_root(
18030 &mut self,
18031 opener: Option<&str>,
18032 target: &str,
18033 scope: Option<ScopeSelector>,
18034 legacy_root: Option<&std::path::Path>,
18035 ) -> Relayed {
18036 let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
18037 if let Some(root) = legacy_root {
18038 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
18039 body["identity"]["project_root"] = serde_json::json!(root);
18040 frame.body = serde_json::to_vec(&body).unwrap();
18041 }
18042 let handler = self.handler.clone();
18043 let task_ctx = ctx.clone();
18044 let task = tokio::spawn(async move {
18045 handler
18046 .handle_control_frame(&task_ctx, frame)
18047 .await
18048 .unwrap()
18049 });
18050 let (_, module_rx) = self.modules.get_mut(target).unwrap();
18051 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
18052 .await
18053 .expect("the target receives the relayed route.bind")
18054 .unwrap()
18055 .frame;
18056 Relayed {
18057 target: target.to_string(),
18058 client: ctx,
18059 client_rx: rx,
18060 task,
18061 bind,
18062 }
18063 }
18064
18065 async fn ack(&self, relayed: &Relayed) {
18066 let (module, _) = &self.modules[&relayed.target];
18067 self.handler
18068 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
18069 .await
18070 .unwrap();
18071 }
18072
18073 async fn bound(
18075 &mut self,
18076 opener: Option<&str>,
18077 target: &str,
18078 scope: Option<ScopeSelector>,
18079 ) -> Bound {
18080 let relayed = self.relayed(opener, target, scope).await;
18081 self.commit_relayed(relayed).await
18082 }
18083
18084 async fn bound_with_legacy_root(
18085 &mut self,
18086 opener: Option<&str>,
18087 target: &str,
18088 scope: Option<ScopeSelector>,
18089 root: &std::path::Path,
18090 ) -> Bound {
18091 let relayed = self
18092 .relayed_with_legacy_root(opener, target, scope, Some(root))
18093 .await;
18094 self.commit_relayed(relayed).await
18095 }
18096
18097 async fn commit_relayed(&self, relayed: Relayed) -> Bound {
18098 self.ack(&relayed).await;
18099 let Relayed {
18100 target,
18101 client,
18102 mut client_rx,
18103 task,
18104 bind,
18105 } = relayed;
18106 assert!(
18107 task.await.unwrap().is_empty(),
18108 "the open is answered by commit"
18109 );
18110 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
18111 Bound {
18112 target,
18113 client,
18114 client_rx,
18115 channel,
18116 epoch,
18117 bind,
18118 }
18119 }
18120
18121 fn live(&self, route: &Bound) -> bool {
18122 matches!(
18123 self.forwarding
18124 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18125 .unwrap(),
18126 DataRoute::Client(DataRouteState::Bound(_))
18127 )
18128 }
18129 }
18130
18131 struct Relayed {
18132 target: String,
18133 client: RouteCtx,
18134 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18135 task: tokio::task::JoinHandle<Vec<Frame>>,
18136 bind: Frame,
18137 }
18138
18139 struct Bound {
18140 target: String,
18141 client: RouteCtx,
18142 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18143 channel: u16,
18144 epoch: u32,
18145 bind: Frame,
18146 }
18147
18148 impl Bound {
18149 fn closed_reason(&mut self) -> RouteCloseReason {
18152 let mut reason = None;
18153 let mut goodbye = false;
18154 while let Ok(outbound) = self.client_rx.try_recv() {
18155 let frame = outbound.frame;
18156 match frame.header.ty {
18157 FrameType::Goodbye => {
18158 assert_eq!(
18159 (frame.header.channel, frame.header.epoch),
18160 (self.channel, self.epoch)
18161 );
18162 goodbye = true;
18163 }
18164 FrameType::Push => {
18165 let ClientControlPush::RouteClosed {
18166 reason: r,
18167 module_id,
18168 ..
18169 } = serde_json::from_slice(&frame.body).unwrap()
18170 else {
18171 panic!("unexpected push");
18172 };
18173 assert_eq!(module_id, self.target);
18174 reason = Some(r);
18175 }
18176 other => panic!("unexpected frame {other:?}"),
18177 }
18178 }
18179 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
18180 reason.expect("the client is told why the route closed")
18181 }
18182
18183 fn untouched(&mut self) -> bool {
18184 self.client_rx.try_recv().is_err()
18185 }
18186
18187 fn stamp(&self) -> Option<ScopeStamp> {
18188 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
18189 ModuleControlRequest::RouteBind { scope, .. } => scope,
18190 other => panic!("expected a route.bind, got {other:?}"),
18191 }
18192 }
18193 }
18194
18195 #[tokio::test]
18196 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
18197 ) {
18198 let mut rig = rig().await;
18199 let mut record = session(1);
18200 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
18201 record.child_owners = vec![Principal::Reserved {
18202 module_id: MAGIC.to_string(),
18203 }];
18204 rig.sync(vec![record]).await;
18205 let scope = || Some(rig_selector("s", Some(1)));
18206
18207 rig.bound(Some(OWNER), PLEXUS, scope()).await;
18210 rig.bound(Some(AFT), OTHER, scope()).await;
18211 rig.bound(Some(BROCA), PLEXUS, scope()).await;
18212
18213 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
18217 assert_eq!(
18218 rig.refused(opener, target, scope()).await,
18219 error_codes::SCOPE_NOT_CARRIER,
18220 "{opener:?} -> {target}"
18221 );
18222 }
18223 }
18224
18225 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18226 ScopeSelector::new(
18227 Principal::Reserved {
18228 module_id: OWNER.to_string(),
18229 },
18230 scope_ref.to_string(),
18231 scope_epoch,
18232 )
18233 }
18234
18235 #[tokio::test]
18236 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
18237 let mut rig = rig().await;
18238 rig.sync(vec![session(1)]).await;
18239 for opener in [OWNER, AFT] {
18240 assert_eq!(
18241 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
18242 .await,
18243 error_codes::SCOPE_EPOCH_REQUIRED,
18244 "{opener}"
18245 );
18246 }
18247 }
18248
18249 #[tokio::test]
18250 async fn admission_separates_not_synced_not_live_and_ended() {
18251 let mut rig = rig().await;
18252 let code = rig
18254 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18255 .await;
18256 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
18257 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
18258
18259 let ghost = ScopeSelector::new(
18261 Principal::Reserved {
18262 module_id: "ghost".to_string(),
18263 },
18264 "s".to_string(),
18265 Some(1),
18266 );
18267 assert_eq!(
18268 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
18269 error_codes::SCOPE_NOT_LIVE
18270 );
18271
18272 rig.sync(vec![session(2)]).await;
18273 assert_eq!(
18274 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
18275 .await,
18276 error_codes::SCOPE_NOT_LIVE
18277 );
18278 for epoch in [1, 3] {
18279 assert_eq!(
18280 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
18281 .await,
18282 error_codes::SCOPE_ENDED,
18283 "epoch {epoch}"
18284 );
18285 }
18286 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
18288 .await;
18289 }
18290
18291 #[tokio::test]
18292 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
18293 let mut rig = rig().await;
18294 rig.sync(vec![session(1)]).await;
18295 let route = rig
18296 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18297 .await;
18298 let stamp = route.stamp().expect("a scoped bind carries the stamp");
18299 assert_eq!(stamp.scope_ref, "s");
18300 assert_eq!(stamp.scope_epoch, 1);
18301 assert_eq!(stamp.kind, ScopeKind::Head);
18302 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
18303 assert!(stamp.attributes.delegates);
18304 assert!(stamp.owner_authorized);
18305 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18306 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
18307
18308 let (broca, mut broca_rx) = wide_ctx(50);
18311 hello_via_sink(
18312 &rig.handler,
18313 &broca,
18314 &mut broca_rx,
18315 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
18316 )
18317 .await;
18318 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
18319 .await
18320 .unwrap();
18321 let own = ScopeSelector::new(
18322 Principal::Reserved {
18323 module_id: BROCA.to_string(),
18324 },
18325 "b".to_string(),
18326 Some(1),
18327 );
18328 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
18329 assert!(!route.stamp().unwrap().owner_authorized);
18330 }
18331
18332 #[tokio::test]
18333 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
18334 let mut rig = rig().await;
18335 let mut record = head("s", 1);
18336 record.carriers = vec![carrier(AFT, None)];
18337 let flow_id = "Flow:run-7/step_2!~";
18338 record.attributes.flow_id = Some(flow_id.to_string());
18339 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
18340 .await
18341 .unwrap();
18342 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
18343 panic!("not a sync reply");
18344 };
18345 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
18346 let route = rig
18347 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18348 .await;
18349 assert!(rig.live(&route), "the stamped bind committed");
18350 let stamp = route.stamp().expect("a flow scope carries a stamp");
18351 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
18352 assert_eq!(stamp.attributes.agent_id, None);
18353 assert!(!stamp.attributes.delegates);
18354 assert!(stamp.owner_authorized);
18355 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18356 assert_eq!(unscoped.stamp(), None);
18357 }
18358
18359 #[tokio::test]
18360 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
18361 let mut rig = rig_with_flow_support(false).await;
18362 let mut record = session(1);
18363 record.attributes.flow_id = Some("flow:7".to_string());
18364 rig.sync(vec![record]).await;
18365 for opener in [OWNER, AFT] {
18366 let body = rig
18367 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18368 .await;
18369 assert_eq!(body["code"], "target_flow_unsupported");
18370 let message = body["message"].as_str().unwrap();
18371 for required in [PLEXUS, "flow-scopes/v1"] {
18372 assert!(message.contains(required), "{message}");
18373 }
18374 }
18375 }
18376
18377 #[tokio::test]
18378 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
18379 let mut rig = rig_with_flow_support(true).await;
18380 let mut record = session(1);
18381 record.attributes.flow_id = Some("flow:7".to_string());
18382 rig.sync(vec![record]).await;
18383 for opener in [OWNER, AFT] {
18384 let route = rig
18385 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18386 .await;
18387 assert!(rig.live(&route));
18388 assert_eq!(
18389 route.stamp().unwrap().attributes.flow_id.as_deref(),
18390 Some("flow:7")
18391 );
18392 }
18393 }
18394
18395 #[tokio::test]
18396 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
18397 use std::future::Future;
18398
18399 let mut rig = rig_with_flow_support(true).await;
18400 let mut record = session(1);
18401 record.attributes.flow_id = Some("flow:7".to_string());
18402 rig.sync(vec![record]).await;
18403 let (client, mut client_rx, frame) =
18404 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
18405 for _ in 0..64 {
18408 client.egress.try_send(route_bind_ack(1)).unwrap();
18409 }
18410 let handler = rig.handler.clone();
18411 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18412 std::future::poll_fn(|cx| {
18413 assert!(open.as_mut().poll(cx).is_pending());
18414 std::task::Poll::Ready(())
18415 })
18416 .await;
18417 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18418
18419 let old_connection = rig.modules[PLEXUS].0.connection_id;
18420 rig.handler.cleanup_connection(old_connection).unwrap();
18421 let (replacement, mut replacement_rx) = wide_ctx(200);
18422 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
18423 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18424 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18425 let hello = Frame::build(
18426 FrameType::Hello,
18427 control_flags(),
18428 0,
18429 0,
18430 200,
18431 serde_json::to_vec(&body).unwrap(),
18432 )
18433 .unwrap();
18434 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
18435
18436 client_rx.try_recv().unwrap();
18437 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18438 .await
18439 .expect("the replacement is refused without waiting for a bind ack")
18440 .unwrap();
18441 assert_eq!(replies.len(), 1);
18442 let body = parse_error(&replies[0]);
18443 assert_eq!(body["code"], "target_flow_unsupported");
18444 for required in [PLEXUS, "flow-scopes/v1"] {
18445 assert!(body["message"].as_str().unwrap().contains(required));
18446 }
18447 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
18448 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18449 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18450 assert!(rig
18451 .handler
18452 .registry
18453 .get_module_by_connection(replacement.connection_id)
18454 .unwrap()
18455 .is_some());
18456 }
18457
18458 #[tokio::test]
18459 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
18460 use std::future::Future;
18461 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
18462 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18463 const NEXT_HASH: &str =
18464 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18465 let mut rig = rig().await;
18466 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
18467 record.carriers.push(ScopeCarrier::new(Principal::Direct));
18468 rig.sync(vec![record.clone()]).await;
18469 let (client, mut rx, frame) = rig.open_frame(
18470 None,
18471 PLEXUS,
18472 Some(
18473 rig.selector("s", Some(1))
18474 .with_bind_proof(Some(PROOF.into())),
18475 ),
18476 );
18477 for _ in 0..64 {
18480 client.egress.try_send(route_bind_ack(1)).unwrap();
18481 }
18482 let handler = rig.handler.clone();
18483 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18484 std::future::poll_fn(|cx| {
18485 assert!(open.as_mut().poll(cx).is_pending());
18486 std::task::Poll::Ready(())
18487 })
18488 .await;
18489 record.bind_proof_sha256 = Some(NEXT_HASH.into());
18490 rig.sync(vec![record]).await;
18491 rx.try_recv().unwrap();
18492 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18493 .await
18494 .unwrap()
18495 .unwrap();
18496 assert_eq!(
18497 parse_error(&replies[0])["code"],
18498 error_codes::SCOPE_BIND_PROOF_MISMATCH
18499 );
18500 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
18501 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18502 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18503 }
18504
18505 #[tokio::test]
18506 async fn scope_bind_proof_reserved_openers_ignore_proof() {
18507 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18508 let mut rig = rig().await;
18509 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18510 .await;
18511 for opener in [OWNER, AFT] {
18512 for proof in [None, Some("malformed+proof=")] {
18513 let route = rig
18514 .bound(
18515 Some(opener),
18516 PLEXUS,
18517 Some(
18518 rig.selector("s", Some(1))
18519 .with_bind_proof(proof.map(str::to_string)),
18520 ),
18521 )
18522 .await;
18523 assert!(rig.live(&route));
18524 }
18525 }
18526 }
18527
18528 #[tokio::test]
18529 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
18530 use std::future::Future;
18531 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18532 const NEXT_HASH: &str =
18533 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18534 let mut rig = rig().await;
18535 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18536 .await;
18537 let (client, mut rx, frame) =
18538 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
18539 for _ in 0..64 {
18540 client.egress.try_send(route_bind_ack(1)).unwrap();
18541 }
18542 let handler = rig.handler.clone();
18543 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18544 std::future::poll_fn(|cx| {
18545 assert!(open.as_mut().poll(cx).is_pending());
18546 std::task::Poll::Ready(())
18547 })
18548 .await;
18549 rig.sync(vec![
18550 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
18551 ])
18552 .await;
18553 rx.try_recv().unwrap();
18554 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18555 let bind = std::future::poll_fn(|cx| {
18556 assert!(open.as_mut().poll(cx).is_pending());
18557 module_rx.poll_recv(cx)
18558 })
18559 .await
18560 .unwrap()
18561 .frame;
18562 handler
18563 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
18564 .await
18565 .unwrap();
18566 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18567 .await
18568 .unwrap()
18569 .unwrap();
18570 assert_eq!(replies.len(), 1);
18571 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
18574 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18575 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
18576 let route = rig
18577 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
18578 .await;
18579 assert!(rig.live(&route));
18580 }
18581
18582 #[tokio::test]
18583 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
18584 {
18585 let mut rig = rig_with_flow_support(false).await;
18586 rig.sync(vec![session(1)]).await;
18587 let route = rig
18588 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18589 .await;
18590 assert!(rig.live(&route));
18591 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
18592 let mut record = session(1);
18593 record.attributes.flow_id = Some("flow:7".to_string());
18594 rig.sync(vec![record]).await;
18595 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
18596 assert!(rig.live(&unscoped));
18597 assert_eq!(unscoped.stamp(), None);
18598 }
18599
18600 #[tokio::test]
18601 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
18602 let mut rig = rig().await;
18603 let mut record = session(1);
18604 record.attributes.flow_id = Some("flow:7".to_string());
18605 rig.sync(vec![record.clone()]).await;
18606 let before = rig.forwarding.published_scope_tag(OWNER, "s");
18607 let mut owner_route = rig
18608 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18609 .await;
18610 let mut carrier_route = rig
18611 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
18612 .await;
18613 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18614 rig.sync(vec![record.clone()]).await;
18615 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18616 assert!(rig.live(&owner_route) && owner_route.untouched());
18617 assert!(rig.live(&carrier_route) && carrier_route.untouched());
18618
18619 record.attributes.flow_id = Some("flow:8".to_string());
18620 rig.sync(vec![record]).await;
18621 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
18622 let before = before.unwrap();
18623 assert_eq!(after.scope_epoch, before.scope_epoch);
18624 assert!(after.version > before.version);
18625 for route in [&mut owner_route, &mut carrier_route] {
18626 assert!(!rig.live(route));
18627 assert_eq!(
18628 route.closed_reason(),
18629 RouteCloseReason::ScopeDelegationChanged
18630 );
18631 }
18632 assert!(rig.live(&unscoped) && unscoped.untouched());
18633 for target in [PLEXUS, OTHER] {
18636 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
18637 let goodbye = module_rx
18638 .try_recv()
18639 .expect("the provider sees the drain")
18640 .frame;
18641 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18642 assert!(module_rx.try_recv().is_err());
18643 }
18644 let rebound = rig
18645 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18646 .await;
18647 assert_eq!(
18648 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
18649 Some("flow:8")
18650 );
18651 }
18652
18653 #[tokio::test]
18658 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
18659 let mut rig = rig().await;
18660 rig.sync(vec![session(1)]).await;
18661 let mut cotenant = rig
18662 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18663 .await;
18664
18665 let mut changed = session(1);
18666 changed.child_owners.push(Principal::Reserved {
18667 module_id: MAGIC.to_string(),
18668 });
18669 let mut ended = None;
18670 for (code, next) in [
18671 (error_codes::SCOPE_CHANGED, vec![changed]),
18672 (error_codes::SCOPE_ENDED, Vec::new()),
18673 ] {
18674 let relayed = rig
18675 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18676 .await;
18677 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
18678 ended = Some(next.is_empty());
18679 rig.sync(next).await;
18680 rig.ack(&relayed).await;
18681 let replies = relayed.task.await.unwrap();
18682 assert_eq!(replies.len(), 1, "{replies:?}");
18683 assert_eq!(parse_error(&replies[0])["code"], code);
18684 assert_eq!(
18685 subc_protocol::error_codes::is_retryable_route_open(code),
18686 code == error_codes::SCOPE_CHANGED
18687 );
18688 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18689 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18691 let mut goodbyes = Vec::new();
18694 while let Ok(outbound) = plexus_rx.try_recv() {
18695 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
18696 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
18697 }
18698 assert!(
18699 goodbyes.contains(&(bind_channel, bind_epoch)),
18700 "{goodbyes:?}"
18701 );
18702 assert!(rig
18703 .handler
18704 .registry
18705 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
18706 .unwrap()
18707 .is_some());
18708 }
18709 assert_eq!(ended, Some(true));
18710 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
18713 }
18714
18715 #[tokio::test]
18718 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
18719 struct Case {
18720 name: &'static str,
18721 change: fn(&mut ScopeRecord),
18722 closed: [Option<RouteCloseReason>; 4],
18725 }
18726 use RouteCloseReason::*;
18727 let cases = [
18728 Case {
18729 name: "a carrier entry removed",
18730 change: |r| {
18731 r.carriers.retain(|c| {
18732 c.principal
18733 != Principal::Reserved {
18734 module_id: AFT.to_string(),
18735 }
18736 })
18737 },
18738 closed: [None, Some(ScopeCarrierRemoved), None, None],
18739 },
18740 Case {
18741 name: "a target removed from a carrier",
18742 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
18743 closed: [None, None, None, Some(ScopeCarrierRemoved)],
18744 },
18745 Case {
18746 name: "a bare carrier narrowed to targets",
18747 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
18748 closed: [None, Some(ScopeCarrierRemoved), None, None],
18749 },
18750 Case {
18751 name: "delegates turned off",
18752 change: |r| r.attributes.delegates = false,
18753 closed: [Some(ScopeDelegationChanged); 4],
18754 },
18755 Case {
18756 name: "agent_id changed",
18757 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
18758 closed: [Some(ScopeDelegationChanged); 4],
18759 },
18760 Case {
18761 name: "a carrier added, child owners changed, the record re-sent",
18762 change: |r| {
18763 r.carriers.push(carrier(MAGIC, None));
18764 r.child_owners.push(Principal::Reserved {
18765 module_id: MAGIC.to_string(),
18766 });
18767 },
18768 closed: [None; 4],
18769 },
18770 Case {
18771 name: "a target added",
18772 change: |r| {
18773 r.carriers[1]
18774 .targets
18775 .as_mut()
18776 .unwrap()
18777 .push("third".to_string())
18778 },
18779 closed: [None; 4],
18780 },
18781 Case {
18782 name: "delegates turned on",
18783 change: |r| r.attributes.delegates = true,
18784 closed: [None; 4],
18785 },
18786 ];
18787 for case in cases {
18788 let mut rig = rig().await;
18789 rig.sync(vec![session(1)]).await;
18790 let scope = || Some(rig_selector("s", Some(1)));
18791 let mut routes = [
18792 rig.bound(Some(OWNER), PLEXUS, scope()).await,
18793 rig.bound(Some(AFT), PLEXUS, scope()).await,
18794 rig.bound(Some(BROCA), PLEXUS, scope()).await,
18795 rig.bound(Some(BROCA), OTHER, scope()).await,
18796 ];
18797 let mut record = session(1);
18798 (case.change)(&mut record);
18799 rig.sync(vec![record]).await;
18800 for (index, expected) in case.closed.iter().enumerate() {
18801 let route = &mut routes[index];
18802 match expected {
18803 Some(reason) => {
18804 assert!(!rig.live(route), "{}: route {index} still live", case.name);
18805 assert_eq!(
18806 route.closed_reason(),
18807 *reason,
18808 "{}: route {index}",
18809 case.name
18810 );
18811 }
18812 None => {
18813 assert!(rig.live(route), "{}: route {index} closed", case.name);
18814 assert!(
18815 route.untouched(),
18816 "{}: route {index} was told something",
18817 case.name
18818 );
18819 }
18820 }
18821 }
18822 }
18823 }
18824
18825 #[tokio::test]
18826 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
18827 for next in [Vec::new(), vec![session(2)]] {
18829 let mut rig = rig().await;
18830 rig.sync(vec![session(1)]).await;
18831 let mut route = rig
18832 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18833 .await;
18834 rig.sync(next).await;
18835 assert!(!rig.live(&route));
18836 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
18837 }
18838
18839 let mut rig = rig().await;
18842 let mut child = session(1);
18843 child.scope_ref = "child".to_string();
18844 child.kind = ScopeKind::Worker;
18845 child.parent = Some(ScopeParent::new(
18846 Principal::Reserved {
18847 module_id: OWNER.to_string(),
18848 },
18849 "s".to_string(),
18850 1,
18851 ));
18852 rig.sync(vec![session(1), child.clone()]).await;
18853 let mut child_route = rig
18854 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
18855 .await;
18856 assert_eq!(
18857 child_route.stamp().unwrap().parent_state,
18858 Some(ParentState::Linked)
18859 );
18860 rig.sync(vec![child]).await;
18861 assert!(!rig.live(&child_route));
18862 assert_eq!(
18863 child_route.closed_reason(),
18864 RouteCloseReason::ScopeParentEnded
18865 );
18866 }
18867
18868 #[tokio::test]
18869 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
18870 ) {
18871 let mut rig = rig().await;
18872 rig.sync(vec![session(1)]).await;
18873 let mut route = rig
18874 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18875 .await;
18876 let before = rig.forwarding.published_scope_tag(OWNER, "s");
18877 rig.sync(vec![session(1)]).await;
18878 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18879 assert!(rig.live(&route) && route.untouched());
18880
18881 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
18885 .forwarding
18886 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18887 .unwrap()
18888 else {
18889 panic!("the route is bound");
18890 };
18891 binding.flow.acquire_tagged(9, false).await.unwrap();
18892 let mut widened = session(1);
18893 widened.carriers.push(carrier(MAGIC, None));
18894 rig.sync(vec![widened]).await;
18895 assert!(rig.live(&route) && route.untouched());
18896 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18897 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
18898 assert_eq!(binding.flow.in_flight(), 1);
18901 binding
18902 .flow
18903 .acquire_tagged(10, false)
18904 .await
18905 .expect("the flow is still open");
18906 }
18907
18908 #[tokio::test]
18911 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
18912 let mut rig = rig().await;
18913 rig.sync(vec![session(1)]).await;
18914 let mut on_incumbent = rig
18915 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18916 .await;
18917
18918 let (candidate, _candidate_rx) = wide_ctx(9);
18921 let registration = rig
18922 .handler
18923 .registry
18924 .register_candidate_with_control_ops(
18925 manifest(PLEXUS, PROTOCOL_VERSION),
18926 PROTOCOL_VERSION,
18927 candidate.connection_id,
18928 module_baseline_control_ops(),
18929 )
18930 .unwrap();
18931 rig.forwarding
18932 .register_candidate_module_connection(
18933 candidate.connection_id,
18934 PLEXUS.to_string(),
18935 PROTOCOL_VERSION,
18936 manifest_concurrency(®istration.manifest),
18937 candidate.egress.clone(),
18938 )
18939 .unwrap();
18940 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
18941 rig.handler
18942 .registry
18943 .promote_candidate(PLEXUS)
18944 .unwrap()
18945 .unwrap();
18946 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
18947
18948 rig.sync(Vec::new()).await;
18949 assert!(!rig.live(&on_incumbent));
18950 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
18951 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18952 let goodbye = incumbent_rx
18953 .try_recv()
18954 .expect("the superseded endpoint is told")
18955 .frame;
18956 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18957 }
18958 }
18959}
18960
18961#[cfg(test)]
18962mod concurrency_default_exposure_tests {
18963 use super::*;
18964
18965 fn hello_body(role_json: &str) -> Vec<u8> {
18966 format!(
18967 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
18968 )
18969 .into_bytes()
18970 }
18971
18972 fn manifest_from(body: &[u8]) -> ModuleManifest {
18973 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
18974 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
18975 .expect("manifest parses")
18976 }
18977
18978 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
18979
18980 #[test]
18981 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
18982 let body = hello_body(&format!(
18983 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
18984 ));
18985 let manifest = manifest_from(&body);
18986 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18989 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
18990 }
18991
18992 #[test]
18993 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
18994 let body = hello_body(&format!(
18995 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
18996 ));
18997 let manifest = manifest_from(&body);
18998 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18999 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19000 }
19001
19002 #[test]
19003 fn non_management_roles_are_never_reported() {
19004 let body = hello_body(
19005 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
19006 );
19007 let manifest = manifest_from(&body);
19008 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19009 }
19010}