1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_confirm_operations,
25 validate_hello_event_declarations, validate_hello_flow_operations,
26 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
27 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
28 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
29 },
30 peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
31 scope::{
32 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
33 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
34 SCOPE_SYNC_OP,
35 },
36 session::{
37 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
38 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
39 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
40 ROLE_VERSIONS_FIELD,
41 },
42 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
43 Priority, RouteTarget, PROTOCOL_VERSION,
44};
45use tokio::time::{timeout_at, Instant};
46use tracing::{debug, info, instrument::WithSubscriber, warn};
47
48use crate::{
49 caller_identity::CallerIdentity,
50 capability_requirements::{
51 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
52 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
53 RequirementStatus, RuntimeModule,
54 },
55 child_roster::DaemonShutdownFlag,
56 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
57 forwarding::{
58 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
59 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
60 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
61 },
62 observability::{
63 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
64 },
65 privileged_prompt::{prompt_text, PrivilegedOperation},
66 provenance::{
67 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
68 },
69 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
70 router::{daemon_error_flags, RouteCtx, RouterError},
71 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
72 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
73 stderr_tail::{CaptureState, TailEntry},
74 supervise::{
75 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
76 SupervisorHandle, SwapHelloAdmission,
77 },
78 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
79};
80
81pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
87
88const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
89const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
90const CAP_PING_PONG: &str = "ping_pong_v1";
91const CAP_SESSION_ATTACH: &str = "session_attach_v1";
92const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
93
94const SUBC_CONTROL_OPS: &[&str] = &[
95 ops::DAEMON_SHUTDOWN_INTENT,
96 ops::ACTIVITY_RECORD,
97 ops::ACTIVITY_QUERY,
98 ops::ACTIVITY_SUBSCRIBE,
99 ops::SERVER_DESCRIBE,
100 ops::CATALOG_LIST,
101 ops::ROUTE_OPEN,
102 ops::ROUTE_POLL,
103 ops::ROUTE_CLOSING,
104 ops::ROUTE_CLOSED,
105 ops::SUPERVISOR_LIST,
106 ops::SUPERVISOR_RESTART,
107 ops::SUPERVISOR_SWAP,
108 ops::SUPERVISOR_RELOAD,
109 ops::SUPERVISOR_RESCAN,
110 ops::SUPERVISOR_RELEASE_RESERVED,
111 ops::SUPERVISOR_SET_ENABLED,
112 ops::SUPERVISOR_HEALTH_PROBE,
113 ops::SUPERVISOR_HEALTH,
114 ops::SUPERVISOR_STDERR_TAIL,
115 ops::SUPERVISOR_TERMINALS,
116 ops::SUPERVISOR_ROUTES,
117 ops::SUPERVISOR_PROVENANCE,
118 ops::SUPERVISOR_SPAWN_SNAPSHOT,
119 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
120];
121
122const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
129
130mod rescan_checked;
131use rescan_checked::{RescanMode, RescanPlan};
132
133const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
134 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
135 "supervisor.live_roots",
136 SCOPE_SYNC_OP,
137 SCOPE_APPLY_OP,
138 SCOPE_DESCRIBE_OP,
139 "operator.confirm",
140];
141
142const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
146
147const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
148
149pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
157
158pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
173
174pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
192
193const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
194const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
195
196fn reload_verdict(
197 configured: &Path,
198 spawned_from: Option<&Path>,
199 image: subc_control::RunningImageAgreement,
200) -> PendingReloadVerdict {
201 let path = match spawned_from {
202 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
203 ReloadPathAgreement::Match
204 }
205 Some(spawned_from) => ReloadPathAgreement::Mismatch {
206 configured: configured.to_path_buf(),
207 spawned_from: spawned_from.to_path_buf(),
208 },
209 None => ReloadPathAgreement::Unavailable {
210 reason: if matches!(
211 image,
212 subc_control::RunningImageAgreement::Unavailable {
213 reason: subc_control::RunningImageUnavailableReason::NotRunning
214 }
215 ) {
216 ReloadPathUnavailableReason::NotRunning
217 } else {
218 ReloadPathUnavailableReason::SpawnedPathUnavailable
219 },
220 },
221 };
222 PendingReloadVerdict {
223 path,
224 image,
225 spec: None,
226 }
227}
228
229fn launch_spec_agreement(
230 configured: &crate::supervise::ModuleSpec,
231 running: bool,
232 spawned: Option<&crate::supervise::LaunchSpecDigest>,
233) -> subc_control::LaunchSpecAgreement {
234 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
235 if !running {
236 return Agreement::Unavailable {
237 reason: Reason::NotRunning,
238 };
239 }
240 match spawned {
241 None => Agreement::Unavailable {
242 reason: Reason::SpawnedSpecUnavailable,
243 },
244 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
245 Agreement::Match
246 }
247 Some(_) => Agreement::Mismatch,
248 }
249}
250
251fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
252 if configured == spawned_from {
253 return true;
254 }
255 #[cfg(windows)]
256 {
257 match (
263 std::fs::canonicalize(configured),
264 std::fs::canonicalize(spawned_from),
265 ) {
266 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
267 _ => false,
268 }
269 }
270 #[cfg(not(windows))]
271 {
272 false
273 }
274}
275
276#[derive(Clone)]
277struct DaemonProvenanceFacts {
278 build: DaemonBuildProvenance,
279 pid: Option<u32>,
280 started_at_ms: Option<u64>,
281 start_clock: Option<crate::clock::StartClock>,
282 executable_path: Option<PathBuf>,
283 executable_identity: Option<SpawnedFileIdentity>,
284 process_start_time: Option<u64>,
285 probe: ExecutableIdentityProbe,
286}
287
288impl Default for DaemonProvenanceFacts {
289 fn default() -> Self {
290 Self {
291 build: DaemonBuildProvenance {
292 build_git_sha: None,
293 build_lock_digest: None,
294 },
295 pid: None,
296 started_at_ms: None,
297 start_clock: None,
298 executable_path: None,
299 executable_identity: None,
300 process_start_time: None,
301 probe: ExecutableIdentityProbe::default(),
302 }
303 }
304}
305
306#[derive(Debug, Clone)]
307struct SupervisorRescanContext {
308 supervisor: Supervisor,
309 config_path: PathBuf,
310 configured_port: Option<u16>,
311 storage_config: Option<crate::daemon_config::StorageConfig>,
312 admission_facts_carrier_module_id: Option<String>,
313 admission_facts_targets: Option<Vec<String>>,
314 scope_authority_owners: Vec<String>,
315 privileged_operator_prompt: PrivilegedOperatorPrompt,
316 generation: Arc<std::sync::atomic::AtomicU64>,
322}
323
324const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
345 "reloading",
346 "supervisor_not_live",
347 "registration_not_active",
348 "no_forwarding_connection",
349 "relay_send_failed",
350];
351
352#[derive(Clone)]
354pub struct ControlHandler {
355 registry: Arc<Registry>,
356 activity: Option<crate::activity::ActivityLedger>,
357 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
358 forwarding: Arc<ForwardingTable>,
359 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
360 supervisor: SupervisorHandle,
361 subc_capabilities: Arc<[String]>,
362 route_bind_relay_timeout: Duration,
366 route_bind_relay_timeouts: BTreeMap<String, Duration>,
370 route_bind_breakers: RouteBindBreakers,
373 route_bind_concurrency: RouteBindConcurrency,
376 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
381 route_bind_breaker_threshold: u32,
383 route_bind_breaker_cooldown: Duration,
385 health_probe_timeout: Duration,
386 storage_config: Option<crate::daemon_config::StorageConfig>,
389 machine_id: Option<crate::machine_id::MachineId>,
393 admission_facts_carrier_module_id: Option<String>,
394 admission_facts_targets: Option<Vec<String>>,
395 scopes: Arc<RwLock<ScopeTable>>,
399 scope_deadline_added: Arc<tokio::sync::Notify>,
403 scope_authority_owners: Vec<String>,
406 privileged_operator_prompt: PrivilegedOperatorPrompt,
408 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
411 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
412 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
413 local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
414 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
415 pub(crate) operator_apps:
416 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
417 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
422 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
428 #[cfg(test)]
430 pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
431 #[cfg(test)]
432 pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
433 #[allow(dead_code)]
437 #[cfg(windows)]
438 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
439 #[allow(dead_code)]
440 #[cfg(windows)]
441 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
442 #[allow(dead_code)]
446 #[cfg(windows)]
447 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
448 #[cfg(all(test, windows))]
451 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
452 #[cfg(test)]
453 caller_lookup: Option<TestCallerLookup>,
454 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
457 rescan: Option<SupervisorRescanContext>,
458 connected_clients: ConnectedClients,
459 counters: DaemonCounters,
460 local_endpoint_unavailable: Option<String>,
463 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
464 daemon_shutdown: DaemonShutdownFlag,
469 daemon_provenance: DaemonProvenanceFacts,
470 #[cfg(test)]
471 control_dispatch_delay: Option<Duration>,
472 #[cfg(test)]
473 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
474 #[cfg(test)]
478 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
479 #[cfg(test)]
485 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
486}
487
488impl fmt::Debug for ControlHandler {
489 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
490 f.debug_struct("ControlHandler")
491 .field("registry", &self.registry)
492 .field("forwarding", &self.forwarding)
493 .field("process_liveness", &self.process_liveness.is_some())
494 .field("supervisor", &self.supervisor)
495 .field("subc_capabilities", &self.subc_capabilities)
496 .finish()
497 }
498}
499
500struct RouteOpenRequest {
501 target: RouteTarget,
502 identity: BindIdentity,
503 consumer_identity: Option<ConsumerIdentity>,
504 consumer_capabilities: Option<Vec<String>>,
505 role_versions: Option<BTreeMap<String, String>>,
506 busy_refusals: Option<bool>,
507 admission_facts: Option<serde_json::Value>,
508 scope: Option<ScopeSelector>,
509}
510
511#[derive(Clone, Copy)]
512enum RouteOpenPhase {
513 Admission,
514 ProjectRoot,
515 EgressReservation,
516 BindSend,
517 BindAckWait,
518}
519
520struct RouteOpenTimings {
524 module_id: String,
525 principal: String,
526 admission: Duration,
527 project_root: Duration,
528 egress_reservation: Duration,
529 bind_send: Duration,
530 bind_ack_wait: Duration,
531 phase: Option<RouteOpenPhase>,
532 phase_started_at: Instant,
533}
534
535impl RouteOpenTimings {
536 fn new() -> Self {
537 Self {
538 module_id: String::new(),
539 principal: "unresolved".to_string(),
541 admission: Duration::ZERO,
542 project_root: Duration::ZERO,
543 egress_reservation: Duration::ZERO,
544 bind_send: Duration::ZERO,
545 bind_ack_wait: Duration::ZERO,
546 phase: Some(RouteOpenPhase::Admission),
547 phase_started_at: Instant::now(),
548 }
549 }
550
551 fn finish(&mut self) {
552 let elapsed = self.phase_started_at.elapsed();
553 match self.phase.take() {
554 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
555 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
556 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
557 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
558 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
559 None => {}
560 }
561 }
562
563 fn enter(&mut self, phase: RouteOpenPhase) {
564 self.finish();
565 self.phase = Some(phase);
566 self.phase_started_at = Instant::now();
567 }
568}
569
570struct RouteBindReservationGuard {
571 forwarding: Arc<ForwardingTable>,
572 endpoint: ModuleEndpointId,
573 relay_corr: u64,
574 armed: bool,
575}
576
577struct ModuleControlRpcGuard {
578 forwarding: Arc<ForwardingTable>,
579 endpoint: ModuleEndpointId,
580 corr: u64,
581 armed: bool,
582}
583
584impl ModuleControlRpcGuard {
585 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
586 Self {
587 forwarding,
588 endpoint,
589 corr,
590 armed: true,
591 }
592 }
593
594 fn disarm(&mut self) {
595 self.armed = false;
596 }
597}
598
599impl Drop for ModuleControlRpcGuard {
600 fn drop(&mut self) {
601 if self.armed {
602 let _ = self
603 .forwarding
604 .cancel_module_control_rpc(self.endpoint, self.corr);
605 }
606 }
607}
608
609impl RouteBindReservationGuard {
610 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
611 Self {
612 forwarding,
613 endpoint,
614 relay_corr,
615 armed: true,
616 }
617 }
618
619 fn release_and_disarm(&mut self) {
620 if !self.armed {
621 return;
622 }
623 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
624 self.endpoint,
625 self.relay_corr,
626 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
627 ) {
628 send_goodbye_target_best_effort(
629 &self.forwarding.counters(),
630 &target,
631 "canceled route.bind",
632 );
633 }
634 self.armed = false;
635 }
636
637 fn disarm(&mut self) {
638 self.armed = false;
639 }
640}
641
642impl Drop for RouteBindReservationGuard {
643 fn drop(&mut self) {
644 self.release_and_disarm();
645 }
646}
647
648#[derive(Debug, Clone, Default)]
678pub(crate) struct RouteBindBreakers {
679 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
680}
681
682#[derive(Debug, Clone, Default)]
683pub(crate) struct RouteBindConcurrency {
684 modules: Arc<Mutex<HashMap<String, usize>>>,
685}
686
687struct RouteBindConcurrencyGuard {
688 concurrency: RouteBindConcurrency,
689 module_id: String,
690}
691
692impl RouteBindConcurrency {
693 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
696 let mut modules = self
697 .modules
698 .lock()
699 .expect("route.bind concurrency mutex poisoned");
700 let in_flight = modules.entry(module_id.to_string()).or_default();
701 if *in_flight >= limit {
702 return Err(*in_flight);
703 }
704 *in_flight += 1;
705 Ok(RouteBindConcurrencyGuard {
706 concurrency: self.clone(),
707 module_id: module_id.to_string(),
708 })
709 }
710}
711
712impl Drop for RouteBindConcurrencyGuard {
713 fn drop(&mut self) {
714 let mut modules = self
715 .concurrency
716 .modules
717 .lock()
718 .expect("route.bind concurrency mutex poisoned");
719 let remove = {
720 let in_flight = modules
721 .get_mut(&self.module_id)
722 .expect("admitted route.bind has a concurrency entry");
723 *in_flight -= 1;
724 *in_flight == 0
725 };
726 if remove {
727 modules.remove(&self.module_id);
728 }
729 }
730}
731
732#[derive(Debug, Default)]
733struct ModuleBreakerState {
734 consecutive_timeouts: u32,
736 cooldown_until: Option<Instant>,
739 probe_in_flight: Option<Arc<()>>,
744}
745
746enum RouteBindAdmission<'a> {
748 Admitted {
749 guard: RouteBindBreakerGuard<'a>,
750 probe: bool,
753 },
754 Refused {
755 consecutive_timeouts: u32,
756 retry_in: Duration,
760 probe_in_flight: bool,
761 },
762}
763
764struct RouteBindBreakerGuard<'a> {
771 breakers: RouteBindBreakers,
772 module_id: &'a str,
773 probe_token: Option<Arc<()>>,
774 settled: bool,
775}
776
777impl RouteBindBreakerGuard<'_> {
778 fn record_accepted(&mut self) -> bool {
782 self.settled = true;
783 self.breakers.record_accepted(self.module_id)
784 }
785
786 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
789 self.settled = true;
790 self.breakers.record_timeout(
791 self.module_id,
792 self.probe_token.as_ref(),
793 threshold,
794 cooldown,
795 )
796 }
797
798 fn record_inconclusive(&mut self) {
807 self.settled = true;
808 self.breakers
809 .record_inconclusive(self.module_id, self.probe_token.as_ref());
810 }
811}
812
813impl Drop for RouteBindBreakerGuard<'_> {
814 fn drop(&mut self) {
815 if !self.settled {
816 self.breakers
817 .record_inconclusive(self.module_id, self.probe_token.as_ref());
818 }
819 }
820}
821
822struct BreakerOpened {
826 consecutive_timeouts: u32,
827 reopened_after_probe: bool,
830}
831
832impl RouteBindBreakers {
833 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
834 self.modules
835 .lock()
836 .expect("route.bind breaker mutex poisoned")
837 }
838
839 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
842 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
843 probe: probe_token.is_some(),
844 guard: RouteBindBreakerGuard {
845 breakers: self.clone(),
846 module_id,
847 probe_token,
848 settled: false,
849 },
850 };
851
852 let mut modules = self.lock();
853 let Some(state) = modules.get_mut(module_id) else {
854 return admitted(None);
855 };
856 let Some(cooldown_until) = state.cooldown_until else {
857 return admitted(None);
858 };
859 if state.probe_in_flight.is_some() {
860 return RouteBindAdmission::Refused {
861 consecutive_timeouts: state.consecutive_timeouts,
862 retry_in: Duration::ZERO,
863 probe_in_flight: true,
864 };
865 }
866 let now = Instant::now();
867 if now < cooldown_until {
868 return RouteBindAdmission::Refused {
869 consecutive_timeouts: state.consecutive_timeouts,
870 retry_in: cooldown_until - now,
871 probe_in_flight: false,
872 };
873 }
874 let token = Arc::new(());
875 state.probe_in_flight = Some(Arc::clone(&token));
876 admitted(Some(token))
877 }
878
879 fn record_accepted(&self, module_id: &str) -> bool {
880 self.lock()
881 .remove(module_id)
882 .is_some_and(|state| state.cooldown_until.is_some())
883 }
884
885 fn record_timeout(
886 &self,
887 module_id: &str,
888 probe_token: Option<&Arc<()>>,
889 threshold: u32,
890 cooldown: Duration,
891 ) -> Option<BreakerOpened> {
892 let mut modules = self.lock();
893 let state = modules.entry(module_id.to_string()).or_default();
894 let was_open = state.cooldown_until.is_some();
895 let was_probe = Self::owns_probe(state, probe_token);
896 if was_probe {
897 state.probe_in_flight = None;
898 }
899 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
900 if state.consecutive_timeouts < threshold {
901 return None;
902 }
903 state.cooldown_until = Some(Instant::now() + cooldown);
904 Some(BreakerOpened {
905 consecutive_timeouts: state.consecutive_timeouts,
906 reopened_after_probe: was_open && was_probe,
907 })
908 }
909
910 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
911 state
915 .probe_in_flight
916 .as_ref()
917 .zip(token)
918 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
919 }
920
921 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
922 if let Some(state) = self.lock().get_mut(module_id) {
923 if Self::owns_probe(state, probe_token) {
924 state.probe_in_flight = None;
925 }
926 }
927 }
928
929 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
944 self.lock()
945 .remove(module_id)
946 .map(|state| state.consecutive_timeouts)
947 .filter(|discarded| *discarded > 0)
948 }
949
950 fn open_snapshot(&self) -> Option<serde_json::Value> {
957 let now = Instant::now();
958 let modules = self.lock();
959 let open = modules
960 .iter()
961 .filter_map(|(module_id, state)| {
962 let cooldown_until = state.cooldown_until?;
963 Some((
964 module_id.clone(),
965 serde_json::json!({
966 "consecutive_timeouts": state.consecutive_timeouts,
967 "cooldown_remaining_ms":
968 cooldown_until.saturating_duration_since(now).as_millis() as u64,
969 "probe_in_flight": state.probe_in_flight.is_some(),
970 }),
971 ))
972 })
973 .collect::<serde_json::Map<String, serde_json::Value>>();
974 (!open.is_empty()).then_some(serde_json::Value::Object(open))
975 }
976}
977
978impl ControlHandler {
979 pub fn new(registry: Arc<Registry>) -> Self {
980 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
981 }
982
983 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
984 let counters = forwarding.counters();
985 let route_bind_breakers = forwarding.route_bind_breakers();
989 let route_bind_concurrency = forwarding.route_bind_concurrency();
990 let route_outages = forwarding.route_outages();
991 Self {
992 registry,
993 activity: None,
994 activity_streams: Arc::new(Mutex::new(HashMap::new())),
995 forwarding,
996 process_liveness: None,
997 supervisor: SupervisorHandle::new(),
998 subc_capabilities: Arc::from([
999 CAP_MANIFEST_REGISTRATION.to_string(),
1000 CAP_CHANNEL_LIFECYCLE.to_string(),
1001 CAP_PING_PONG.to_string(),
1002 CAP_SESSION_ATTACH.to_string(),
1003 CAP_ADMISSION_FACTS_RELAY.to_string(),
1004 CAP_SCOPES_V1.to_string(),
1005 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1006 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1007 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1008 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1009 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1010 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1011 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1012 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1013 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1014 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1015 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1016 ]),
1017 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1018 route_bind_relay_timeouts: BTreeMap::new(),
1019 route_bind_breakers,
1020 route_bind_concurrency,
1021 route_outages,
1022 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1023 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1024 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1025 storage_config: None,
1026 machine_id: None,
1027 admission_facts_carrier_module_id: None,
1028 admission_facts_targets: None,
1029 scopes: Arc::new(RwLock::new(ScopeTable::new(
1030 crate::daemon_config::default_scope_authority_owners(),
1031 ))),
1032 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1033 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1034 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1035 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1036 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1037 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1038 local_connections: Arc::new(Mutex::new(HashSet::new())),
1039 operator_app_policy: Default::default(),
1040 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1041 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1042 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1043 #[cfg(test)]
1044 app_open_test_platform: None,
1045 #[cfg(test)]
1046 app_open_test_launch: None,
1047 #[cfg(windows)]
1048 windows_verify: None,
1049 #[cfg(windows)]
1050 windows_flash: None,
1051 #[cfg(windows)]
1052 windows_track: None,
1053 #[cfg(all(test, windows))]
1054 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1055 #[cfg(test)]
1056 caller_lookup: None,
1057 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1058 rescan: None,
1059 connected_clients: ConnectedClients::new(),
1060 local_endpoint_unavailable: None,
1061 counters,
1062 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1063 daemon_shutdown: DaemonShutdownFlag::default(),
1064 daemon_provenance: DaemonProvenanceFacts::default(),
1065 #[cfg(test)]
1066 control_dispatch_delay: None,
1067 #[cfg(test)]
1068 provenance_probe_override: None,
1069 #[cfg(test)]
1070 route_bind_relay_sent_hold: None,
1071 #[cfg(test)]
1072 hello_registry_to_forwarding_hold: None,
1073 }
1074 }
1075
1076 pub fn with_storage_config(
1079 mut self,
1080 storage_config: Option<crate::daemon_config::StorageConfig>,
1081 ) -> Self {
1082 self.storage_config = storage_config;
1083 self
1084 }
1085
1086 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1089 self.activity = Some(crate::activity::ActivityLedger::open(
1090 run_dir.join("activity.jsonl"),
1091 incarnation,
1092 ));
1093 self.supervisor.set_activity(self.activity.clone());
1094 self
1095 }
1096
1097 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1100 self.machine_id = machine_id;
1101 self
1102 }
1103
1104 pub fn with_admission_facts_config(
1108 mut self,
1109 carrier_module_id: Option<String>,
1110 targets: Option<Vec<String>>,
1111 ) -> Self {
1112 self.admission_facts_carrier_module_id = carrier_module_id;
1113 self.admission_facts_targets = targets;
1114 self
1115 }
1116
1117 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1121 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1122 self.scope_authority_owners = owners;
1123 self
1124 }
1125
1126 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1128 self.privileged_operator_prompt = mode;
1129 self
1130 }
1131
1132 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1135 self.route_bind_relay_timeout = timeout;
1136 self
1137 }
1138
1139 pub fn with_route_bind_relay_timeouts(
1145 mut self,
1146 timeouts: impl IntoIterator<Item = (String, Duration)>,
1147 ) -> Self {
1148 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1149 self
1150 }
1151
1152 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1158 self.route_bind_relay_timeouts
1159 .get(module_id)
1160 .copied()
1161 .unwrap_or(self.route_bind_relay_timeout)
1162 }
1163
1164 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1172 self.route_bind_breaker_threshold = threshold.max(1);
1173 self.route_bind_breaker_cooldown = cooldown;
1174 self
1175 }
1176
1177 #[cfg(test)]
1178 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1179 self.health_probe_timeout = timeout;
1180 self
1181 }
1182
1183 #[cfg(test)]
1184 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1185 self.control_dispatch_delay = Some(delay);
1186 self
1187 }
1188
1189 #[cfg(test)]
1193 pub(crate) fn with_route_bind_relay_sent_hold(
1194 mut self,
1195 hold: Arc<tokio::sync::Barrier>,
1196 ) -> Self {
1197 self.route_bind_relay_sent_hold = Some(hold);
1198 self
1199 }
1200
1201 #[cfg(test)]
1206 pub(crate) fn with_hello_registry_to_forwarding_hold(
1207 mut self,
1208 hold: Arc<std::sync::Barrier>,
1209 ) -> Self {
1210 self.hello_registry_to_forwarding_hold = Some(hold);
1211 self
1212 }
1213
1214 #[cfg(test)]
1215 fn hold_between_registry_and_forwarding(&self) {
1216 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1217 hold.wait();
1218 hold.wait();
1219 }
1220 }
1221
1222 pub fn with_process_liveness(
1223 mut self,
1224 process_liveness: Arc<dyn ModuleProcessLiveness>,
1225 ) -> Self {
1226 self.process_liveness = Some(process_liveness);
1227 self
1228 }
1229
1230 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1231 self.supervisor = supervisor;
1232 self.supervisor.set_activity(self.activity.clone());
1233 self
1234 }
1235
1236 pub fn with_daemon_provenance(
1237 mut self,
1238 pid: u32,
1239 started_at_ms: u64,
1240 executable_path: Option<PathBuf>,
1241 build_git_sha: Option<String>,
1242 build_lock_digest: Option<String>,
1243 ) -> Self {
1244 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1245 let process_start_time = process_start_time(pid);
1246 self.daemon_provenance = DaemonProvenanceFacts {
1247 build: DaemonBuildProvenance {
1248 build_git_sha,
1249 build_lock_digest,
1250 },
1251 pid: Some(pid),
1252 started_at_ms: Some(started_at_ms),
1253 start_clock: None,
1254 executable_path,
1255 executable_identity,
1256 process_start_time,
1257 probe: ExecutableIdentityProbe::default(),
1258 };
1259 self
1260 }
1261
1262 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1263 self.daemon_provenance.start_clock = Some(clock);
1264 self
1265 }
1266
1267 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1269 self.scopes
1270 .write()
1271 .unwrap_or_else(|poisoned| poisoned.into_inner())
1272 .set_wall_clock(Arc::new(clock));
1273 self
1274 }
1275
1276 #[cfg(test)]
1277 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1278 self.provenance_probe_override = Some(result);
1279 self
1280 }
1281
1282 pub fn with_capability_config(
1286 self,
1287 modules: impl IntoIterator<Item = (String, bool)>,
1288 reserved_capabilities: BTreeMap<String, String>,
1289 ) -> Self {
1290 self.capability_evaluator
1291 .configure(modules, reserved_capabilities);
1292 self
1293 }
1294
1295 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1299 self.daemon_shutdown = flag;
1300 self
1301 }
1302
1303 pub fn with_supervisor_rescan(
1304 mut self,
1305 supervisor: Supervisor,
1306 config_path: impl Into<PathBuf>,
1307 configured_port: Option<u16>,
1308 ) -> Self {
1309 self.rescan = Some(SupervisorRescanContext {
1310 supervisor,
1311 config_path: config_path.into(),
1312 configured_port,
1313 storage_config: self.storage_config.clone(),
1314 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1315 admission_facts_targets: self.admission_facts_targets.clone(),
1316 scope_authority_owners: self.scope_authority_owners.clone(),
1317 privileged_operator_prompt: self.privileged_operator_prompt,
1318 generation: Arc::default(),
1319 });
1320 self
1321 }
1322
1323 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1324 self.connected_clients = connected_clients;
1325 self
1326 }
1327
1328 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1331 self.local_endpoint_unavailable = reason;
1332 self
1333 }
1334
1335 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1336 Arc::clone(&self.forwarding)
1337 }
1338
1339 pub(crate) fn counters(&self) -> DaemonCounters {
1340 self.counters.clone()
1341 }
1342
1343 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1346 tokio::spawn(async move {
1347 loop {
1348 self.capability_evaluator
1349 .wait_for_change_or_deadline()
1350 .await;
1351 self.refresh_capability_requirements();
1352 }
1353 });
1354 }
1355
1356 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1362 tokio::spawn(async move {
1363 loop {
1364 let has_deadlines = self
1365 .scopes
1366 .read()
1367 .unwrap_or_else(|poisoned| poisoned.into_inner())
1368 .has_deadlines();
1369 if !has_deadlines {
1370 self.scope_deadline_added.notified().await;
1371 continue;
1372 }
1373 tokio::time::sleep(Duration::from_secs(1)).await;
1374 if let Err(error) = self.sweep_expired_scopes() {
1375 warn!(%error, "scope expiry sweep failed");
1376 }
1377 }
1378 });
1379 }
1380
1381 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1382 let mut table = self
1383 .scopes
1384 .write()
1385 .unwrap_or_else(|poisoned| poisoned.into_inner());
1386 let swept = table.sweep_expired();
1387 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1388 drop(table);
1389 self.close_scope_drained_routes(drained);
1390 Ok(())
1391 }
1392
1393 fn publish_scope_changes(
1394 &self,
1395 changes: &[crate::scopes::ScopeTagChange],
1396 expired: &[crate::scopes::ScopeExpired],
1397 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1398 if changes.is_empty() {
1399 return Ok(Vec::new());
1400 }
1401 let drained = self
1402 .forwarding
1403 .publish_scope_changes(changes)
1404 .map_err(RouterError::Forwarding)?;
1405 let mut counts = HashMap::new();
1406 for route in &drained {
1407 *counts
1408 .entry((
1409 route.scope.owner.as_str(),
1410 route.scope.scope_ref.as_str(),
1411 route.scope.tag.scope_epoch,
1412 ))
1413 .or_insert(0usize) += 1;
1414 }
1415 for scope in expired {
1416 let routes_closed = counts
1417 .get(&(
1418 scope.owner.as_str(),
1419 scope.scope_ref.as_str(),
1420 scope.scope_epoch,
1421 ))
1422 .copied()
1423 .unwrap_or(0);
1424 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1425 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1426 routes_closed, "scope expired");
1427 }
1428 Ok(drained)
1429 }
1430
1431 fn runtime_capability_snapshot(
1432 &self,
1433 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1434 let runtime = self
1435 .supervisor
1436 .list()
1437 .into_iter()
1438 .map(|module| {
1439 let status = module.status().map_err(|err| {
1440 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1441 })?;
1442 Ok(RuntimeModule {
1443 module_id: status.module_id,
1444 state: status.state,
1445 enabled: status.enabled,
1446 })
1447 })
1448 .collect::<Result<Vec<_>, RouterError>>()?;
1449 let registrations = self.registry.list_registered_modules().map_err(|err| {
1455 RouterError::backend(
1456 0,
1457 0,
1458 format!("failed to list capability registrations: {err}"),
1459 )
1460 })?;
1461 let registrations = registrations
1462 .into_iter()
1463 .map(|registration| RegisteredModule {
1464 module_id: registration.manifest.module_id,
1465 module_version: registration.manifest.module_version,
1466 capabilities: registration.manifest.capabilities,
1467 })
1468 .collect();
1469 Ok((runtime, registrations))
1470 }
1471
1472 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1479 let cached_registration = RegisteredModule {
1480 module_id: registration.manifest.module_id.clone(),
1481 module_version: registration.manifest.module_version.clone(),
1482 capabilities: registration.manifest.capabilities.clone(),
1483 };
1484 let change = self.capability_evaluator.record_hello(&cached_registration);
1485 log_manifest_change(&cached_registration.module_id, &change);
1486 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1487 self.enforce_capability_denies();
1488 }
1489 self.refresh_capability_requirements();
1490 }
1491
1492 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1496 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1497 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1498 self.supervisor.set_swap_promotion_observer(observer);
1499 }
1500
1501 pub fn refresh_capability_requirements(&self) {
1502 match self.runtime_capability_snapshot() {
1503 Ok((runtime, registrations)) => {
1504 log_requirement_events(
1505 self.capability_evaluator
1506 .evaluate_now(&runtime, ®istrations),
1507 &self.daemon_shutdown,
1508 );
1509 }
1510 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1511 }
1512 }
1513
1514 fn enforce_capability_denies(&self) {
1518 let registrations = match self.registry.list_registered_modules() {
1519 Ok(snapshot) => snapshot,
1520 Err(err) => {
1521 warn!(error = %err, "failed to read registrations for capability deny census");
1522 return;
1523 }
1524 };
1525 let manifests = registrations
1526 .into_iter()
1527 .map(|registration| {
1528 (
1529 registration.manifest.module_id.clone(),
1530 registration.manifest,
1531 )
1532 })
1533 .collect::<BTreeMap<_, _>>();
1534 let census = match self.forwarding.route_census(None) {
1535 Ok(census) => census,
1536 Err(err) => {
1537 warn!(error = %err, "failed to read route census for capability deny enforcement");
1538 return;
1539 }
1540 };
1541
1542 for (target_module_id, routes) in census {
1543 let Some(target_manifest) = manifests.get(&target_module_id) else {
1544 continue;
1545 };
1546 let mut closed_routes = Vec::new();
1547 let mut goodbyes = Vec::new();
1548 for route in routes {
1549 let Principal::Reserved {
1550 module_id: opening_module_id,
1551 } = &route.principal
1552 else {
1553 continue;
1554 };
1555 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1556 continue;
1557 };
1558 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1559 continue;
1560 };
1561
1562 match self.forwarding.release_client_route(
1563 route.goodbye_target.connection_id,
1564 route.goodbye_target.channel,
1565 route.goodbye_target.epoch,
1566 ) {
1567 Ok(RouteRelease::Removed(module_goodbye)) => {
1568 warn!(
1569 opening_module_id,
1570 target_module_id,
1571 capability,
1572 "force-closing route because an attested capability deny edge now matches"
1573 );
1574 goodbyes.push(module_goodbye);
1575 goodbyes.push(route.goodbye_target.clone());
1576 closed_routes.push(route);
1577 }
1578 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1579 Err(err) => warn!(
1580 opening_module_id,
1581 target_module_id,
1582 capability,
1583 error = %err,
1584 "failed to force-close capability-denied route"
1585 ),
1586 }
1587 }
1588
1589 if closed_routes.is_empty() {
1590 continue;
1591 }
1592 send_route_control_pushes(
1593 &self.forwarding,
1594 closed_routes,
1595 ClientControlPush::RouteClosed {
1596 module_id: target_module_id,
1597 channels: Vec::new(),
1598 reason: RouteCloseReason::CapabilityDenied,
1599 drained: false,
1600 abandoned: 0,
1601 excluded_subscriptions: 0,
1602 terminal: Some(false),
1603 },
1604 );
1605 self.emit_route_goodbyes(goodbyes);
1608 }
1609 }
1610
1611 fn not_ready_reason(
1617 &self,
1618 registration: &crate::registry::ModuleRegistration,
1619 ) -> Option<NotReadyReason> {
1620 if !registration.ready {
1621 return Some(NotReadyReason {
1622 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1623 capability: None,
1624 });
1625 }
1626 self.first_unprovided_required_capability(registration)
1627 .map(|capability| NotReadyReason {
1628 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1629 capability: Some(capability),
1630 })
1631 }
1632
1633 fn first_unprovided_required_capability(
1651 &self,
1652 registration: &crate::registry::ModuleRegistration,
1653 ) -> Option<String> {
1654 let required = registration
1655 .manifest
1656 .capabilities
1657 .iter()
1658 .flat_map(|declarations| declarations.requires.iter())
1659 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1660 .map(|requirement| requirement.capability.as_str())
1661 .collect::<BTreeSet<_>>();
1662 if required.is_empty() {
1663 return None;
1664 }
1665 let module_id = registration.manifest.module_id.as_str();
1666 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1667 if required
1668 .iter()
1669 .any(|capability| verdict(capability).is_none())
1670 {
1671 self.refresh_capability_requirements();
1672 }
1673 required
1674 .into_iter()
1675 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1676 .map(str::to_string)
1677 }
1678
1679 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1680 self.capability_evaluator
1681 .statuses()
1682 .into_iter()
1683 .map(capability_requirement_status)
1684 .collect()
1685 }
1686
1687 fn deregister_connection(
1694 &self,
1695 connection_id: ConnectionId,
1696 reason: RegistrationEndReason,
1697 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1698 self.registry
1699 .deregister_connection_with_reason(connection_id, reason)
1700 }
1701
1702 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1703 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1704 return None;
1705 }
1706 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1707 parse_client_control_request(&frame.body)
1708 else {
1709 return None;
1710 };
1711 Some(target_module_id(&target).to_string())
1712 }
1713
1714 pub(crate) fn route_open_capacity_refusal(
1715 &self,
1716 ctx: &RouteCtx,
1717 frame: &Frame,
1718 target_module_id: &str,
1719 in_flight: usize,
1720 limit: usize,
1721 ) -> Result<Frame, RouterError> {
1722 let caller = self.route_open_caller_from_frame(frame);
1725 self.route_open_admission_refusal_frame(
1726 ctx,
1727 &caller,
1728 frame,
1729 target_module_id,
1730 "open_admission_full",
1731 (in_flight, limit),
1732 format!(
1733 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1734 ),
1735 )
1736 }
1737
1738 fn route_open_target_capacity_refusal(
1739 &self,
1740 ctx: &RouteCtx,
1741 caller: &RouteOpenCaller,
1742 frame: &Frame,
1743 target_module_id: &str,
1744 in_flight: usize,
1745 ) -> Result<Frame, RouterError> {
1746 self.route_open_admission_refusal_frame(
1747 ctx,
1748 caller,
1749 frame,
1750 target_module_id,
1751 "target_binds_full",
1752 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1753 format!(
1754 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1755 ),
1756 )
1757 }
1758
1759 #[allow(clippy::too_many_arguments)]
1767 fn route_open_admission_refusal_frame(
1768 &self,
1769 ctx: &RouteCtx,
1770 caller: &RouteOpenCaller,
1771 frame: &Frame,
1772 target_module_id: &str,
1773 reason: &'static str,
1774 (in_flight, limit): (usize, usize),
1775 message: impl Into<String>,
1776 ) -> Result<Frame, RouterError> {
1777 let code = error_codes::TARGET_UNAVAILABLE;
1778 self.counters.increment_route_open_refused(code);
1779 info!(
1780 target: "control",
1781 code,
1782 reason,
1783 module_id = ?target_module_id,
1784 connection_id = ctx.connection_id.get(),
1785 principal = %caller.principal,
1786 scope = %caller.scope,
1787 harness = %caller.harness,
1788 in_flight,
1789 limit,
1790 "route.open refused"
1791 );
1792 control_error_frame(frame, code, message.into())
1793 }
1794
1795 #[cfg(test)]
1801 pub fn handle_control(
1802 &self,
1803 connection_id: ConnectionId,
1804 frame: Frame,
1805 ) -> Result<Vec<Frame>, RouterError> {
1806 match frame.header.ty {
1807 FrameType::Ping => Ok(vec![pong(&frame)?]),
1808 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1809 FrameType::Goodbye => self.handle_goodbye(connection_id),
1810 ty => Ok(vec![control_error_frame(
1811 &frame,
1812 "unsupported_control_frame",
1813 format!("unsupported channel-0 frame {ty:?}"),
1814 )?]),
1815 }
1816 }
1817
1818 pub async fn handle_control_frame(
1819 &self,
1820 ctx: &RouteCtx,
1821 frame: Frame,
1822 ) -> Result<Vec<Frame>, RouterError> {
1823 self.handle_control_frame_timed(ctx, frame, None).await
1824 }
1825
1826 #[cfg(any(test, feature = "test-support"))]
1827 pub fn with_test_operator_app_policy(
1828 mut self,
1829 policy: crate::operator_apps::TestOperatorAppPolicy,
1830 ) -> Self {
1831 self.operator_app_policy = policy.into();
1832 self
1833 }
1834
1835 #[cfg(all(windows, any(test, feature = "test-support")))]
1839 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
1840 self.windows_verify = Some(verify);
1841 self
1842 }
1843
1844 #[cfg(all(windows, any(test, feature = "test-support")))]
1846 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
1847 self.windows_flash = Some(flash);
1848 self
1849 }
1850
1851 #[cfg(all(windows, any(test, feature = "test-support")))]
1855 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
1856 self.windows_track = Some(track);
1857 self
1858 }
1859
1860 pub(crate) fn record_captured_peer(
1861 &self,
1862 connection: ConnectionId,
1863 peer: subc_os::local_peer::CapturedPeer,
1864 ) {
1865 self.record_observed_peer(connection, peer.observation().clone());
1866 #[cfg(windows)]
1867 if let Some(stamp) = self.stamp_windows_launch(&peer) {
1868 self.operator_apps
1869 .lock()
1870 .unwrap_or_else(|p| p.into_inner())
1871 .insert(connection, stamp);
1872 }
1873 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1874 self.operator_apps
1875 .lock()
1876 .unwrap_or_else(|p| p.into_inner())
1877 .insert(connection, stamp);
1878 }
1879 }
1880
1881 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
1882 self.local_connections
1883 .lock()
1884 .unwrap_or_else(|p| p.into_inner())
1885 .insert(connection);
1886 }
1887
1888 fn connection_operator_app(
1891 &self,
1892 stamp: Option<&crate::operator_apps::OperatorAppStamp>,
1893 ) -> Option<subc_protocol::peer::OperatorApp> {
1894 stamp
1895 .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
1896 .map(|stamp| stamp.app.clone())
1897 }
1898
1899 fn handle_connection_describe(
1900 &self,
1901 ctx: &RouteCtx,
1902 frame: Frame,
1903 ) -> Result<Vec<Frame>, RouterError> {
1904 if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
1905 return Ok(vec![control_error_frame(
1906 &frame,
1907 "invalid_control_body",
1908 "malformed connection.describe body",
1909 )?]);
1910 }
1911 let local = self
1912 .local_connections
1913 .lock()
1914 .unwrap_or_else(|p| p.into_inner())
1915 .contains(&ctx.connection_id);
1916 let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
1917 let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
1918 Ok(vec![control_response_body_frame(
1919 &frame,
1920 &subc_control::ConnectionDescribeReply::new(local, operator_app),
1921 ops::CONNECTION_DESCRIBE,
1922 )?])
1923 }
1924
1925 pub(crate) fn record_observed_peer(
1926 &self,
1927 connection: ConnectionId,
1928 peer: subc_os::local_peer::ObservedPeer,
1929 ) {
1930 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1931 let peer = match peer {
1932 OsPeer::Unobserved => ObservedPeer::unobserved(),
1933 OsPeer::Observed(peer) => {
1934 let observed = match peer.user {
1935 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1936 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1937 };
1938 observed
1939 .with_exe(
1940 peer.exe
1941 .and_then(|path| path.into_os_string().into_string().ok()),
1942 )
1943 .with_signing_id(peer.signing_id)
1944 }
1945 };
1946 self.observed_peers
1947 .lock()
1948 .unwrap_or_else(|p| p.into_inner())
1949 .insert(connection, peer);
1950 }
1951
1952 pub(crate) fn record_caller_endpoints(
1953 &self,
1954 connection: ConnectionId,
1955 endpoints: Option<(SocketAddr, SocketAddr)>,
1956 ) {
1957 let mut callers = self
1958 .caller_endpoints
1959 .lock()
1960 .unwrap_or_else(|p| p.into_inner());
1961 callers.remove(&connection);
1962 if let Some(endpoints) = endpoints {
1963 callers.insert(connection, endpoints);
1964 }
1965 }
1966
1967 #[cfg(test)]
1968 pub(crate) fn with_caller_lookup_for_test(
1969 mut self,
1970 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1971 ) -> Self {
1972 self.caller_lookup = Some(Arc::new(lookup));
1973 self
1974 }
1975
1976 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1977 let endpoints = self
1978 .caller_endpoints
1979 .lock()
1980 .unwrap_or_else(|p| p.into_inner())
1981 .get(&connection)
1982 .copied();
1983 #[cfg(test)]
1984 let lookup = self.caller_lookup.clone();
1985 tokio::task::spawn_blocking(move || {
1986 #[cfg(test)]
1987 if let Some(lookup) = lookup {
1988 return lookup(endpoints);
1989 }
1990 endpoints
1991 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1992 .unwrap_or_default()
1993 })
1994 .await
1995 .unwrap_or_default()
1996 }
1997
1998 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1999 frame.header.channel == 0
2000 && frame.header.ty == FrameType::Request
2001 && self
2002 .forwarding
2003 .module_endpoint_for_connection(ctx.connection_id)
2004 .is_ok_and(|endpoint| endpoint.is_none())
2005 && parse_client_control_request(&frame.body)
2006 .ok()
2007 .is_some_and(|request| privileged_operation(&request).is_some())
2008 }
2009
2010 pub(crate) async fn handle_control_frame_timed(
2011 &self,
2012 ctx: &RouteCtx,
2013 frame: Frame,
2014 dispatch_started_at: Option<StdInstant>,
2015 ) -> Result<Vec<Frame>, RouterError> {
2016 match frame.header.ty {
2017 FrameType::Ping => Ok(vec![pong(&frame)?]),
2018 FrameType::Hello => {
2019 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
2020 }
2021 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2022 FrameType::Cancel => {
2023 if self
2026 .forwarding
2027 .operator_confirms()
2028 .cancel(ctx.connection_id, frame.header.corr)
2029 || self
2030 .supervisor
2031 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2032 || self.cancel_activity_subscription(ctx, &frame)?
2033 {
2034 Ok(Vec::new())
2035 } else {
2036 Ok(vec![control_error_frame(
2037 &frame,
2038 "unknown_subscription",
2039 "no subscription has this correlation id",
2040 )?])
2041 }
2042 }
2043 FrameType::Request => {
2044 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2047 if op
2048 .as_ref()
2049 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2050 {
2051 return self.handle_activity_record(ctx, frame).await;
2052 }
2053 if op.as_ref().is_some_and(|probe| {
2054 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2055 }) {
2056 return self.handle_activity_read(ctx, frame).await;
2057 }
2058 if op
2059 .as_ref()
2060 .is_some_and(|probe| probe.op == "operator.confirm")
2061 {
2062 return self.handle_operator_confirm(ctx, frame);
2063 }
2064 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2065 return self
2066 .observe_control_wait(
2067 ctx,
2068 None,
2069 ops::APP_OPEN,
2070 self.handle_app_open(ctx, frame),
2071 )
2072 .await;
2073 }
2074 if op
2075 .as_ref()
2076 .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2077 {
2078 return self.handle_connection_describe(ctx, frame);
2079 }
2080 if op
2081 .as_ref()
2082 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2083 {
2084 return self.handle_scope_served_by(ctx.connection_id, frame);
2085 }
2086 if self
2087 .forwarding
2088 .module_endpoint_for_connection(ctx.connection_id)
2089 .map_err(RouterError::Forwarding)?
2090 .is_some()
2091 {
2092 if !is_known_module_request_op(&frame.body) {
2093 return Ok(vec![control_error_frame(
2094 &frame,
2095 "unsupported_control_frame",
2096 "module-originated channel-0 REQUEST is not supported",
2097 )?]);
2098 }
2099 let request = match parse_module_control_request_from_module(&frame.body) {
2100 Ok(request) => request,
2101 Err((err, ControlRequestBodyError::UnknownOp)) => {
2102 return Ok(vec![control_error_frame(
2103 &frame,
2104 "unsupported_control_frame",
2105 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2106 )?])
2107 }
2108 Err((err, ControlRequestBodyError::InvalidBody)) => {
2109 return Ok(vec![control_error_frame(
2110 &frame,
2111 "invalid_control_body",
2112 format!("malformed module control body: {err}"),
2113 )?])
2114 }
2115 };
2116 let op = module_control_request_op(&request);
2117 let corr = frame.header.corr;
2118 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2119 let result =
2120 self.handle_module_control_request(ctx.connection_id, frame, request);
2121 log_slow_control_dispatch(
2122 dispatch_started_at,
2123 op,
2124 ctx.connection_id,
2125 corr,
2126 None,
2127 );
2128 return result;
2129 }
2130
2131 if is_known_module_request_op(&frame.body) {
2132 return Ok(vec![control_error_frame(
2133 &frame,
2134 "not_registered",
2135 "catalog.update requires an active module registration owned by this connection",
2136 )?]);
2137 }
2138
2139 let request = match parse_client_control_request(&frame.body) {
2140 Ok(request) => request,
2141 Err((err, ControlRequestBodyError::UnknownOp)) => {
2142 return Ok(vec![control_error_frame(
2143 &frame,
2144 "unknown_control_op",
2145 format!("unknown client control op: {err}"),
2146 )?])
2147 }
2148 Err((err, ControlRequestBodyError::InvalidBody)) => {
2149 return Ok(vec![control_error_frame(
2150 &frame,
2151 "invalid_control_body",
2152 format!("malformed client control body: {err}"),
2153 )?])
2154 }
2155 };
2156 let op = client_control_request_op(&request);
2157 let corr = frame.header.corr;
2158 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2159 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2160 #[cfg(test)]
2161 if let Some(delay) = self.control_dispatch_delay {
2162 tokio::time::sleep(delay).await;
2163 }
2164 let result = self
2165 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2166 .await;
2167 if let Some(timings) = route_open_timings.as_mut() {
2168 timings.finish();
2169 }
2170 log_slow_control_dispatch(
2171 dispatch_started_at,
2172 op,
2173 ctx.connection_id,
2174 corr,
2175 route_open_timings.as_ref(),
2176 );
2177 result
2178 }
2179 FrameType::Push => {
2180 let Some(endpoint) = self
2181 .forwarding
2182 .module_endpoint_for_connection(ctx.connection_id)
2183 .map_err(RouterError::Forwarding)?
2184 else {
2185 return Ok(vec![control_error_frame(
2186 &frame,
2187 "unsupported_control_frame",
2188 "client-originated channel-0 PUSH is not supported",
2189 )?]);
2190 };
2191 self.handle_status_update(endpoint, frame)
2192 }
2193 FrameType::Response | FrameType::Error
2194 if self
2195 .forwarding
2196 .module_endpoint_for_connection(ctx.connection_id)
2197 .map_err(RouterError::Forwarding)?
2198 .is_some() =>
2199 {
2200 self.handle_module_relay_response(ctx.connection_id, frame)
2201 }
2202 ty => Ok(vec![control_error_frame(
2203 &frame,
2204 "unsupported_control_frame",
2205 format!("unsupported channel-0 frame {ty:?}"),
2206 )?]),
2207 }
2208 }
2209
2210 pub fn cleanup_connection(
2211 &self,
2212 connection_id: ConnectionId,
2213 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2214 self.local_connections
2215 .lock()
2216 .unwrap_or_else(|p| p.into_inner())
2217 .remove(&connection_id);
2218 self.caller_endpoints
2219 .lock()
2220 .unwrap_or_else(|p| p.into_inner())
2221 .remove(&connection_id);
2222 self.build_class_warned
2223 .lock()
2224 .unwrap_or_else(|p| p.into_inner())
2225 .remove(&connection_id);
2226 self.observed_peers
2227 .lock()
2228 .unwrap_or_else(|p| p.into_inner())
2229 .remove(&connection_id);
2230 self.operator_apps
2231 .lock()
2232 .unwrap_or_else(|p| p.into_inner())
2233 .remove(&connection_id);
2234 self.forwarding
2235 .operator_confirms()
2236 .connection_closed(connection_id);
2237 self.cleanup_connection_with_end_reason(
2238 connection_id,
2239 RegistrationEndReason::ConnectionClosed,
2240 )
2241 }
2242
2243 fn cleanup_connection_with_end_reason(
2244 &self,
2245 connection_id: ConnectionId,
2246 requested_reason: RegistrationEndReason,
2247 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2248 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2249 self.registry
2250 .get_module_by_connection(connection_id)?
2251 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2252 .and_then(|module| module.registration_end_reason().ok().flatten())
2253 .unwrap_or(requested_reason)
2254 } else {
2255 requested_reason
2256 };
2257 let crash_closed = self
2258 .registry
2259 .get_module_by_connection(connection_id)?
2260 .and_then(|registration| {
2261 self.forwarding
2262 .module_endpoint_for_connection(connection_id)
2263 .ok()
2264 .flatten()
2265 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2266 .map(|routes| (registration.manifest.module_id, routes))
2267 });
2268 let crash_closed = crash_closed.map(|(module_id, routes)| {
2269 let terminal = match self.supervisor.get(&module_id) {
2270 None => false,
2271 Some(module) => match module.will_recover_after_connection_loss() {
2272 Ok(will_recover) => !will_recover,
2273 Err(err) => {
2274 warn!(
2275 %module_id,
2276 error = %err,
2277 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2278 );
2279 false
2280 }
2281 },
2282 };
2283 let (reason, drained) = match self
2287 .forwarding
2288 .daemon_drain_result_for_connection(connection_id)
2289 {
2290 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2291 Ok(None) => (RouteCloseReason::Crash, false),
2292 Err(err) => {
2293 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2294 (RouteCloseReason::Crash, false)
2295 }
2296 };
2297 (module_id, routes, reason, drained, terminal)
2298 });
2299 let registrations = self.deregister_connection(connection_id, end_reason);
2300 let cleanup = if crash_closed.is_some() {
2301 self.forwarding.cleanup_connection_counted(connection_id)
2302 } else {
2303 self.forwarding
2309 .cleanup_connection(connection_id)
2310 .map(|released| crate::forwarding::ConnectionCleanup {
2311 released,
2312 abandoned_relays: 0,
2313 })
2314 };
2315 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2320 let abandoned = cleanup
2321 .as_ref()
2322 .map_or(0, |cleanup| cleanup.abandoned_relays);
2323 send_route_control_pushes(
2324 &self.forwarding,
2325 routes,
2326 ClientControlPush::RouteClosed {
2327 module_id,
2328 channels: Vec::new(),
2329 reason,
2330 drained,
2331 abandoned,
2332 excluded_subscriptions: 0,
2333 terminal: Some(terminal),
2334 },
2335 );
2336 }
2337 if let Ok(cleanup) = cleanup {
2338 self.emit_route_goodbyes(cleanup.released);
2339 }
2340 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2344 crate::supervise::notify_registration_release();
2345 self.capability_evaluator.wake_deadline_loop();
2346 self.refresh_capability_requirements();
2347 }
2348 self.supervisor.remove_spawn_subscribers(connection_id);
2349 self.activity_streams
2350 .lock()
2351 .unwrap_or_else(|p| p.into_inner())
2352 .retain(|(id, _), task| {
2353 if *id == connection_id {
2354 task.abort();
2355 false
2356 } else {
2357 true
2358 }
2359 });
2360 self.hello_launch_nonces
2363 .lock()
2364 .unwrap_or_else(|poisoned| poisoned.into_inner())
2365 .forget(connection_id);
2366 self.scopes
2367 .write()
2368 .unwrap_or_else(|poisoned| poisoned.into_inner())
2369 .release_connection(connection_id);
2370 registrations
2371 }
2372
2373 pub(crate) fn handle_route_goodbye(
2374 &self,
2375 connection_id: ConnectionId,
2376 route_channel: u16,
2377 route_epoch: u32,
2378 ) -> Result<bool, RouterError> {
2379 debug!(
2380 connection_id = connection_id.get(),
2381 route_channel, route_epoch, "handling route GOODBYE"
2382 );
2383 let RouteRelease::Removed(released_route) = self
2384 .forwarding
2385 .release_client_route(connection_id, route_channel, route_epoch)
2386 .map_err(RouterError::Forwarding)?
2387 else {
2388 return Ok(false);
2389 };
2390 self.emit_route_goodbyes(vec![released_route]);
2391 Ok(true)
2392 }
2393
2394 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2395 for released in released_routes {
2396 let frame = match Frame::build_with_version(
2397 released.negotiated_ver,
2398 FrameType::Goodbye,
2399 control_flags(),
2400 released.channel,
2401 released.epoch,
2402 0,
2403 Vec::new(),
2404 ) {
2405 Ok(frame) => frame,
2406 Err(err) => {
2407 warn!(
2408 route_channel = released.channel,
2409 error = %err,
2410 "failed to build route GOODBYE frame"
2411 );
2412 continue;
2413 }
2414 };
2415 if !released.close_on_delivery_failure() {
2416 crate::forwarding::send_module_route_goodbye(
2417 &self.counters,
2418 &released.sink,
2419 frame,
2420 released.module_id.as_deref(),
2421 "client route released",
2422 );
2423 continue;
2424 }
2425 if let Err(err) = released.sink.try_send(frame) {
2426 warn!(
2427 target_connection_id = released.connection_id.get(),
2428 route_channel = released.channel,
2429 error = %err,
2430 "route GOODBYE was not delivered to client; closing target connection"
2431 );
2432 if self
2433 .forwarding
2434 .escalate_client_delivery_failure(
2435 released.connection_id,
2436 released.channel,
2437 released.epoch,
2438 CloseReason::new(
2439 "route_goodbye_delivery_failed",
2440 format!(
2441 "failed to enqueue route GOODBYE for channel {}: {err}",
2442 released.channel
2443 ),
2444 ),
2445 crate::forwarding::UndeliveredFrame {
2446 module_id: released.module_id.as_deref(),
2447 sink: &released.sink,
2448 },
2449 )
2450 .unwrap_or(false)
2451 {
2452 self.counters.increment_goodbye_relay_client_failed();
2453 }
2454 }
2455 }
2456 }
2457
2458 fn send_abandoned_route_bind_goodbye(
2470 &self,
2471 module_sink: &crate::FrameSink,
2472 negotiated_ver: u8,
2473 module_channel: u16,
2474 module_epoch: u32,
2475 ) {
2476 let frame = match Frame::build_with_version(
2477 negotiated_ver,
2478 FrameType::Goodbye,
2479 control_flags(),
2480 module_channel,
2481 module_epoch,
2482 0,
2483 Vec::new(),
2484 ) {
2485 Ok(frame) => frame,
2486 Err(err) => {
2487 warn!(
2488 route_channel = module_channel,
2489 error = %err,
2490 "failed to build GOODBYE for abandoned route.bind"
2491 );
2492 return;
2493 }
2494 };
2495 crate::forwarding::send_module_route_goodbye(
2496 &self.counters,
2497 module_sink,
2498 frame,
2499 None,
2500 "abandoned route.bind",
2501 );
2502 }
2503
2504 fn handle_hello(
2505 &self,
2506 connection_id: ConnectionId,
2507 sink: Option<crate::FrameSink>,
2508 frame: Frame,
2509 ) -> Result<Vec<Frame>, RouterError> {
2510 debug!(
2511 connection_id = connection_id.get(),
2512 corr = frame.header.corr,
2513 "handling HELLO"
2514 );
2515 if self
2519 .registry
2520 .get_module_by_connection(connection_id)
2521 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2522 .is_some()
2523 {
2524 return Ok(vec![control_error_frame(
2525 &frame,
2526 "invalid_hello",
2527 "connection is already registered as a module",
2528 )?]);
2529 }
2530 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2531 Ok(value) => value,
2532 Err(err) => {
2533 return Ok(vec![control_error_frame(
2534 &frame,
2535 "invalid_hello",
2536 format!("malformed HELLO body: {err}"),
2537 )?])
2538 }
2539 };
2540 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2541 return Ok(vec![control_error_frame(
2542 &frame,
2543 "invalid_capability_grammar",
2544 err.to_string(),
2545 )?]);
2546 }
2547 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2548 return Ok(vec![control_error_frame(
2549 &frame,
2550 "invalid_manifest",
2551 err.to_string(),
2552 )?]);
2553 }
2554 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2555 return Ok(vec![control_error_frame(
2556 &frame,
2557 "invalid_event_declaration",
2558 err.to_string(),
2559 )?]);
2560 }
2561 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2562 return Ok(vec![control_error_frame(
2563 &frame,
2564 "invalid_manifest",
2565 err.to_string(),
2566 )?]);
2567 }
2568 if let Some(provenance) = hello_value
2569 .get("manifest")
2570 .and_then(|manifest| manifest.get("provenance"))
2571 {
2572 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2573 return Ok(vec![control_error_frame(
2574 &frame,
2575 "invalid_manifest",
2576 format!("malformed manifest provenance: {err}"),
2577 )?]);
2578 }
2579 }
2580 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2581 let module_id = hello_value["manifest"]["module_id"]
2582 .as_str()
2583 .unwrap_or("<unknown>");
2584 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2585 }
2586 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2587 let module_id = hello_value["manifest"]["module_id"]
2588 .as_str()
2589 .unwrap_or("<unknown>");
2590 warn!(module_id, %reason, "dropping flow_operations from HELLO");
2591 }
2592 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2593 let module_id = hello_value["manifest"]["module_id"]
2594 .as_str()
2595 .unwrap_or("<unknown>");
2596 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2597 }
2598 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2599 Ok(hello) => hello,
2600 Err(err) => {
2601 return Ok(vec![control_error_frame(
2602 &frame,
2603 "invalid_hello",
2604 format!("malformed HELLO body: {err}"),
2605 )?])
2606 }
2607 };
2608
2609 if hello.protocol_ver != hello.manifest.protocol_ver {
2610 return Ok(vec![control_error_frame(
2611 &frame,
2612 "invalid_manifest",
2613 format!(
2614 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2615 hello.protocol_ver, hello.manifest.protocol_ver
2616 ),
2617 )?]);
2618 }
2619
2620 if hello.manifest.module_id.trim().is_empty() {
2621 return Ok(vec![control_error_frame(
2622 &frame,
2623 "invalid_manifest",
2624 "manifest module_id must not be empty",
2625 )?]);
2626 }
2627
2628 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2629 Ok(negotiated_ver) => negotiated_ver,
2630 Err(message) => {
2631 return Ok(vec![control_error_frame(
2632 &frame,
2633 "version_unsupported",
2634 message,
2635 )?])
2636 }
2637 };
2638
2639 let swap_admission = self
2647 .supervisor
2648 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2649 if swap_admission == SwapHelloAdmission::Refused {
2650 warn!(
2651 module_id = %hello.manifest.module_id,
2652 connection_id = connection_id.get(),
2653 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2654 );
2655 return Ok(vec![control_error_frame(
2656 &frame,
2657 "swap_token_invalid",
2658 format!(
2659 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2660 hello.manifest.module_id
2661 ),
2662 )?]);
2663 }
2664 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2665
2666 let nonce_verified = if swap_candidate {
2671 true
2672 } else {
2673 match self.supervisor.reserved_hello_verification(
2674 &hello.manifest.module_id,
2675 hello.launch_nonce.as_deref(),
2676 ) {
2677 Ok(verified) => verified,
2678 Err(rejection) => {
2679 let message = match rejection {
2680 ReservedHelloRejection::Exact { module_id } => format!(
2681 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2682 ),
2683 ReservedHelloRejection::Prefix {
2684 prefix,
2685 owner_module_id,
2686 } => format!(
2687 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2688 hello.manifest.module_id
2689 ),
2690 };
2691 return Ok(vec![control_error_frame(
2692 &frame,
2693 "reserved_module",
2694 message,
2695 )?]);
2696 }
2697 }
2698 };
2699
2700 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2701 &hello.manifest.module_id,
2702 hello.manifest.capabilities.as_ref(),
2703 );
2704 if let Some(refusal) = reserved_capability_refusals.first() {
2705 let capability = refusal.capability.clone();
2706 let bound_module = refusal.claimants[0].clone();
2707 log_duplicate_claim_events(reserved_capability_refusals);
2708 return Ok(vec![control_error_frame(
2709 &frame,
2710 "reserved_capability",
2711 format!(
2712 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2713 capability, bound_module, hello.manifest.module_id
2714 ),
2715 )?]);
2716 }
2717
2718 if self
2721 .forwarding
2722 .connection_has_client_routes(connection_id)
2723 .map_err(RouterError::Forwarding)?
2724 {
2725 return Ok(vec![control_error_frame(
2726 &frame,
2727 "invalid_hello",
2728 "connection has open client routes and cannot also register as a module",
2729 )?]);
2730 }
2731
2732 self.hello_launch_nonces
2737 .lock()
2738 .unwrap_or_else(|poisoned| poisoned.into_inner())
2739 .record(connection_id, hello.launch_nonce.as_deref());
2740 let control_ops = effective_module_control_ops(hello.control_ops);
2741 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2744 if swap_candidate {
2745 return self.register_swap_candidate(
2746 connection_id,
2747 sink,
2748 &frame,
2749 hello.manifest,
2750 negotiated_ver,
2751 control_ops,
2752 hello_ack,
2753 );
2754 }
2755 let registration = match self
2762 .registry
2763 .register_unpublished_verified_with_control_ops(
2764 hello.manifest,
2765 negotiated_ver,
2766 connection_id,
2767 control_ops,
2768 nonce_verified,
2769 ) {
2770 Ok(registration) => registration,
2771 Err(RegistryError::DuplicateModuleId { module_id }) => {
2772 return Ok(vec![control_error_frame(
2773 &frame,
2774 "duplicate_module_id",
2775 format!(
2776 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2777 ),
2778 )?])
2779 }
2780 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2781 return Ok(vec![control_error_frame(
2782 &frame,
2783 "invalid_module_id",
2784 err.to_string(),
2785 )?])
2786 }
2787 Err(err) => {
2788 return Ok(vec![control_error_frame(
2789 &frame,
2790 "registry_error",
2791 err.to_string(),
2792 )?])
2793 }
2794 };
2795 #[cfg(test)]
2796 self.hold_between_registry_and_forwarding();
2797
2798 let reply = if let Some(sink) = sink {
2799 let concurrency = manifest_concurrency(®istration.manifest);
2815 if let Err(err) = self.forwarding.register_module_connection_acked(
2816 connection_id,
2817 registration.manifest.module_id.clone(),
2818 negotiated_ver,
2819 concurrency,
2820 sink,
2821 hello_ack,
2822 ) {
2823 if matches!(
2828 self.deregister_connection(
2829 connection_id,
2830 RegistrationEndReason::RegistrationFailed,
2831 ),
2832 Ok(r) if !r.is_empty()
2833 ) {
2834 crate::supervise::notify_registration_release();
2835 }
2836 return Ok(vec![control_error_frame(
2837 &frame,
2838 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2839 "invalid_hello"
2840 } else {
2841 forwarding_error_code(&err)
2842 },
2843 err.to_string(),
2844 )?]);
2845 }
2846 Vec::new()
2847 } else {
2848 vec![hello_ack]
2851 };
2852
2853 if !self
2858 .registry
2859 .publish_registration(connection_id)
2860 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2861 {
2862 debug!(
2863 module_id = %registration.manifest.module_id,
2864 connection_id = connection_id.get(),
2865 "registration was removed before it could be published"
2866 );
2867 }
2868
2869 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2878 info!(
2879 module_id = %registration.manifest.module_id,
2880 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2881 );
2882 }
2883
2884 self.apply_registration_capabilities(®istration);
2885
2886 self.supervisor
2887 .record_activity_registration(®istration.manifest.module_id);
2888
2889 info!(
2890 module_id = %registration.manifest.module_id,
2891 module_version = %registration.manifest.module_version,
2892 negotiated_ver,
2893 routable_provider = manifest_provides_routable_role(®istration.manifest),
2894 connection_id = connection_id.get(),
2895 "module registered"
2896 );
2897
2898 Ok(reply)
2899 }
2900
2901 #[allow(clippy::too_many_arguments)]
2918 fn register_swap_candidate(
2919 &self,
2920 connection_id: ConnectionId,
2921 sink: Option<crate::FrameSink>,
2922 frame: &Frame,
2923 manifest: ModuleManifest,
2924 negotiated_ver: u8,
2925 control_ops: Vec<String>,
2926 hello_ack: Frame,
2927 ) -> Result<Vec<Frame>, RouterError> {
2928 let module_id = manifest.module_id.clone();
2929 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2930 manifest,
2931 negotiated_ver,
2932 connection_id,
2933 control_ops,
2934 ) {
2935 Ok(registration) => registration,
2936 Err(RegistryError::DuplicateModuleId { module_id }) => {
2937 return Ok(vec![control_error_frame(
2938 frame,
2939 "duplicate_module_id",
2940 format!(
2941 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2942 ),
2943 )?])
2944 }
2945 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2946 return Ok(vec![control_error_frame(
2947 frame,
2948 "invalid_module_id",
2949 err.to_string(),
2950 )?])
2951 }
2952 Err(err) => {
2953 return Ok(vec![control_error_frame(
2954 frame,
2955 "registry_error",
2956 err.to_string(),
2957 )?])
2958 }
2959 };
2960 #[cfg(test)]
2961 self.hold_between_registry_and_forwarding();
2962 let reply = if let Some(sink) = sink {
2963 let concurrency = manifest_concurrency(®istration.manifest);
2967 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2968 connection_id,
2969 module_id.clone(),
2970 negotiated_ver,
2971 concurrency,
2972 sink,
2973 hello_ack,
2974 ) {
2975 if matches!(
2976 self.deregister_connection(
2977 connection_id,
2978 RegistrationEndReason::RegistrationFailed,
2979 ),
2980 Ok(r) if !r.is_empty()
2981 ) {
2982 crate::supervise::notify_registration_release();
2983 }
2984 return Ok(vec![control_error_frame(
2985 frame,
2986 forwarding_error_code(&err),
2987 err.to_string(),
2988 )?]);
2989 }
2990 Vec::new()
2991 } else {
2992 vec![hello_ack]
2993 };
2994 if !self
2995 .registry
2996 .publish_registration(connection_id)
2997 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2998 {
2999 debug!(
3000 module_id = %module_id,
3001 connection_id = connection_id.get(),
3002 "swap candidate registration was removed before it could be published"
3003 );
3004 }
3005 self.supervisor.mark_swap_candidate_admitted(&module_id);
3006 info!(
3007 module_id = %module_id,
3008 module_version = %registration.manifest.module_version,
3009 negotiated_ver,
3010 ready = registration.ready,
3011 connection_id = connection_id.get(),
3012 "swap candidate registered; not routable until cutover"
3013 );
3014 Ok(reply)
3015 }
3016
3017 fn build_hello_ack(
3018 &self,
3019 frame: &Frame,
3020 negotiated_ver: u8,
3021 module_id: &str,
3022 ) -> Result<Frame, RouterError> {
3023 let ack = ModuleHelloAckBody {
3024 negotiated_ver,
3025 subc_ops: module_subc_ops(),
3026 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
3027 storage: self
3028 .storage_config
3029 .as_ref()
3030 .map(|cfg| cfg.descriptor_for(module_id)),
3031 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3032 };
3033 let body = serde_json::to_vec(&ack).map_err(|err| {
3034 RouterError::backend(
3035 0,
3036 frame.header.corr,
3037 format!("failed to encode HELLO_ACK: {err}"),
3038 )
3039 })?;
3040
3041 Frame::build_with_version(
3042 negotiated_ver,
3043 FrameType::HelloAck,
3044 control_flags(),
3045 0,
3046 0,
3047 frame.header.corr,
3048 body,
3049 )
3050 .map_err(RouterError::FrameBuild)
3051 }
3052
3053 async fn handle_client_control_request(
3054 &self,
3055 ctx: &RouteCtx,
3056 frame: Frame,
3057 request: ClientControlRequest,
3058 route_open_timings: &mut Option<RouteOpenTimings>,
3059 ) -> Result<Vec<Frame>, RouterError> {
3060 if let Some(operation) = privileged_operation(&request) {
3061 let caller = self.resolve_caller(ctx.connection_id).await;
3062 let op = client_control_request_op(&request);
3063 let text = prompt_text(operation, &caller);
3064 match self.privileged_operator_prompt {
3065 PrivilegedOperatorPrompt::Log => {
3066 info!(target: "subc_daemon::privileged_prompt", operation = op,
3067 target = ?operation, caller = %caller.description(), summary = %text,
3068 "privileged request would prompt the operator");
3069 }
3070 PrivilegedOperatorPrompt::Enforce => {
3071 let key =
3072 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
3073 let stamp = self
3074 .operator_apps
3075 .lock()
3076 .unwrap_or_else(|p| p.into_inner())
3077 .get(&ctx.connection_id)
3078 .cloned();
3079 let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
3080 &self.operator_app_policy,
3081 &self.launch_table,
3082 stamp.as_ref(),
3083 )
3084 .then(|| self.shortcut_context(stamp, None, None));
3085 let outcome = self
3086 .forwarding
3087 .operator_confirms()
3088 .approve_privileged_with_shortcut(
3089 ctx,
3090 &frame,
3091 op,
3092 text,
3093 caller.description(),
3094 key,
3095 shortcut,
3096 )
3097 .await;
3098 if outcome.code != "confirmed" {
3099 return Ok(vec![outcome.frame(&frame)]);
3100 }
3101 }
3102 }
3103 }
3104 match request {
3105 ClientControlRequest::DaemonShutdownIntent { intent } => {
3106 if intent.as_str() != Some("restart") {
3107 return Ok(vec![control_error_frame(
3108 &frame,
3109 "invalid_params",
3110 "intent must be 'restart'",
3111 )?]);
3112 }
3113 let response = ClientControlResponse::DaemonShutdownIntent {
3114 intent: "restart".to_owned(),
3115 expires_at_ms: self.supervisor.announce_daemon_restart(),
3116 };
3117 Ok(vec![control_response_body_frame(
3118 &frame,
3119 &response,
3120 "ClientControlResponse::DaemonShutdownIntent",
3121 )?])
3122 }
3123 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
3124 ClientControlRequest::CatalogList { module_id } => {
3125 self.handle_catalog_list(frame, module_id)
3126 }
3127 ClientControlRequest::RouteOpen {
3128 target,
3129 identity,
3130 consumer_identity,
3131 consumer_capabilities,
3132 role_versions,
3133 busy_refusals,
3134 admission_facts,
3135 scope,
3136 ..
3137 } => {
3138 self.handle_route_open(
3139 ctx,
3140 frame,
3141 RouteOpenRequest {
3142 target,
3143 identity,
3144 consumer_identity,
3145 consumer_capabilities,
3146 role_versions,
3147 busy_refusals,
3148 admission_facts,
3149 scope,
3150 },
3151 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
3152 )
3153 .await
3154 }
3155 ClientControlRequest::RoutePoll {
3156 route_channel,
3157 route_epoch,
3158 kind,
3159 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
3160 ClientControlRequest::SupervisorList {} => {
3161 let handler = self.clone();
3163 let sink = ctx.egress.clone();
3164 tokio::spawn(
3165 async move {
3166 let result = handler.handle_supervisor_list(frame.clone()).await;
3167 let replies = match result {
3168 Ok(replies) => replies,
3169 Err(error) => {
3170 control_error_frame(&frame, "backend_error", error.to_string())
3171 .into_iter()
3172 .collect()
3173 }
3174 };
3175 for reply in replies {
3176 if sink.send(reply).await.is_err() {
3177 break;
3178 }
3179 }
3180 }
3181 .with_current_subscriber(),
3182 );
3183 Ok(Vec::new())
3184 }
3185 ClientControlRequest::SupervisorSpawnSnapshot {} => {
3186 self.handle_supervisor_spawn_snapshot(frame)
3187 }
3188 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
3189 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
3190 }
3191 ClientControlRequest::SupervisorRestart {
3192 module_id,
3193 drain_timeout_ms,
3194 } => {
3195 self.log_supervisor_request_received(
3196 ctx,
3197 frame.header.corr,
3198 ops::SUPERVISOR_RESTART,
3199 Some(&module_id),
3200 None,
3201 )?;
3202 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3203 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
3204 .await
3205 }
3206 ClientControlRequest::SupervisorSwap {
3207 module_id,
3208 ready_timeout_ms,
3209 } => {
3210 self.log_supervisor_request_received(
3211 ctx,
3212 frame.header.corr,
3213 ops::SUPERVISOR_SWAP,
3214 Some(&module_id),
3215 None,
3216 )?;
3217 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3218 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3219 .await
3220 }
3221 ClientControlRequest::SupervisorReload { module_id } => {
3222 self.log_supervisor_request_received(
3223 ctx,
3224 frame.header.corr,
3225 ops::SUPERVISOR_RELOAD,
3226 Some(&module_id),
3227 None,
3228 )?;
3229 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3230 self.handle_supervisor_reload(frame, module_id, requester)
3231 .await
3232 }
3233 ClientControlRequest::SupervisorRescan { preview } => {
3234 if !preview {
3235 self.log_supervisor_request_received(
3236 ctx,
3237 frame.header.corr,
3238 ops::SUPERVISOR_RESCAN,
3239 None,
3240 None,
3241 )?;
3242 }
3243 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3244 let operation = self.handle_supervisor_rescan(
3245 frame,
3246 if preview {
3247 RescanMode::Preview
3248 } else {
3249 RescanMode::Execute
3250 },
3251 requester,
3252 );
3253 self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
3254 .await
3255 }
3256 ClientControlRequest::SupervisorRescanChecked {
3257 preview,
3258 candidate,
3259 expect,
3260 } => {
3261 self.observe_control_wait(
3262 ctx,
3263 None,
3264 ops::SUPERVISOR_RESCAN,
3265 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
3266 )
3267 .await
3268 }
3269 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3270 self.log_supervisor_request_received(
3271 ctx,
3272 frame.header.corr,
3273 ops::SUPERVISOR_RELEASE_RESERVED,
3274 Some(&module_id),
3275 None,
3276 )?;
3277 self.handle_supervisor_release_reserved(frame, module_id)
3278 .await
3279 }
3280 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3281 self.log_supervisor_request_received(
3282 ctx,
3283 frame.header.corr,
3284 ops::SUPERVISOR_SET_ENABLED,
3285 Some(&module_id),
3286 Some(enabled),
3287 )?;
3288 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3289 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3290 .await
3291 }
3292 ClientControlRequest::SupervisorHealthProbe { module_id } => {
3293 self.handle_supervisor_health_probe(ctx, frame, module_id)
3294 .await
3295 }
3296 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3297 ClientControlRequest::SupervisorRoutes { module_id } => {
3298 self.handle_supervisor_routes(frame, module_id)
3299 }
3300 ClientControlRequest::SupervisorProvenance { module_id } => {
3301 self.handle_supervisor_provenance(ctx, frame, module_id)
3302 .await
3303 }
3304 ClientControlRequest::SupervisorStderrTail {
3305 module_id,
3306 max_lines,
3307 max_bytes,
3308 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3309 ClientControlRequest::SupervisorTerminals { module_id } => {
3310 self.handle_supervisor_terminals(ctx, frame, module_id)
3311 .await
3312 }
3313 }
3314 }
3315
3316 fn log_supervisor_request_received(
3317 &self,
3318 ctx: &RouteCtx,
3319 corr: u64,
3320 op: &'static str,
3321 module_id: Option<&str>,
3322 enabled: Option<bool>,
3323 ) -> Result<(), RouterError> {
3324 let caller = self.supervisor_requester(ctx, corr)?;
3325 let caller = principal_label(&caller);
3326 info!(
3329 target: "control",
3330 op,
3331 module_id,
3332 enabled,
3333 connection_id = ctx.connection_id.get(),
3334 caller = %caller,
3335 "supervisor request received"
3336 );
3337 Ok(())
3338 }
3339
3340 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3341 Ok(self
3342 .registry
3343 .get_module_by_connection(ctx.connection_id)
3344 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3345 .map(|registration| Principal::Reserved {
3346 module_id: registration.manifest.module_id,
3347 })
3348 .unwrap_or(Principal::Direct))
3349 }
3350
3351 fn handle_module_control_request(
3352 &self,
3353 connection_id: ConnectionId,
3354 frame: Frame,
3355 request: ModuleControlRequestFromModule,
3356 ) -> Result<Vec<Frame>, RouterError> {
3357 match request {
3358 ModuleControlRequestFromModule::CatalogUpdate {
3359 provides,
3360 capabilities,
3361 ready,
3362 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3363 ModuleControlRequestFromModule::LiveRoots {} => {
3364 let registered = self
3365 .registry
3366 .get_module_by_connection(connection_id)
3367 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3368 let Some(registration) = registered else {
3369 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3370 };
3371 let response = self
3372 .forwarding
3373 .live_roots(®istration.manifest.module_id)
3374 .map_err(RouterError::Forwarding)?;
3375 Ok(vec![control_response_body_frame(
3376 &frame,
3377 &response,
3378 "ModuleControlResponseToModule::LiveRoots",
3379 )?])
3380 }
3381 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3382 self.handle_scope_sync(connection_id, frame, generation, scopes)
3383 }
3384 ModuleControlRequestFromModule::ScopeApply {
3385 generation,
3386 upsert,
3387 end,
3388 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3389 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3390 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3391 }
3392 }
3393 }
3394
3395 fn handle_scope_served_by(
3396 &self,
3397 connection_id: ConnectionId,
3398 frame: Frame,
3399 ) -> Result<Vec<Frame>, RouterError> {
3400 let registered = self
3401 .registry
3402 .get_module_by_connection(connection_id)
3403 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3404 let Some(registered) = registered else {
3405 return Ok(vec![control_error_frame(
3406 &frame,
3407 "not_registered",
3408 "scope.served_by requires an active module registration owned by this connection",
3409 )?]);
3410 };
3411 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3412 {
3413 Ok(request) => request,
3414 Err(err) => {
3415 return Ok(vec![control_error_frame(
3416 &frame,
3417 "invalid_control_body",
3418 format!("malformed scope.served_by body: {err}"),
3419 )?])
3420 }
3421 };
3422 let answer = self
3423 .scopes
3424 .read()
3425 .unwrap_or_else(|poisoned| poisoned.into_inner())
3426 .served_by(
3427 ®istered.manifest.module_id,
3428 &request.scope_ref,
3429 request.scope_epoch,
3430 &request.module_id,
3431 );
3432 let reply = subc_control::ScopeServedByReply::new(
3433 answer,
3434 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3435 );
3436 Ok(vec![control_response_body_frame(
3437 &frame,
3438 &reply,
3439 "ScopeServedByReply",
3440 )?])
3441 }
3442
3443 fn handle_operator_confirm(
3444 &self,
3445 ctx: &RouteCtx,
3446 frame: Frame,
3447 ) -> Result<Vec<Frame>, RouterError> {
3448 use crate::operator_confirm::{audit, Outcome};
3449 let registration = self
3450 .registry
3451 .get_module_by_connection(ctx.connection_id)
3452 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3453 let Some(registration) = registration else {
3454 let outcome = Outcome::refusal("not_registered");
3455 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3456 return Ok(vec![outcome.frame(&frame)]);
3457 };
3458 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3459 Ok(request) => request,
3460 Err(_) => {
3461 let outcome = Outcome::refusal("invalid_control_body");
3462 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3463 return Ok(vec![outcome.frame(&frame)]);
3464 }
3465 };
3466 let declared = matches!(
3467 request.kind,
3468 Some(subc_protocol::session::OperatorConfirmKind::Operation)
3469 ) && request.operation.as_ref().is_some_and(|operation| {
3470 registration
3471 .manifest
3472 .confirm_operations
3473 .as_ref()
3474 .is_some_and(|operations| operations.contains(operation))
3475 });
3476 let module_id = registration.manifest.module_id;
3477 let nonce = self
3481 .hello_launch_nonces
3482 .lock()
3483 .unwrap_or_else(|p| p.into_inner())
3484 .nonce(ctx.connection_id)
3485 .map(str::to_owned);
3486 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3487 self.supervisor
3488 .spawned_consumer_authorized(&module_id, nonce)
3489 });
3490 let confirms = self.forwarding.operator_confirms();
3491 self.forwarding
3492 .with_operator_route(
3493 ctx.connection_id,
3494 request.route_channel,
3495 request.route_epoch,
3496 |binding| {
3497 let shortcut = binding
3498 .filter(|binding| {
3499 nonce_proven
3500 && crate::operator_confirm::valid_summary(&request.summary)
3501 && declared
3502 && crate::operator_confirm::ShortcutContext::route_eligible(
3503 &self.operator_app_policy,
3504 &self.launch_table,
3505 binding,
3506 )
3507 })
3508 .map(|binding| {
3509 let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
3510 &self.operator_app_policy,
3511 binding,
3512 );
3513 let stamp = if phone.is_some() {
3514 None
3515 } else {
3516 binding.operator_app.clone()
3517 };
3518 self.shortcut_context(
3519 stamp,
3520 phone,
3521 Some((request.route_channel, request.route_epoch)),
3522 )
3523 });
3524 confirms.admit_with_shortcut(
3525 ctx,
3526 frame,
3527 module_id,
3528 nonce_proven,
3529 request,
3530 binding,
3531 shortcut,
3532 )
3533 },
3534 )
3535 .map_err(RouterError::Forwarding)
3536 }
3537
3538 fn shortcut_context(
3539 &self,
3540 stamp: Option<crate::operator_apps::OperatorAppStamp>,
3541 phone_fingerprint: Option<String>,
3542 route: Option<(u16, u32)>,
3543 ) -> crate::operator_confirm::ShortcutContext {
3544 crate::operator_confirm::ShortcutContext {
3545 ledger: self.activity.clone(),
3546 forwarding: self.forwarding.clone(),
3547 launches: self.launch_table.clone(),
3548 stamp,
3549 phone_fingerprint,
3550 route,
3551 connections: self.operator_apps.clone(),
3552 #[cfg(test)]
3553 hooks: Default::default(),
3554 }
3555 }
3556
3557 #[cfg(any(test, feature = "test-support"))]
3559 pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
3560 self.operator_app_policy.phone_enabled = enabled;
3561 self
3562 }
3563
3564 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3567 let mut scopes = ScopeRecords::from(scopes);
3568 for (index, record) in scopes.records.iter_mut().enumerate() {
3569 if let Some(refusal) = Self::canonicalize_workdir(record) {
3570 scopes.refusals.insert(index, refusal);
3571 }
3572 }
3573 scopes
3574 }
3575
3576 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3577 let workdir = record.workdir.as_mut()?;
3578 let path = std::path::Path::new(workdir);
3579 if !path.is_absolute() {
3580 return Some((
3581 error_codes::INVALID_WORKDIR,
3582 "workdir must be absolute".to_string(),
3583 ));
3584 }
3585 match ProjectRootId::from_path_allowing_missing(path) {
3586 Ok(root) => match root.as_path().to_str() {
3587 Some(canonical) => {
3588 *workdir = canonical.to_string();
3589 None
3590 }
3591 None => Some((
3592 error_codes::INVALID_WORKDIR,
3593 "canonical workdir is not UTF-8".to_string(),
3594 )),
3595 },
3596 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3597 }
3598 }
3599
3600 fn handle_scope_sync(
3604 &self,
3605 connection_id: ConnectionId,
3606 frame: Frame,
3607 generation: u64,
3608 scopes: Vec<ScopeRecord>,
3609 ) -> Result<Vec<Frame>, RouterError> {
3610 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3611 }
3612
3613 fn handle_scope_change(
3614 &self,
3615 connection_id: ConnectionId,
3616 frame: Frame,
3617 generation: u64,
3618 scopes: Vec<ScopeRecord>,
3619 end: Option<Vec<ScopeEnd>>,
3620 ) -> Result<Vec<Frame>, RouterError> {
3621 let is_apply = end.is_some();
3622 let Some(registration) = self
3623 .registry
3624 .get_module_by_connection(connection_id)
3625 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3626 else {
3627 return Ok(vec![control_error_frame(
3628 &frame,
3629 "not_registered",
3630 "scope.sync and scope.apply require an active module registration owned by this connection",
3631 )?]);
3632 };
3633 let owner = registration.manifest.module_id;
3634 let scopes = Self::prepare_scope_records(scopes);
3635 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3636 let is_current_launch = |connection: ConnectionId| {
3637 self.hello_launch_nonces
3638 .lock()
3639 .unwrap_or_else(|poisoned| poisoned.into_inner())
3640 .presented(connection, current_nonce.as_deref())
3641 };
3642 let mut table = self
3647 .scopes
3648 .write()
3649 .unwrap_or_else(|poisoned| poisoned.into_inner());
3650 let outcome = match end {
3651 Some(end) => table.apply(
3652 &owner,
3653 connection_id,
3654 is_current_launch,
3655 generation,
3656 scopes,
3657 end,
3658 ),
3659 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3660 };
3661 let drained = match &outcome {
3662 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3663 Err(_) => Vec::new(),
3664 };
3665 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3666 drop(table);
3667 if has_deadlines {
3668 self.scope_deadline_added.notify_one();
3669 }
3670 match outcome {
3671 Ok(applied) => {
3672 let counts = ScopeOutcomeCounts::of(&applied.results);
3673 info!(
3674 owner = %owner,
3675 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3676 generation,
3677 records = applied.results.len(),
3678 created = counts.created,
3679 replaced = counts.replaced,
3680 updated = counts.updated,
3681 unchanged = counts.unchanged,
3682 refused = counts.refused,
3683 ended = applied.ended.len(),
3684 tag_changes = applied.tag_changes.len(),
3685 routes_closed = drained.len(),
3686 "scope change accepted"
3687 );
3688 for refused in applied
3694 .results
3695 .iter()
3696 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3697 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3698 {
3699 warn!(
3700 owner = %owner,
3701 generation,
3702 scope_ref = %refused.scope_ref,
3703 scope_epoch = refused.scope_epoch,
3704 code = refused.code.as_deref().unwrap_or(""),
3705 "scope record refused"
3706 );
3707 }
3708 self.close_scope_drained_routes(drained);
3709 let response = if is_apply {
3710 ModuleControlResponseToModule::ScopeApply {
3711 generation,
3712 results: applied.results,
3713 end_results: applied.end_results,
3714 ended: applied.ended,
3715 }
3716 } else {
3717 ModuleControlResponseToModule::ScopeSync {
3718 generation,
3719 results: applied.results,
3720 ended: applied.ended,
3721 }
3722 };
3723 Ok(vec![control_response_body_frame(
3724 &frame,
3725 &response,
3726 "scope change response",
3727 )?])
3728 }
3729 Err(refusal) => {
3730 info!(
3731 owner = %owner,
3732 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3733 generation,
3734 code = refusal.code,
3735 "scope change refused"
3736 );
3737 Ok(vec![control_error_frame(
3738 &frame,
3739 refusal.code,
3740 refusal.message,
3741 )?])
3742 }
3743 }
3744 }
3745
3746 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3752 if drained.is_empty() {
3753 return;
3754 }
3755 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3756 BTreeMap::new();
3757 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3758 for route in drained {
3759 warn!(
3760 module_id = %route.module_id,
3761 reason = ?route.reason,
3762 client_connection_id = route.client.connection_id.get(),
3763 route_channel = route.client.channel,
3764 "closing route because its scope changed"
3765 );
3766 pushes
3767 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3768 .or_insert_with(|| (route.reason, Vec::new()))
3769 .1
3770 .push(EndpointRoute {
3771 goodbye_target: route.client.clone(),
3772 principal: Principal::Unverified,
3773 bound_at: Instant::now(),
3774 draining: false,
3775 drain_reason: None,
3776 });
3777 goodbyes.push(route.module);
3778 goodbyes.push(route.client);
3779 }
3780 for ((module_id, _), (reason, routes)) in pushes {
3781 send_route_control_pushes(
3782 &self.forwarding,
3783 routes,
3784 ClientControlPush::RouteClosed {
3785 module_id,
3786 channels: Vec::new(),
3787 reason,
3788 drained: false,
3789 abandoned: 0,
3790 excluded_subscriptions: 0,
3791 terminal: Some(false),
3792 },
3793 );
3794 }
3795 self.emit_route_goodbyes(goodbyes);
3796 }
3797
3798 fn handle_scope_describe(
3801 &self,
3802 connection_id: ConnectionId,
3803 frame: Frame,
3804 owner: Principal,
3805 scope_ref: String,
3806 ) -> Result<Vec<Frame>, RouterError> {
3807 let registered = self
3808 .registry
3809 .get_module_by_connection(connection_id)
3810 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3811 let Some(registered) = registered else {
3812 return Ok(vec![control_error_frame(
3813 &frame,
3814 "not_registered",
3815 "scope.describe requires an active module registration owned by this connection",
3816 )?]);
3817 };
3818 let mut description = self
3819 .scopes
3820 .read()
3821 .unwrap_or_else(|poisoned| poisoned.into_inner())
3822 .describe(&owner, &scope_ref);
3823 if !registered
3826 .manifest
3827 .capabilities
3828 .as_ref()
3829 .is_some_and(|caps| {
3830 caps.provides
3831 .iter()
3832 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3833 })
3834 {
3835 if let Some(stamp) = description.stamp.as_mut() {
3836 stamp.attributes.on_behalf_of_agent = None;
3837 }
3838 }
3839 if !registered
3840 .manifest
3841 .capabilities
3842 .as_ref()
3843 .is_some_and(|caps| {
3844 caps.provides
3845 .iter()
3846 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3847 })
3848 {
3849 if let Some(stamp) = description.stamp.as_mut() {
3850 stamp.attributes.flow_version = None;
3851 stamp.attributes.flow_code_hash = None;
3852 stamp.attributes.flow_install_id = None;
3853 }
3854 }
3855 let owner_configured = match &owner {
3856 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3861 _ => false,
3862 };
3863 let response = ModuleControlResponseToModule::ScopeDescribe {
3864 status: description.status,
3865 scope_epoch: description.scope_epoch,
3866 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3867 owner_synced: description.owner_synced,
3868 owner_configured,
3869 scope: description.stamp,
3870 };
3871 Ok(vec![control_response_body_frame(
3872 &frame,
3873 &response,
3874 "ModuleControlResponseToModule::ScopeDescribe",
3875 )?])
3876 }
3877
3878 fn handle_catalog_update(
3879 &self,
3880 connection_id: ConnectionId,
3881 frame: Frame,
3882 provides: Vec<ProviderRole>,
3883 capabilities: Option<CapabilityDeclarations>,
3884 ready: Option<bool>,
3885 ) -> Result<Vec<Frame>, RouterError> {
3886 self.refresh_capability_requirements();
3887 let Some(registration) = self
3888 .registry
3889 .get_module_by_connection(connection_id)
3890 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3891 else {
3892 return Ok(vec![control_error_frame(
3893 &frame,
3894 "not_registered",
3895 "catalog.update requires an active module registration owned by this connection",
3896 )?]);
3897 };
3898
3899 if let Some(message) =
3900 catalog_update_frozen_field_message(®istration.manifest, &provides)
3901 {
3902 return Ok(vec![control_error_frame(
3903 &frame,
3904 "catalog_update_frozen_field",
3905 message,
3906 )?]);
3907 }
3908
3909 let mut candidate = registration.manifest.clone();
3910 candidate.provides = provides.clone();
3911 candidate.capabilities = capabilities
3912 .clone()
3913 .or_else(|| registration.manifest.capabilities.clone());
3914 if let Some(rich_ui) = &candidate.rich_ui {
3915 if let Err(err) = rich_ui.validate() {
3916 return Ok(vec![control_error_frame(
3917 &frame,
3918 "invalid_manifest",
3919 err.to_string(),
3920 )?]);
3921 }
3922 }
3923 if let Err(err) = candidate.validate_capability_grammar() {
3924 return Ok(vec![control_error_frame(
3925 &frame,
3926 "invalid_capability_grammar",
3927 err.to_string(),
3928 )?]);
3929 }
3930
3931 let mut conflicts = self
3934 .capability_evaluator
3935 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3936 if let Some(conflict) = conflicts.first() {
3937 let message = format!(
3938 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3939 conflict.capability, conflict.claimants[0], candidate.module_id
3940 );
3941 for conflict in &mut conflicts {
3942 conflict.source = DuplicateClaimSource::CatalogUpdate;
3943 }
3944 log_duplicate_claim_events(conflicts);
3945 return Ok(vec![control_error_frame(
3946 &frame,
3947 "reserved_capability",
3948 message,
3949 )?]);
3950 }
3951
3952 let updated = self
3953 .registry
3954 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3955 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3956 if updated.is_none() {
3957 return Ok(vec![control_error_frame(
3958 &frame,
3959 "not_registered",
3960 "catalog.update requires an active module registration owned by this connection",
3961 )?]);
3962 }
3963 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3964 log_duplicate_claim_events(
3965 self.capability_evaluator
3966 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3967 );
3968 }
3969 if capability_census_trigger(
3970 registration.manifest.capabilities.as_ref(),
3971 updated
3972 .as_ref()
3973 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3974 ) {
3975 self.enforce_capability_denies();
3976 }
3977 self.refresh_capability_requirements();
3978
3979 let response = ModuleControlResponseToModule::CatalogUpdate {};
3980 control_response_body_frame(
3981 &frame,
3982 &response,
3983 "ModuleControlResponseToModule::CatalogUpdate",
3984 )
3985 .map(|frame| vec![frame])
3986 }
3987
3988 pub(crate) fn with_activity_ledger_config(
3989 mut self,
3990 run_dir: &Path,
3991 incarnation: String,
3992 config: crate::activity::ActivityConfig,
3993 ) -> Self {
3994 self.activity = Some(crate::activity::ActivityLedger::open_config(
3995 run_dir.join("activity.jsonl"),
3996 incarnation,
3997 config,
3998 ));
3999 self.supervisor.set_activity(self.activity.clone());
4000 self
4001 }
4002
4003 pub(crate) async fn configure_activity(
4004 &self,
4005 config: crate::activity::ActivityConfig,
4006 ) -> Result<(), &'static str> {
4007 match &self.activity {
4008 Some(ledger) => ledger.configure(config).await,
4009 None => Ok(()),
4010 }
4011 }
4012
4013 fn cancel_activity_subscription(
4014 &self,
4015 ctx: &RouteCtx,
4016 frame: &Frame,
4017 ) -> Result<bool, RouterError> {
4018 let task = self
4019 .activity_streams
4020 .lock()
4021 .unwrap_or_else(|p| p.into_inner())
4022 .remove(&(ctx.connection_id, frame.header.corr));
4023 let Some(task) = task else {
4024 return Ok(false);
4025 };
4026 task.abort();
4027 let end = Frame::build_with_version(
4028 response_version(frame),
4029 FrameType::StreamEnd,
4030 control_flags(),
4031 0,
4032 0,
4033 frame.header.corr,
4034 Vec::new(),
4035 )
4036 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
4037 let sink = ctx.egress.clone();
4038 tokio::spawn(async move {
4039 let _ = sink.send(end).await;
4040 });
4041 Ok(true)
4042 }
4043
4044 async fn handle_activity_read(
4045 &self,
4046 ctx: &RouteCtx,
4047 frame: Frame,
4048 ) -> Result<Vec<Frame>, RouterError> {
4049 use subc_control::{
4050 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
4051 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
4052 };
4053 #[derive(Deserialize)]
4054 struct Identity {
4055 #[serde(default)]
4056 consumer_identity: Option<ConsumerIdentity>,
4057 }
4058 let principal = match self
4062 .forwarding
4063 .module_id_for_connection(ctx.connection_id)
4064 .map_err(RouterError::Forwarding)?
4065 {
4066 Some(module_id) => Ok(Principal::Reserved { module_id }),
4067 None => {
4068 let identity: Identity = match serde_json::from_slice(&frame.body) {
4069 Ok(identity) => identity,
4070 Err(_) => {
4071 return Ok(vec![control_error_frame(
4072 &frame,
4073 ACTIVITY_MALFORMED,
4074 "malformed activity read",
4075 )?])
4076 }
4077 };
4078 self.route_open_principal(&frame, identity.consumer_identity)?
4079 }
4080 };
4081 let admitted = matches!(principal, Ok(Principal::Direct))
4082 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
4083 if !admitted {
4084 return Ok(vec![control_error_frame(
4085 &frame,
4086 ACTIVITY_READ_FORBIDDEN,
4087 "activity reads require an operator or trusted mirror principal",
4088 )?]);
4089 }
4090 let Some(ledger) = &self.activity else {
4091 return Ok(vec![control_error_frame(
4092 &frame,
4093 ACTIVITY_UNAVAILABLE,
4094 "activity ledger unavailable",
4095 )?]);
4096 };
4097 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
4098 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
4099 {
4100 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
4101 Ok(request) => ledger.query(request).await,
4102 Err(_) => Err(ACTIVITY_MALFORMED),
4103 };
4104 return match result {
4105 Ok(reply) => Ok(vec![control_response_body_frame(
4106 &frame,
4107 &reply,
4108 ops::ACTIVITY_QUERY,
4109 )?]),
4110 Err(code) => Ok(vec![control_error_frame(
4111 &frame,
4112 code,
4113 "activity query refused",
4114 )?]),
4115 };
4116 }
4117 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
4118 Ok(request) => request,
4119 Err(_) => {
4120 return Ok(vec![control_error_frame(
4121 &frame,
4122 ACTIVITY_MALFORMED,
4123 "malformed activity subscription",
4124 )?])
4125 }
4126 };
4127 let mut subscription = match ledger.subscribe(request.since).await {
4128 Ok(subscription) => subscription,
4129 Err(code) => {
4130 return Ok(vec![control_error_frame(
4131 &frame,
4132 code,
4133 "activity subscription refused",
4134 )?])
4135 }
4136 };
4137 let key = (ctx.connection_id, frame.header.corr);
4138 let sink = ctx.egress.clone();
4139 let mut streams = self
4140 .activity_streams
4141 .lock()
4142 .unwrap_or_else(|p| p.into_inner());
4143 streams.retain(|_, task| !task.is_finished());
4144 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
4145 return Ok(vec![control_error_frame(
4146 &frame,
4147 "duplicate_subscription",
4148 "activity subscription correlation id already in use",
4149 )?]);
4150 }
4151 let task = tokio::spawn(async move {
4152 loop {
4153 let entry = if let Some(entry) = subscription.replay.pop_front() {
4154 Some(entry)
4155 } else {
4156 subscription.receiver.recv().await
4157 };
4158 let Some(entry) = entry else {
4159 break;
4160 };
4161 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
4162 return;
4163 };
4164 let Ok(next) = Frame::build_with_version(
4165 response_version(&frame),
4166 FrameType::StreamData,
4167 control_flags(),
4168 0,
4169 0,
4170 frame.header.corr,
4171 body,
4172 ) else {
4173 return;
4174 };
4175 if sink.send(next).await.is_err() {
4176 return;
4177 }
4178 }
4179 if let Some(code) = subscription.terminal_code() {
4182 if let Ok(end) =
4183 control_error_frame(&frame, code, "activity subscriber fell behind")
4184 {
4185 let _ = sink.send(end).await;
4186 }
4187 }
4188 });
4189 streams.insert(key, task.abort_handle());
4190 Ok(Vec::new())
4191 }
4192
4193 async fn handle_activity_record(
4194 &self,
4195 ctx: &RouteCtx,
4196 frame: Frame,
4197 ) -> Result<Vec<Frame>, RouterError> {
4198 use subc_control::{
4199 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
4200 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
4201 };
4202 let result = async {
4203 let module = self
4204 .forwarding
4205 .module_id_for_connection(ctx.connection_id)
4206 .ok()
4207 .flatten()
4208 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
4209 let request: ActivityRecordRequest =
4210 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
4211 crate::activity::validate(&module, &request)?;
4212 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
4213 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
4214 let at_ms = std::time::SystemTime::now()
4215 .duration_since(std::time::UNIX_EPOCH)
4216 .unwrap_or_default()
4217 .as_millis() as u64;
4218 ledger.record(module, request, actor, at_ms).await
4219 }
4220 .await;
4221 match result {
4222 Ok(entry) => Ok(vec![control_response_body_frame(
4223 &frame,
4224 &ActivityRecordResponse { seq: entry.seq },
4225 "activity.record",
4226 )?]),
4227 Err(code) => Ok(vec![control_error_frame(
4228 &frame,
4229 code,
4230 "activity recording refused",
4231 )?]),
4232 }
4233 }
4234
4235 fn activity_actor(
4236 &self,
4237 connection: ConnectionId,
4238 module: &str,
4239 on: Option<&subc_control::ActivityOn>,
4240 ) -> Result<subc_control::ActivityActor, &'static str> {
4241 use crate::forwarding::{DataRoute, DataRouteState};
4242 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
4243 let principal = |id: &str| Principal::Reserved {
4244 module_id: id.to_owned(),
4245 };
4246 let mut actor = ActivityActor {
4247 principal: principal(module),
4248 scope_owner: None,
4249 agent_id: None,
4250 run_id: None,
4251 flow_id: None,
4252 project_root: None,
4253 workdir: None,
4254 project_id: None,
4255 };
4256 let scope = match on {
4257 None => return Ok(actor),
4258 Some(ActivityOn::Scope(scope)) => {
4259 let selector = subc_protocol::scope::ScopeSelector::new(
4260 principal(module),
4261 scope.scope_ref.clone(),
4262 Some(scope.epoch),
4263 );
4264 self.scopes
4265 .read()
4266 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4267 .attribute(&principal(module), module, &selector, true)
4268 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4269 }
4270 Some(ActivityOn::Route(route)) => {
4271 let DataRoute::Module(DataRouteState::Bound(binding)) = self
4272 .forwarding
4273 .lookup_data_route(connection, route.channel, route.epoch)
4274 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4275 else {
4276 return Err(ACTIVITY_ACTOR_UNKNOWN);
4277 };
4278 if binding.module_id != module {
4279 return Err(ACTIVITY_ACTOR_UNKNOWN);
4280 }
4281 actor.principal = binding.principal.clone();
4282 actor.project_root = binding
4283 .project_root
4284 .as_ref()
4285 .map(|root| root.as_path().to_path_buf());
4286 let Some(bound) = &binding.scope else {
4287 return Ok(actor);
4288 };
4289 let owner = principal(&bound.owner);
4290 let selector = subc_protocol::scope::ScopeSelector::new(
4291 owner.clone(),
4292 bound.scope_ref.clone(),
4293 Some(bound.tag.scope_epoch),
4294 );
4295 let admitted = self
4296 .scopes
4297 .read()
4298 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4299 .attribute(&owner, module, &selector, true)
4300 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4301 if admitted.tag != bound.tag {
4302 return Err(ACTIVITY_ACTOR_UNKNOWN);
4303 }
4304 admitted
4305 }
4306 };
4307 actor.scope_owner = Some(scope.stamp.owner);
4308 actor.agent_id = scope.stamp.attributes.agent_id;
4309 actor.run_id = scope.stamp.attributes.run_id;
4310 actor.flow_id = scope.stamp.attributes.flow_id;
4311 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4312 actor.project_id = scope.stamp.project_id;
4313 Ok(actor)
4314 }
4315
4316 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4317 self.refresh_capability_requirements();
4318 let mut counters = self.counters.snapshot();
4324 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4325 obj.insert(
4326 "activity_unparseable_lines".into(),
4327 ledger.read_errors().into(),
4328 );
4329 obj.insert(
4330 "activity_lifecycle_append_failures".into(),
4331 ledger.lifecycle_failures().into(),
4332 );
4333 obj.insert(
4334 "activity_started_digest_skips".into(),
4335 ledger.lifecycle_digest_skips().into(),
4336 );
4337 }
4338 if let (Ok((connections_with_routes, max)), Some(obj)) = (
4339 self.forwarding.client_route_concentration(),
4340 counters.as_object_mut(),
4341 ) {
4342 obj.insert(
4343 "client_connections_with_routes".into(),
4344 connections_with_routes.into(),
4345 );
4346 obj.insert("max_routes_on_one_connection".into(), max.into());
4347 }
4348 if let (Some(open_breakers), Some(obj)) = (
4354 self.route_bind_breakers.open_snapshot(),
4355 counters.as_object_mut(),
4356 ) {
4357 obj.insert("route_bind_breakers_open".into(), open_breakers);
4358 }
4359 if let (Some(reason), Some(obj)) = (
4364 self.local_endpoint_unavailable.clone(),
4365 counters.as_object_mut(),
4366 ) {
4367 obj.insert("local_endpoint_unavailable".into(), reason.into());
4368 }
4369 let response = ClientControlResponse::ServerDescribe {
4370 protocol_ver: PROTOCOL_VERSION,
4371 subc_ops: subc_ops(),
4372 capabilities: self.describe_capabilities(),
4373 connected_clients: self.connected_clients.count(),
4374 counters: Some(counters),
4375 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4376 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4377 capability_requirements: self.capability_requirement_statuses(),
4378 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4379 config_path: self
4380 .rescan
4381 .as_ref()
4382 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4383 };
4384 Ok(vec![control_response_body_frame(
4385 &frame,
4386 &response,
4387 "ClientControlResponse::ServerDescribe",
4388 )?])
4389 }
4390
4391 fn handle_catalog_list(
4392 &self,
4393 frame: Frame,
4394 module_id: Option<String>,
4395 ) -> Result<Vec<Frame>, RouterError> {
4396 let (generation, modules) = self
4397 .registry
4398 .list_modules_with_nonce_verification()
4399 .map_err(|err| {
4400 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4401 })?;
4402 let entries = modules
4403 .into_iter()
4404 .filter(|(registration, _)| {
4405 module_id
4406 .as_deref()
4407 .map(|wanted| registration.manifest.module_id == wanted)
4408 .unwrap_or(true)
4409 })
4410 .map(|(registration, nonce_verified)| {
4411 let not_ready = self.not_ready_reason(®istration);
4412 let roles = registration.manifest.provides;
4413 CatalogEntry::new(
4414 registration.manifest.module_id,
4415 roles,
4416 registration.control_ops,
4417 )
4418 .with_ready(not_ready.is_none())
4419 .with_not_ready(not_ready)
4420 .with_module_version(Some(registration.manifest.module_version))
4421 .with_provenance(registration.manifest.provenance)
4422 .with_capabilities(registration.manifest.capabilities)
4423 .with_self_signals(registration.manifest.self_signals)
4424 .with_events(registration.manifest.events)
4425 .with_operation_delivery(registration.manifest.operation_delivery)
4426 .with_operation_payload(registration.manifest.operation_payload)
4427 .with_flow_operations(registration.manifest.flow_operations)
4428 .with_confirm_operations(registration.manifest.confirm_operations)
4429 .with_rich_ui(registration.manifest.rich_ui)
4430 .with_role_declarations(registration.manifest.role_declarations)
4431 .with_nonce_verified(nonce_verified)
4432 })
4433 .collect();
4434 let response = ClientControlResponse::CatalogList {
4435 generation,
4436 modules: entries,
4437 subc_ops: subc_ops(),
4438 };
4439 Ok(vec![control_response_body_frame(
4440 &frame,
4441 &response,
4442 "ClientControlResponse::CatalogList",
4443 )?])
4444 }
4445
4446 fn attest_route_open_consumer(
4451 &self,
4452 consumer_identity: Option<&ConsumerIdentity>,
4453 ) -> Option<Principal> {
4454 let Some(consumer_identity) = consumer_identity else {
4455 return Some(Principal::Direct);
4456 };
4457 self.supervisor
4458 .spawned_consumer_authorized(
4459 &consumer_identity.module_id,
4460 &consumer_identity.launch_nonce,
4461 )
4462 .then(|| Principal::Reserved {
4463 module_id: consumer_identity.module_id.clone(),
4464 })
4465 }
4466
4467 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4471 let Ok(ClientControlRequest::RouteOpen {
4472 identity,
4473 consumer_identity,
4474 scope,
4475 ..
4476 }) = parse_client_control_request(&frame.body)
4477 else {
4478 return RouteOpenCaller::unknown();
4479 };
4480 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4481 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4482 }
4483
4484 fn route_open_principal(
4485 &self,
4486 frame: &Frame,
4487 consumer_identity: Option<ConsumerIdentity>,
4488 ) -> Result<Result<Principal, Frame>, RouterError> {
4489 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4490 self.route_open_principal_attested(frame, consumer_identity, attested)
4491 }
4492
4493 fn route_open_principal_attested(
4497 &self,
4498 frame: &Frame,
4499 consumer_identity: Option<ConsumerIdentity>,
4500 attested: Option<Principal>,
4501 ) -> Result<Result<Principal, Frame>, RouterError> {
4502 if let Some(principal) = attested {
4503 return Ok(Ok(principal));
4504 }
4505 let Some(consumer_identity) = consumer_identity else {
4506 return Ok(Ok(Principal::Direct));
4507 };
4508
4509 Ok(Err(control_error_frame(
4510 frame,
4511 "bad_consumer_identity",
4512 format!(
4513 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4514 consumer_identity.module_id
4515 ),
4516 )?))
4517 }
4518
4519 #[allow(clippy::too_many_arguments)]
4526 fn route_open_refusal_frame(
4527 &self,
4528 ctx: &RouteCtx,
4529 caller: &RouteOpenCaller,
4530 frame: &Frame,
4531 module_id: &str,
4532 reason: &'static str,
4533 code: &'static str,
4534 message: impl Into<String>,
4535 ) -> Result<Frame, RouterError> {
4536 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4537 control_error_frame(frame, code, message.into())
4538 }
4539
4540 #[allow(clippy::too_many_arguments)]
4549 fn route_bind_rejection(
4550 &self,
4551 ctx: &RouteCtx,
4552 caller: &RouteOpenCaller,
4553 frame: &Frame,
4554 target_module_id: &str,
4555 body: ErrorBody,
4556 daemon_authored: bool,
4557 ) -> Result<Vec<Frame>, RouterError> {
4558 if daemon_authored {
4564 let scope_code = match body.code.as_str() {
4565 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4566 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4567 _ => None,
4568 };
4569 if let Some(code) = scope_code {
4570 self.observe_route_open_refusal(
4571 ctx,
4572 caller,
4573 target_module_id,
4574 "scope_changed_before_commit",
4575 code,
4576 );
4577 return Ok(vec![control_error_body_frame(frame, body)?]);
4578 }
4579 }
4580 self.counters
4581 .increment_route_open_refused("module_rejected");
4582 info!(
4583 target: "control",
4584 code = "module_rejected",
4585 module_code = ?body.code,
4586 module_id = ?target_module_id,
4587 connection_id = ctx.connection_id.get(),
4588 principal = %caller.principal,
4589 scope = %caller.scope,
4590 harness = %caller.harness,
4591 "route.open refused"
4592 );
4593 let refusal = if daemon_authored {
4594 control_error_body_frame(frame, body)?
4595 } else {
4596 control_relayed_error_body_frame(frame, body)?
4597 };
4598 Ok(vec![refusal])
4599 }
4600
4601 #[allow(clippy::too_many_arguments)]
4619 fn route_open_breaker_refusal_frame(
4620 &self,
4621 ctx: &RouteCtx,
4622 caller: &RouteOpenCaller,
4623 frame: &Frame,
4624 module_id: &str,
4625 consecutive_timeouts: u32,
4626 retry_in: Duration,
4627 probe_in_flight: bool,
4628 ) -> Result<Frame, RouterError> {
4629 self.counters
4630 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4631 debug!(
4632 target: "control",
4633 code = "module_timeout",
4634 module_id = ?module_id,
4635 connection_id = ctx.connection_id.get(),
4636 principal = %caller.principal,
4637 scope = %caller.scope,
4638 harness = %caller.harness,
4639 consecutive_timeouts,
4640 retry_in_ms = retry_in.as_millis() as u64,
4641 probe_in_flight,
4642 "route.open refused by open bind-relay breaker"
4643 );
4644 let detail = if probe_in_flight {
4651 "one test route is already being tried; retry once it settles".to_string()
4652 } else {
4653 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4654 };
4655 control_error_frame(
4656 frame,
4657 "module_timeout",
4658 format!(
4659 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4660 timed out in a row); {detail}; its established routes are unaffected"
4661 ),
4662 )
4663 }
4664
4665 fn observe_route_open_refusal(
4678 &self,
4679 ctx: &RouteCtx,
4680 caller: &RouteOpenCaller,
4681 module_id: &str,
4682 reason: &'static str,
4683 code: &'static str,
4684 ) {
4685 self.counters.increment_route_open_refused(code);
4686 info!(
4687 target: "control",
4688 code,
4689 reason,
4690 module_id = ?module_id,
4691 connection_id = ctx.connection_id.get(),
4692 principal = %caller.principal,
4693 scope = %caller.scope,
4694 harness = %caller.harness,
4695 "route.open refused"
4696 );
4697 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4698 self.route_outages.record_not_serving(module_id, reason);
4699 }
4700 }
4701
4702 fn observe_route_open_accept(
4747 &self,
4748 ctx: &RouteCtx,
4749 module_id: &str,
4750 principal: &str,
4751 scope: &str,
4752 ) {
4753 self.route_outages.record_accepted(module_id);
4754 self.counters.increment_route_open_accepted(principal);
4755 info!(
4756 target: "control",
4757 principal,
4758 module_id,
4759 connection_id = ctx.connection_id.get(),
4760 scope = %scope,
4761 "route.open accepted"
4762 );
4763 }
4764
4765 fn supervised_absent_route_open_refusal_frame(
4766 &self,
4767 ctx: &RouteCtx,
4768 caller: &RouteOpenCaller,
4769 frame: &Frame,
4770 module_id: &str,
4771 code: &'static str,
4772 status: &crate::supervise::ModuleStatus,
4773 ) -> Result<Frame, RouterError> {
4774 self.counters.increment_route_open_refused(code);
4775 info!(
4776 target: "control",
4777 code,
4778 reason = "supervised_not_registered",
4779 module_id = ?module_id,
4780 connection_id = ctx.connection_id.get(),
4781 principal = %caller.principal,
4782 scope = %caller.scope,
4783 harness = %caller.harness,
4784 state = %status.state,
4785 enabled = status.enabled,
4786 live = status.live,
4787 "route.open refused"
4788 );
4789 self.route_outages
4793 .record_not_serving(module_id, "supervised_not_registered");
4794 let message = if let Some(pid) = status.waiting_for_previous_pid {
4795 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4796 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4797 })
4798 } else {
4799 format!(
4800 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4801 status.state, status.enabled, status.live
4802 )
4803 };
4804 control_error_frame(frame, code, message)
4805 }
4806
4807 async fn handle_route_open(
4808 &self,
4809 ctx: &RouteCtx,
4810 frame: Frame,
4811 request: RouteOpenRequest,
4812 timings: &mut RouteOpenTimings,
4813 ) -> Result<Vec<Frame>, RouterError> {
4814 let RouteOpenRequest {
4815 target,
4816 mut identity,
4817 consumer_identity,
4818 consumer_capabilities,
4819 role_versions,
4820 busy_refusals,
4821 admission_facts,
4822 scope,
4823 } = request;
4824 let target_module_id = target_module_id(&target).to_string();
4825 timings.module_id.clone_from(&target_module_id);
4826 if self
4827 .registry
4828 .get_module_by_connection(ctx.connection_id)
4829 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4830 .is_some()
4831 {
4832 return Ok(vec![control_error_frame(
4833 &frame,
4834 "invalid_request",
4835 "module connections cannot open client routes",
4836 )?]);
4837 }
4838 debug!(
4839 connection_id = ctx.connection_id.get(),
4840 corr = frame.header.corr,
4841 module_id = %target_module_id,
4842 "handling route.open"
4843 );
4844
4845 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4850 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4851
4852 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4858 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4859 self.observe_route_open_refusal(
4860 ctx,
4861 &caller,
4862 &target_module_id,
4863 "invalid_role_versions",
4864 error_codes::INVALID_REQUEST,
4865 );
4866 return Ok(vec![control_error_body_frame(
4867 &frame,
4868 ErrorBody {
4869 code: error_codes::INVALID_REQUEST.to_string(),
4870 message: error.to_string(),
4871 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4872 },
4873 )?]);
4874 }
4875
4876 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4877 if status.waiting_for_previous_pid.is_some() {
4878 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4879 ctx,
4880 &caller,
4881 &frame,
4882 &target_module_id,
4883 "module_warming",
4884 &status,
4885 )?]);
4886 }
4887 }
4888
4889 let Some(registration) = self
4908 .registry
4909 .get_module(&target_module_id)
4910 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4911 else {
4912 if let Some((status, warming)) =
4913 self.supervisor_status(&target_module_id, frame.header.corr)?
4914 {
4915 if status.protocol == ModuleProtocol::None {
4922 return Ok(vec![self.route_open_refusal_frame(
4923 ctx,
4924 &caller,
4925 &frame,
4926 &target_module_id,
4927 "protocol_none",
4928 error_codes::MODULE_NO_PROTOCOL,
4929 format!(
4930 "module_id '{target_module_id}' is declared protocol: none; \
4931 it speaks no subc wire and serves no routes"
4932 ),
4933 )?]);
4934 }
4935 let code = if warming {
4936 "module_warming"
4937 } else {
4938 "target_unavailable"
4939 };
4940 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4941 ctx,
4942 &caller,
4943 &frame,
4944 &target_module_id,
4945 code,
4946 &status,
4947 )?]);
4948 }
4949 if let Some(removed_ago_ms) =
4950 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4951 {
4952 return Ok(vec![self.route_open_refusal_frame(
4953 ctx,
4954 &caller,
4955 &frame,
4956 &target_module_id,
4957 "removed",
4958 error_codes::MODULE_REMOVED,
4959 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4960 )?]);
4961 }
4962 return Ok(vec![self.route_open_refusal_frame(
4963 ctx,
4964 &caller,
4965 &frame,
4966 &target_module_id,
4967 "not_registered",
4968 error_codes::UNKNOWN_MODULE,
4969 format!("module_id '{target_module_id}' is not registered"),
4970 )?]);
4971 };
4972
4973 if !registration.ready {
4977 self.counters
4978 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4979 info!(
4980 target: "control",
4981 code = error_codes::MODULE_WARMING,
4982 module_id = ?target_module_id,
4983 connection_id = ctx.connection_id.get(),
4984 principal = %caller.principal,
4985 scope = %caller.scope,
4986 harness = %caller.harness,
4987 reason = "declared_not_ready",
4988 "route.open refused"
4989 );
4990 self.route_outages
4993 .record_not_serving(&target_module_id, "declared_not_ready");
4994 return Ok(vec![control_error_body_frame(
4995 &frame,
4996 ErrorBody {
4997 code: error_codes::MODULE_WARMING.to_string(),
4998 message: format!(
4999 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
5000 ),
5001 detail: Some(serde_json::json!({
5002 "reason": "declared_not_ready"
5003 })),
5004 },
5005 )?]);
5006 }
5007
5008 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
5027 self.counters
5028 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
5029 info!(
5030 target: "control",
5031 code = error_codes::MODULE_WARMING,
5032 module_id = ?target_module_id,
5033 connection_id = ctx.connection_id.get(),
5034 principal = %caller.principal,
5035 scope = %caller.scope,
5036 harness = %caller.harness,
5037 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5038 capability = %capability,
5039 "route.open refused"
5040 );
5041 return Ok(vec![control_error_body_frame(
5042 &frame,
5043 ErrorBody {
5044 code: error_codes::MODULE_WARMING.to_string(),
5045 message: format!(
5046 "module_id '{target_module_id}' requires capability '{capability}', \
5047 which no registered module provides; retry"
5048 ),
5049 detail: Some(serde_json::json!({
5050 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5051 "capability": capability,
5052 })),
5053 },
5054 )?]);
5055 }
5056
5057 if !target_has_required_role(&target, ®istration.manifest.provides) {
5058 return Ok(vec![self.route_open_refusal_frame(
5059 ctx,
5060 &caller,
5061 &frame,
5062 &target_module_id,
5063 "role_not_provided",
5064 "target_unavailable",
5065 format!("module_id '{target_module_id}' does not provide the requested target"),
5066 )?]);
5067 }
5068
5069 if registration.state != ChannelState::Active {
5070 return Ok(vec![self.route_open_refusal_frame(
5071 ctx,
5072 &caller,
5073 &frame,
5074 &target_module_id,
5075 "registration_not_active",
5076 "target_unavailable",
5077 format!("module_id '{target_module_id}' is not active"),
5078 )?]);
5079 }
5080
5081 if self
5082 .forwarding
5083 .module_is_draining(&target_module_id)
5084 .map_err(RouterError::Forwarding)?
5085 {
5086 return Ok(vec![self.route_open_refusal_frame(
5087 ctx,
5088 &caller,
5089 &frame,
5090 &target_module_id,
5091 "reloading",
5092 "module_reloading",
5093 format!("module_id '{target_module_id}' is reloading"),
5094 )?]);
5095 }
5096
5097 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
5098 process_liveness.process_live(&target_module_id) == Some(false)
5099 }) {
5100 if process_liveness.process_replacing(&target_module_id) {
5107 return Ok(vec![self.route_open_refusal_frame(
5108 ctx,
5109 &caller,
5110 &frame,
5111 &target_module_id,
5112 "reloading",
5113 "module_reloading",
5114 format!("module_id '{target_module_id}' is reloading"),
5115 )?]);
5116 }
5117 return Ok(vec![self.route_open_refusal_frame(
5118 ctx,
5119 &caller,
5120 &frame,
5121 &target_module_id,
5122 "supervisor_not_live",
5123 "target_unavailable",
5124 format!("module_id '{target_module_id}' is not live"),
5125 )?]);
5126 }
5127
5128 if !self
5129 .forwarding
5130 .has_live_module_connection(&target_module_id)
5131 .map_err(RouterError::Forwarding)?
5132 {
5133 return Ok(vec![self.route_open_refusal_frame(
5134 ctx,
5135 &caller,
5136 &frame,
5137 &target_module_id,
5138 "no_forwarding_connection",
5139 "target_unavailable",
5140 format!("module_id '{target_module_id}' has no live forwarding connection"),
5141 )?]);
5142 }
5143
5144 if let Some(error) =
5145 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
5146 {
5147 self.observe_route_open_refusal(
5148 ctx,
5149 &caller,
5150 &target_module_id,
5151 "op_not_allowed",
5152 "op_not_allowed",
5153 );
5154 return Ok(vec![error]);
5155 }
5156
5157 let principal =
5158 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
5159 Ok(principal) => principal,
5160 Err(error) => {
5161 self.observe_route_open_refusal(
5162 ctx,
5163 &caller,
5164 &target_module_id,
5165 "bad_consumer_identity",
5166 "bad_consumer_identity",
5167 );
5168 return Ok(vec![error]);
5169 }
5170 };
5171 timings.principal = principal_label(&principal);
5172
5173 let build_class = identity.build_class.take().filter(|build_class| {
5176 if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
5177 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
5178 } else {
5179 if self
5180 .build_class_warned
5181 .lock()
5182 .unwrap_or_else(|p| p.into_inner())
5183 .insert(ctx.connection_id)
5184 {
5185 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
5186 "ignoring build_class from a non-Callosum opener");
5187 }
5188 false
5189 }
5190 });
5191
5192 if let Principal::Reserved {
5196 module_id: opening_module_id,
5197 } = &principal
5198 {
5199 if let Some(opening_registration) = self
5200 .registry
5201 .get_module(opening_module_id)
5202 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5203 {
5204 if let Some(capability) =
5205 denied_capability(&opening_registration.manifest, ®istration.manifest)
5206 {
5207 warn!(
5208 opening_module_id,
5209 target_module_id,
5210 capability,
5211 "refusing route.open because an attested capability deny edge matches"
5212 );
5213 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5214 &frame,
5215 &target_module_id,
5216 "capability_deny_edge",
5217 "capability_forbidden",
5218 format!(
5219 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
5220 ),
5221 )?]);
5222 }
5223 }
5224 }
5225
5226 if admission_facts.is_some() {
5227 let carrier_matches = matches!(
5228 &principal,
5229 Principal::Reserved { module_id }
5230 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
5231 );
5232 if !carrier_matches {
5233 return Ok(vec![self.route_open_refusal_frame(
5234 ctx,
5235 &caller,
5236 &frame,
5237 &target_module_id,
5238 "admission_facts_carrier_not_permitted",
5239 "admission_facts_not_permitted",
5240 "admission facts may only be carried by the configured reserved module",
5241 )?]);
5242 }
5243
5244 let target_allowed = self
5245 .admission_facts_targets
5246 .as_ref()
5247 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
5248 if !target_allowed {
5249 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5250 &frame,
5251 &target_module_id,
5252 "admission_facts_target_not_listed",
5253 "admission_facts_target_not_allowed",
5254 format!(
5255 "admission facts are not permitted for target module_id '{target_module_id}'"
5256 ),
5257 )?]);
5258 }
5259
5260 }
5264
5265 let (bound_scope, mut scope_stamp) = match &scope {
5271 None => (None, None),
5272 Some(selector) => {
5273 let owner_configured = match &selector.owner {
5274 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5280 _ => false,
5281 };
5282 let mut scopes = self
5283 .scopes
5284 .write()
5285 .unwrap_or_else(|poisoned| poisoned.into_inner());
5286 let admitted = scopes
5287 .admit(&principal, &target_module_id, selector, owner_configured)
5288 .and_then(|admission| {
5289 crate::scopes::check_target_flow_support(
5290 &admission.stamp,
5291 &target_module_id,
5292 registration.manifest.capabilities.as_ref(),
5293 )?;
5294 crate::scopes::check_target_agent_run_support(
5295 &admission.stamp,
5296 &target_module_id,
5297 registration.manifest.capabilities.as_ref(),
5298 )?;
5299 let bound = scopes.reserve_bind(
5300 &admission,
5301 &principal,
5302 ctx.connection_id,
5303 selector.hold_presence,
5304 );
5305 Ok((admission, bound))
5306 });
5307 match admitted {
5308 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
5309 Err(refusal) => {
5310 return Ok(vec![self.route_open_refusal_frame(
5311 ctx,
5312 &caller,
5313 &frame,
5314 &target_module_id,
5315 refusal.code,
5316 refusal.code,
5317 refusal.message,
5318 )?]);
5319 }
5320 }
5321 }
5322 };
5323 if let Some(stamp) = &scope_stamp {
5326 caller.scope = route_open_scope_label(stamp);
5327 }
5328
5329 timings.enter(RouteOpenPhase::ProjectRoot);
5337 let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
5338 Ok(project) => project,
5339 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5340 };
5341 timings.enter(RouteOpenPhase::Admission);
5342 let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
5343 Ok(root) => root,
5344 Err(err) => {
5345 return Ok(vec![control_error_frame(
5346 &frame,
5347 "invalid_project_root",
5348 err.to_string(),
5349 )?])
5350 }
5351 };
5352 let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
5358
5359 let _concurrency_guard = match self
5370 .route_bind_concurrency
5371 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5372 {
5373 Ok(guard) => guard,
5374 Err(in_flight) => {
5375 return Ok(vec![self.route_open_target_capacity_refusal(
5376 ctx,
5377 &caller,
5378 &frame,
5379 &target_module_id,
5380 in_flight,
5381 )?]);
5382 }
5383 };
5384
5385 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5388 RouteBindAdmission::Admitted { guard, probe } => {
5389 if probe {
5390 info!(
5391 module_id = %target_module_id,
5392 connection_id = ctx.connection_id.get(),
5393 "route.bind breaker half-open: admitting one probe"
5394 );
5395 }
5396 guard
5397 }
5398 RouteBindAdmission::Refused {
5399 consecutive_timeouts,
5400 retry_in,
5401 probe_in_flight,
5402 } => {
5403 return Ok(vec![self.route_open_breaker_refusal_frame(
5404 ctx,
5405 &caller,
5406 &frame,
5407 &target_module_id,
5408 consecutive_timeouts,
5409 retry_in,
5410 probe_in_flight,
5411 )?]);
5412 }
5413 };
5414
5415 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5420 let relay_deadline = Instant::now() + route_bind_relay_timeout;
5421 let operator_app = self
5422 .operator_apps
5423 .lock()
5424 .unwrap_or_else(|p| p.into_inner())
5425 .get(&ctx.connection_id)
5426 .cloned();
5427 timings.enter(RouteOpenPhase::EgressReservation);
5428 let pending = match self
5429 .forwarding
5430 .begin_route_bind_relay_for(
5431 ctx.connection_id,
5432 ctx.egress.clone(),
5433 response_version(&frame),
5434 frame.header.corr,
5435 &target_module_id,
5436 principal.clone(),
5437 bound_scope,
5438 project_root,
5439 identity.harness.clone(),
5440 build_class,
5441 operator_app.clone(),
5442 relay_deadline,
5443 busy_refusals == Some(true),
5444 )
5445 .await
5446 {
5447 Ok(pending) => pending,
5448 Err(err) => {
5449 return Ok(vec![self.route_open_refusal_frame(
5450 ctx,
5451 &caller,
5452 &frame,
5453 &target_module_id,
5454 "relay_reservation_failed",
5455 forwarding_error_code(&err),
5456 err.to_string(),
5457 )?])
5458 }
5459 };
5460 timings.enter(RouteOpenPhase::Admission);
5461 let crate::forwarding::PendingRouteBindRelay {
5462 endpoint,
5463 module_sink,
5464 negotiated_ver,
5465 client_channel,
5466 client_epoch,
5467 module_channel,
5468 module_epoch,
5469 corr: relay_corr,
5470 receiver,
5471 } = pending;
5472 let mut reservation =
5473 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5474
5475 let relay_registration = self
5480 .registry
5481 .get_module_by_connection(endpoint.connection_id)
5482 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5483 let relay_capabilities = relay_registration
5484 .as_ref()
5485 .and_then(|registration| registration.manifest.capabilities.as_ref());
5486 if let Some(selector) = &scope {
5487 if let Err(refusal) = self
5490 .scopes
5491 .read()
5492 .unwrap_or_else(|poisoned| poisoned.into_inner())
5493 .check_bind_proof(&principal, selector)
5494 {
5495 reservation.release_and_disarm();
5496 return Ok(vec![self.route_open_refusal_frame(
5497 ctx,
5498 &caller,
5499 &frame,
5500 &target_module_id,
5501 refusal.code,
5502 refusal.code,
5503 refusal.message,
5504 )?]);
5505 }
5506 }
5507 if let Some(stamp) = scope_stamp
5508 .as_ref()
5509 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5510 {
5511 if let Err(refusal) = crate::scopes::check_target_flow_support(
5512 stamp,
5513 &target_module_id,
5514 relay_capabilities,
5515 )
5516 .and_then(|()| {
5517 crate::scopes::check_target_agent_run_support(
5518 stamp,
5519 &target_module_id,
5520 relay_capabilities,
5521 )
5522 }) {
5523 reservation.release_and_disarm();
5524 return Ok(vec![self.route_open_refusal_frame(
5525 ctx,
5526 &caller,
5527 &frame,
5528 &target_module_id,
5529 refusal.code,
5530 refusal.code,
5531 refusal.message,
5532 )?]);
5533 }
5534 }
5535
5536 if !relay_capabilities.is_some_and(|caps| {
5539 caps.provides
5540 .iter()
5541 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5542 }) {
5543 if let Some(stamp) = scope_stamp.as_mut() {
5544 stamp.workdir = None;
5545 stamp.project_id = None;
5546 }
5547 }
5548 let worker_run = scope_stamp
5553 .as_ref()
5554 .is_some_and(crate::scopes::is_worker_run)
5555 && relay_capabilities.is_some_and(|caps| {
5556 caps.provides
5557 .iter()
5558 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5559 });
5560 if !worker_run
5561 && !relay_capabilities.is_some_and(|caps| {
5562 caps.provides
5563 .iter()
5564 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5565 })
5566 {
5567 if let Some(stamp) = scope_stamp.as_mut() {
5568 stamp.attributes.on_behalf_of_agent = None;
5569 }
5570 }
5571 if !relay_capabilities.is_some_and(|caps| {
5572 caps.provides
5573 .iter()
5574 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5575 }) {
5576 if let Some(stamp) = scope_stamp.as_mut() {
5577 stamp.attributes.flow_version = None;
5578 stamp.attributes.flow_code_hash = None;
5579 stamp.attributes.flow_install_id = None;
5580 }
5581 }
5582 let call_mode = if worker_run
5583 || relay_capabilities
5584 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5585 {
5586 Some(derive_call_mode(scope_stamp.as_ref()))
5587 } else {
5588 if let Some(stamp) = scope_stamp.as_mut() {
5589 stamp.expires_at_ms = None;
5590 }
5591 None
5592 };
5593
5594 debug!(
5595 connection_id = ctx.connection_id.get(),
5596 client_channel,
5597 client_epoch,
5598 module_channel,
5599 module_epoch,
5600 "reserved route handle pair"
5601 );
5602 let route_principal_label = principal_label(&principal);
5605 let relay = ModuleControlRequest::RouteBind {
5606 route_channel: module_channel,
5607 epoch: module_epoch,
5608 target,
5609 identity,
5610 principal: Some(principal),
5611 consumer_capabilities,
5612 role_versions,
5613 admission_facts,
5614 scope: scope_stamp,
5615 call_mode,
5616 };
5617 let peer = RouteBindPeer::new()
5618 .with_observed_peer(
5619 relay_capabilities
5620 .is_some_and(|caps| {
5621 caps.provides
5622 .iter()
5623 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5624 })
5625 .then(|| {
5626 self.observed_peers
5627 .lock()
5628 .unwrap_or_else(|p| p.into_inner())
5629 .get(&ctx.connection_id)
5630 .cloned()
5631 .unwrap_or_else(ObservedPeer::unobserved)
5632 }),
5633 )
5634 .with_operator_app(
5635 self.connection_operator_app(operator_app.as_ref())
5636 .filter(|_| {
5637 relay_capabilities.is_some_and(|caps| {
5638 caps.provides
5639 .iter()
5640 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5641 })
5642 }),
5643 );
5644 let relay_body = crate::bind_compat::encode_bind(
5645 &relay,
5646 &peer,
5647 &legacy_bind_project,
5648 relay_capabilities,
5649 )
5650 .map_err(|err| {
5651 RouterError::backend(
5652 0,
5653 frame.header.corr,
5654 format!("failed to encode route.bind request: {err}"),
5655 )
5656 })?;
5657 let relay_frame = Frame::build_with_version(
5658 negotiated_ver,
5659 FrameType::Request,
5660 control_flags(),
5661 0,
5662 0,
5663 relay_corr,
5664 relay_body,
5665 )
5666 .map_err(RouterError::FrameBuild)?;
5667
5668 timings.enter(RouteOpenPhase::BindSend);
5669 let sent = module_sink.send(relay_frame).await;
5670 timings.enter(RouteOpenPhase::Admission);
5671 if let Err(err) = sent {
5672 reservation.release_and_disarm();
5673 return Ok(vec![self.route_open_refusal_frame(
5674 ctx,
5675 &caller,
5676 &frame,
5677 &target_module_id,
5678 "relay_send_failed",
5679 "target_unavailable",
5680 err.to_string(),
5681 )?]);
5682 }
5683
5684 #[cfg(test)]
5685 if let Some(hold) = &self.route_bind_relay_sent_hold {
5686 hold.wait().await;
5690 hold.wait().await;
5691 }
5692
5693 if self
5698 .forwarding
5699 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5700 .map_err(RouterError::Forwarding)?
5701 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5702 {
5703 self.send_abandoned_route_bind_goodbye(
5704 &module_sink,
5705 negotiated_ver,
5706 module_channel,
5707 module_epoch,
5708 );
5709 }
5710
5711 timings.enter(RouteOpenPhase::BindAckWait);
5712 let outcome = timeout_at(relay_deadline, receiver).await;
5713 timings.enter(RouteOpenPhase::Admission);
5714 match outcome {
5715 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5716 reservation.disarm();
5717 if breaker.record_accepted() {
5718 info!(
5719 module_id = %target_module_id,
5720 "route.bind breaker closed: the probe was accepted"
5721 );
5722 }
5723 self.observe_route_open_accept(
5724 ctx,
5725 &target_module_id,
5726 &route_principal_label,
5727 &caller.scope,
5728 );
5729 Ok(Vec::new())
5730 }
5731 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5732 reservation.release_and_disarm();
5733 breaker.record_inconclusive();
5737 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5738 }
5739 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5740 reservation.release_and_disarm();
5741 breaker.record_inconclusive();
5742 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5743 }
5744 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5745 reservation.release_and_disarm();
5746 breaker.record_inconclusive();
5747 tracing::warn!(
5754 module_id = %target_module_id,
5755 "route.bind relay abandoned: {message}"
5756 );
5757 Ok(vec![self.route_open_refusal_frame(
5758 ctx,
5759 &caller,
5760 &frame,
5761 &target_module_id,
5762 "relay_abandoned",
5763 "target_unavailable",
5764 message,
5765 )?])
5766 }
5767 Ok(Err(_)) => {
5768 reservation.release_and_disarm();
5769 breaker.record_inconclusive();
5770 Ok(vec![self.route_open_refusal_frame(
5771 ctx,
5772 &caller,
5773 &frame,
5774 &target_module_id,
5775 "relay_waiter_canceled",
5776 "target_unavailable",
5777 "route.bind relay waiter was canceled before the module responded",
5778 )?])
5779 }
5780 Err(_) => {
5781 reservation.release_and_disarm();
5782 if let Some(opened) = breaker.record_timeout(
5786 self.route_bind_breaker_threshold,
5787 self.route_bind_breaker_cooldown,
5788 ) {
5789 warn!(
5790 module_id = %target_module_id,
5791 consecutive_timeouts = opened.consecutive_timeouts,
5792 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5793 reopened_after_probe = opened.reopened_after_probe,
5794 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5795 );
5796 }
5797 tracing::warn!(
5805 module_id = %target_module_id,
5806 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5807 "route.bind relay timed out: module did not ack within budget"
5808 );
5809 Ok(vec![self.route_open_refusal_frame(
5810 ctx,
5811 &caller,
5812 &frame,
5813 &target_module_id,
5814 "relay_timed_out",
5815 "module_timeout",
5816 format!(
5817 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5818 route_bind_relay_timeout
5819 ),
5820 )?])
5821 }
5822 }
5823 }
5824
5825 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5826 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5827 snapshot: self.supervisor.spawn_snapshot(),
5828 };
5829 Ok(vec![control_response_body_frame(
5830 &frame,
5831 &response,
5832 "ClientControlResponse::SupervisorSpawnSnapshot",
5833 )?])
5834 }
5835
5836 fn handle_supervisor_spawn_subscribe(
5837 &self,
5838 ctx: &RouteCtx,
5839 frame: Frame,
5840 since: Option<SpawnCursor>,
5841 ) -> Result<Vec<Frame>, RouterError> {
5842 match self.supervisor.subscribe_spawns(
5843 ctx.connection_id,
5844 frame.header.corr,
5845 response_version(&frame),
5846 since,
5847 ctx.egress.clone(),
5848 ) {
5849 Ok(()) => Ok(Vec::new()),
5850 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5851 Ok(vec![control_error_body_frame(
5852 &frame,
5853 ErrorBody {
5854 code: "spawn_cursor_incarnation_mismatch".to_string(),
5855 message: "spawn cursor belongs to a different daemon incarnation"
5856 .to_string(),
5857 detail: Some(serde_json::json!({
5858 "current_daemon_incarnation": current
5859 })),
5860 },
5861 )?])
5862 }
5863 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5864 &frame,
5865 ErrorBody {
5866 code: "spawn_cursor_too_old".to_string(),
5867 message: "spawn cursor predates the retained event ring".to_string(),
5868 detail: Some(serde_json::json!({
5869 "oldest_retained_cursor": oldest
5870 })),
5871 },
5872 )?]),
5873 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5874 0,
5875 frame.header.corr,
5876 format!("failed to open supervisor spawn subscription: {error}"),
5877 )),
5878 }
5879 }
5880
5881 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5882 let generation = self
5883 .registry
5884 .generation()
5885 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5886 let supervised = self.supervisor.list();
5887 let targets = supervised
5888 .iter()
5889 .map(|module| module.subtree_target())
5890 .collect();
5891 let subtree = tokio::spawn(crate::subtree_resources::read(targets));
5892 let mut modules = Vec::new();
5893 for module in supervised {
5894 let status = module.status_for_control("list").map_err(|err| {
5895 RouterError::backend(
5896 0,
5897 frame.header.corr,
5898 format!("failed to read supervisor status: {err}"),
5899 )
5900 })?;
5901 let (configured, _) = module.configuration().map_err(|err| {
5902 RouterError::backend(
5903 0,
5904 frame.header.corr,
5905 format!("failed to read module configuration: {err}"),
5906 )
5907 })?;
5908 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5909 RouterError::backend(
5910 0,
5911 frame.header.corr,
5912 format!("failed to read retry time: {err}"),
5913 )
5914 })?;
5915 let image = module.running_image_agreement().await;
5917 let resources = Some(crate::subtree_resources::read_process(module.clone()).await);
5920 let mut verdict =
5921 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5922 verdict.spec = Some(launch_spec_agreement(
5923 &configured,
5924 status.process_alive,
5925 status.launch_spec_digest.as_ref(),
5926 ));
5927 let pending_reload = Some(verdict);
5928 let configured_enabled = self
5929 .capability_evaluator
5930 .configured_enabled(&status.module_id);
5931 modules.push(
5934 SupervisorEntry::new(
5935 status.module_id,
5936 status.state.to_string(),
5937 status.enabled,
5938 status.live,
5939 status.health.status,
5940 )
5941 .with_configured_enabled(configured_enabled)
5942 .with_next_retry_at_ms(next_retry_at_ms)
5943 .with_launch_nonce_env(Some(!cfg!(unix)))
5944 .with_launch_nonce_source(status.launch_nonce_source)
5945 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5946 .with_protocol(status.protocol)
5947 .with_pending_reload(pending_reload)
5948 .with_last_probe_ms(status.health.last_probe_ms)
5949 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5950 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5951 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5952 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5953 .with_restart_count(Some(status.restart_count))
5954 .with_max_restarts(Some(status.max_restarts))
5955 .with_lifetime_restarts(Some(status.lifetime_restarts))
5956 .with_spawn_generation(Some(status.spawn_generation))
5957 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5958 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5959 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5960 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5961 .with_resources(resources),
5962 );
5963 }
5964 let readings = subtree
5965 .await
5966 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5967 for (module, reading) in modules.iter_mut().zip(readings) {
5968 if reading.reason.as_deref() == Some("timeout") {
5969 crate::subtree_resources::log_timeout(&module.module_id, "subtree");
5970 }
5971 module.subtree = Some(reading);
5972 }
5973 let response = ClientControlResponse::SupervisorList {
5974 generation,
5975 modules,
5976 };
5977 Ok(vec![control_response_body_frame(
5978 &frame,
5979 &response,
5980 "ClientControlResponse::SupervisorList",
5981 )?])
5982 }
5983
5984 fn handle_supervisor_stderr_tail(
5985 &self,
5986 frame: Frame,
5987 module_id: String,
5988 max_lines: Option<u32>,
5989 max_bytes: Option<u32>,
5990 ) -> Result<Vec<Frame>, RouterError> {
5991 let Some(module) = self.supervisor.get(&module_id) else {
5992 return Ok(vec![control_error_frame(
5993 &frame,
5994 "unknown_module",
5995 format!("module_id '{module_id}' is not supervised"),
5996 )?]);
5997 };
5998
5999 let snapshot = module.stderr_tail(
6000 max_lines.map(|value| value as usize),
6001 max_bytes.map(|value| value as usize),
6002 );
6003
6004 let response = ClientControlResponse::SupervisorStderrTail {
6005 module_id,
6006 tail: StderrTail {
6007 capture: match snapshot.capture {
6008 CaptureState::Captured => StderrCaptureState::Captured,
6009 CaptureState::Incomplete { reason } => {
6010 StderrCaptureState::Incomplete { reason }
6011 }
6012 CaptureState::NotCaptured { reason } => {
6013 StderrCaptureState::NotCaptured { reason }
6014 }
6015 },
6016 entries: snapshot
6017 .entries
6018 .into_iter()
6019 .map(|entry| match entry {
6020 TailEntry::Line {
6021 text,
6022 truncated,
6023 at_ms,
6024 } => StderrTailEntry::Line {
6025 text,
6026 truncated,
6027 at_ms,
6028 },
6029 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
6030 })
6031 .collect(),
6032 dropped_lines: snapshot.dropped_lines,
6033 },
6034 };
6035 Ok(vec![control_response_body_frame(
6036 &frame,
6037 &response,
6038 "ClientControlResponse::SupervisorStderrTail",
6039 )?])
6040 }
6041
6042 async fn observe_control_wait<F: std::future::Future>(
6043 &self,
6044 ctx: &RouteCtx,
6045 target: Option<&str>,
6046 kind: &'static str,
6047 future: F,
6048 ) -> F::Output {
6049 self.counters
6050 .observe_reader_wait(
6051 || {
6052 let module = self
6053 .forwarding
6054 .module_id_for_connection(ctx.connection_id)
6055 .ok()
6056 .flatten();
6057 let principal = if module.is_some() { "module" } else { "direct" };
6058 (
6059 self.resolve_reader_wait_target(kind, target)
6060 .or(module)
6061 .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
6062 principal.into(),
6063 )
6064 },
6065 kind,
6066 future,
6067 )
6068 .await
6069 }
6070
6071 pub(crate) fn resolve_reader_wait_target(
6074 &self,
6075 op: &str,
6076 selector: Option<&str>,
6077 ) -> Option<String> {
6078 let selector = selector?;
6079 match op {
6080 ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
6081 .supervisor
6082 .get(selector)
6083 .map(|module| module.module_id().to_owned()),
6084 ops::SUPERVISOR_HEALTH_PROBE => self
6085 .registry
6086 .get_module(selector)
6087 .ok()
6088 .flatten()
6089 .map(|registration| registration.manifest.module_id),
6090 _ => None,
6091 }
6092 }
6093
6094 async fn handle_supervisor_terminals(
6095 &self,
6096 ctx: &RouteCtx,
6097 frame: Frame,
6098 module_id: String,
6099 ) -> Result<Vec<Frame>, RouterError> {
6100 let Some(module) = self.supervisor.get(&module_id) else {
6101 return Ok(vec![control_error_frame(
6102 &frame,
6103 "unknown_module",
6104 format!("module_id '{module_id}' is not supervised"),
6105 )?]);
6106 };
6107
6108 let terminals = self
6111 .observe_control_wait(
6112 ctx,
6113 Some(module.module_id()),
6114 ops::SUPERVISOR_TERMINALS,
6115 module.read_durable_terminal_history(),
6116 )
6117 .await
6118 .map_err(|error| {
6119 RouterError::backend(
6120 0,
6121 frame.header.corr,
6122 format!("failed to read terminal history: {error}"),
6123 )
6124 })?;
6125 let response = ClientControlResponse::SupervisorTerminals {
6126 module_id,
6127 terminals,
6128 };
6129 Ok(vec![control_response_body_frame(
6130 &frame,
6131 &response,
6132 "ClientControlResponse::SupervisorTerminals",
6133 )?])
6134 }
6135
6136 fn handle_supervisor_routes(
6137 &self,
6138 frame: Frame,
6139 module_id: Option<String>,
6140 ) -> Result<Vec<Frame>, RouterError> {
6141 let modules = self
6142 .forwarding
6143 .route_census(module_id.as_deref())
6144 .map_err(RouterError::Forwarding)?
6145 .into_iter()
6146 .map(|(module_id, routes)| SupervisorRouteModule {
6147 module_id,
6148 routes: routes
6149 .into_iter()
6150 .map(|route| SupervisorRoute {
6151 consumer: match route.principal {
6152 Principal::Reserved { module_id } => {
6153 SupervisorRouteConsumer::Reserved { module_id }
6154 }
6155 Principal::Direct | Principal::Unverified => {
6156 SupervisorRouteConsumer::Direct {
6157 connection_id: route.goodbye_target.connection_id.get(),
6158 }
6159 }
6160 },
6161 age_ms: Instant::now()
6162 .saturating_duration_since(route.bound_at)
6163 .as_millis()
6164 .try_into()
6165 .unwrap_or(u64::MAX),
6166 draining: route.draining,
6167 drain_reason: route.drain_reason,
6168 })
6169 .collect(),
6170 })
6171 .collect();
6172 let response = ClientControlResponse::SupervisorRoutes { modules };
6173 Ok(vec![control_response_body_frame(
6174 &frame,
6175 &response,
6176 "ClientControlResponse::SupervisorRoutes",
6177 )?])
6178 }
6179
6180 async fn handle_supervisor_provenance(
6181 &self,
6182 ctx: &RouteCtx,
6183 frame: Frame,
6184 module_id: Option<String>,
6185 ) -> Result<Vec<Frame>, RouterError> {
6186 let targeted = module_id.is_some();
6187 let selected = if let Some(module_id) = module_id {
6188 let Some(module) = self.supervisor.get(&module_id) else {
6189 return Ok(vec![control_error_frame(
6190 &frame,
6191 "unknown_module",
6192 format!("module_id '{module_id}' is not supervised"),
6193 )?]);
6194 };
6195 vec![module]
6196 } else {
6197 self.supervisor.list()
6198 };
6199
6200 let target = targeted.then(|| selected[0].module_id());
6201 self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
6202 let mut modules = Vec::with_capacity(selected.len());
6203 for module in &selected {
6204 let (status, observed_image) = module
6205 .status_and_running_image_agreement()
6206 .await
6207 .map_err(|err| {
6208 RouterError::backend(
6209 0,
6210 frame.header.corr,
6211 format!("failed to read supervisor status: {err}"),
6212 )
6213 })?;
6214 let module_declared = self
6215 .registry
6216 .get_module(&status.module_id)
6217 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6218 .and_then(|registration| registration.manifest.provenance)
6219 .map(|build| ModuleDeclaredProvenance::Reported { build })
6220 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
6221 #[cfg(test)]
6222 let running_image = match &self.provenance_probe_override {
6223 Some(result) => result.clone(),
6224 None => observed_image,
6225 };
6226 #[cfg(not(test))]
6227 let running_image = observed_image;
6228 modules.push(SupervisorModuleProvenance {
6229 module_id: status.module_id,
6230 module_declared,
6231 daemon_observed: SupervisorObservedProcess {
6232 launch_nonce_source: status.launch_nonce_source,
6233 pid: status.pid,
6234 spawned_at_ms: status.spawned_at_ms,
6235 spawned_from: status.spawned_from,
6236 running_image,
6237 },
6238 });
6239 }
6240 let daemon = SupervisorDaemonProvenance {
6241 daemon_build: self.daemon_provenance.build.clone(),
6242 daemon_observed: DaemonObservedProcess {
6243 pid: self.daemon_provenance.pid,
6244 started_at_ms: self
6245 .daemon_provenance
6246 .start_clock
6247 .map(|clock| clock.started_at_ms())
6248 .or(self.daemon_provenance.started_at_ms),
6249 running_image: self
6250 .daemon_provenance
6251 .probe
6252 .observe(
6253 self.daemon_provenance.pid,
6254 self.daemon_provenance.executable_path.as_deref(),
6255 self.daemon_provenance.executable_identity.clone(),
6256 self.daemon_provenance.process_start_time,
6257 )
6258 .await,
6259 },
6260 };
6261 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
6262 Ok(vec![control_response_body_frame(
6263 &frame,
6264 &response,
6265 "ClientControlResponse::SupervisorProvenance",
6266 )?])
6267 })
6268 .await
6269 }
6270
6271 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6272 self.refresh_capability_requirements();
6273 let generation = self
6274 .registry
6275 .generation()
6276 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6277 let modules = self
6278 .supervisor
6279 .list()
6280 .into_iter()
6281 .map(|module| {
6282 let status = module.status_for_control("health").map_err(|err| {
6283 RouterError::backend(
6284 0,
6285 frame.header.corr,
6286 format!("failed to read supervisor health: {err}"),
6287 )
6288 })?;
6289 let module_id = status.module_id;
6290 let capability_detail = self
6291 .capability_evaluator
6292 .required_problem_detail(&module_id);
6293 Ok(SupervisorHealthEntry {
6294 module_id,
6295 status: status.health.status,
6296 detail: append_capability_problem_detail(
6297 status.health.detail,
6298 capability_detail,
6299 ),
6300 metrics: status.health.metrics,
6301 consecutive_failures: status.health.consecutive_failures,
6302 late_answer_count: status.health.late_answer_count,
6303 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
6304 last_action: status.health.last_action,
6305 last_action_ms: status.health.last_action_ms,
6306 last_probe_ms: status.health.last_probe_ms,
6307 })
6308 })
6309 .collect::<Result<Vec<_>, RouterError>>()?;
6310 let response = ClientControlResponse::SupervisorHealth {
6311 generation,
6312 modules,
6313 };
6314 Ok(vec![control_response_body_frame(
6315 &frame,
6316 &response,
6317 "ClientControlResponse::SupervisorHealth",
6318 )?])
6319 }
6320
6321 async fn handle_supervisor_restart(
6322 &self,
6323 frame: Frame,
6324 module_id: String,
6325 drain_timeout_ms: Option<u64>,
6326 requester: Principal,
6327 ) -> Result<Vec<Frame>, RouterError> {
6328 let operation_lock = self.supervisor.operation_lock();
6329 let _operation_guard = operation_lock.lock().await;
6330 let Some(module) = self.supervisor.get(&module_id) else {
6331 return Ok(vec![control_error_frame(
6332 &frame,
6333 "unknown_module",
6334 format!("module_id '{module_id}' is not supervised"),
6335 )?]);
6336 };
6337
6338 self.route_outages.mark_operator_action(&module_id);
6339 if let Err(err) = module
6340 .restart_requested(drain_timeout_ms, Some(requester))
6341 .await
6342 {
6343 self.route_outages
6344 .operator_action_ended_unrefused(&module_id);
6345 let (code, message) = match err {
6346 crate::supervise::SuperviseError::Disabled { .. } => {
6347 ("module_disabled", err.to_string())
6348 }
6349 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6350 ("swap_in_progress", err.to_string())
6351 }
6352 _ => (
6353 "target_unavailable",
6354 format!("failed to restart module_id '{module_id}': {err}"),
6355 ),
6356 };
6357 return Ok(vec![control_error_frame(&frame, code, message)?]);
6358 }
6359
6360 let response = ClientControlResponse::SupervisorAck {
6361 module_id,
6362 applied: true,
6363 };
6364 Ok(vec![control_response_body_frame(
6365 &frame,
6366 &response,
6367 "ClientControlResponse::SupervisorAck",
6368 )?])
6369 }
6370
6371 async fn handle_supervisor_swap(
6375 &self,
6376 frame: Frame,
6377 module_id: String,
6378 ready_timeout_ms: Option<u64>,
6379 requester: Principal,
6380 ) -> Result<Vec<Frame>, RouterError> {
6381 let module = {
6388 let operation_lock = self.supervisor.operation_lock();
6389 let _operation_guard = operation_lock.lock().await;
6390 self.supervisor.get(&module_id)
6391 };
6392 let Some(module) = module else {
6393 return Ok(vec![control_error_frame(
6394 &frame,
6395 "unknown_module",
6396 format!("module_id '{module_id}' is not supervised"),
6397 )?]);
6398 };
6399
6400 self.route_outages.mark_operator_action(&module_id);
6401 if let Err(err) = module
6402 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6403 .await
6404 {
6405 self.route_outages
6406 .operator_action_ended_unrefused(&module_id);
6407 use crate::supervise::SuperviseError;
6408 let message = err.to_string();
6409 let error = match err {
6410 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6411 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6412 code: "swap_refused".to_string(),
6413 message,
6414 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6415 },
6416 SuperviseError::SwapFailed {
6417 arm,
6418 candidate_exit,
6419 ..
6420 } => ErrorBody {
6421 code: "swap_failed".to_string(),
6422 message,
6423 detail: Some(serde_json::json!({
6424 "arm": arm.as_str(),
6425 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6426 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6427 })),
6428 },
6429 _ => ErrorBody::new(
6430 "target_unavailable",
6431 format!("failed to swap module_id '{module_id}': {message}"),
6432 ),
6433 };
6434 return Ok(vec![control_error_body_frame(&frame, error)?]);
6435 }
6436 self.route_outages
6440 .operator_action_ended_unrefused(&module_id);
6441
6442 let response = ClientControlResponse::SupervisorAck {
6443 module_id,
6444 applied: true,
6445 };
6446 Ok(vec![control_response_body_frame(
6447 &frame,
6448 &response,
6449 "ClientControlResponse::SupervisorAck",
6450 )?])
6451 }
6452
6453 async fn handle_supervisor_reload(
6454 &self,
6455 frame: Frame,
6456 module_id: String,
6457 requester: Principal,
6458 ) -> Result<Vec<Frame>, RouterError> {
6459 let operation_lock = self.supervisor.operation_lock();
6460 let _operation_guard = operation_lock.lock().await;
6461 let Some(module) = self.supervisor.get(&module_id) else {
6462 return Ok(vec![control_error_frame(
6463 &frame,
6464 "unknown_module",
6465 format!("module_id '{module_id}' is not supervised"),
6466 )?]);
6467 };
6468
6469 self.route_outages.mark_operator_action(&module_id);
6470 if let Err(err) = module.reload_requested(Some(requester)).await {
6471 self.route_outages
6472 .operator_action_ended_unrefused(&module_id);
6473 let (code, message) = match err {
6474 crate::supervise::SuperviseError::Disabled { .. } => {
6475 ("module_disabled", err.to_string())
6476 }
6477 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6478 ("swap_in_progress", err.to_string())
6479 }
6480 _ => (
6481 "reload_failed",
6482 format!("failed to reload module_id '{module_id}': {err}"),
6483 ),
6484 };
6485 return Ok(vec![control_error_frame(&frame, code, message)?]);
6486 }
6487
6488 let response = ClientControlResponse::SupervisorAck {
6489 module_id,
6490 applied: true,
6491 };
6492 Ok(vec![control_response_body_frame(
6493 &frame,
6494 &response,
6495 "ClientControlResponse::SupervisorAck",
6496 )?])
6497 }
6498
6499 async fn handle_supervisor_rescan(
6503 &self,
6504 frame: Frame,
6505 mode: RescanMode,
6506 requester: Principal,
6507 ) -> Result<Vec<Frame>, RouterError> {
6508 let preview = mode.previews();
6509 let Some(context) = self.rescan.clone() else {
6510 return Ok(vec![control_error_frame(
6511 &frame,
6512 "rescan_unavailable",
6513 "the daemon was not started with a reloadable config path".to_string(),
6514 )?]);
6515 };
6516
6517 let operation_lock = self.supervisor.operation_lock();
6518 let _operation_guard = operation_lock.lock().await;
6519 let loaded = match self.load_rescan_config(&mode, &context) {
6523 Ok(config) => config,
6524 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6525 };
6526 let Some((config, activity_config)) = loaded else {
6534 return Ok(vec![control_error_frame(
6535 &frame,
6536 "invalid_daemon_config",
6537 format!(
6538 "daemon config not found at {}; refusing to rescan (an absent config would \
6539 retire every supervised module)",
6540 context.config_path.display()
6541 ),
6542 )?]);
6543 };
6544 let (
6545 configured_port,
6546 storage_config,
6547 admission_facts_carrier_module_id,
6548 admission_facts_targets,
6549 scope_authority_owners,
6550 privileged_operator_prompt,
6551 modules,
6552 reserved_capabilities,
6553 ) = (
6554 config.port,
6555 config.storage,
6556 config.admission_facts_carrier_module_id,
6557 config.admission_facts_targets,
6558 config.scope_authority_owners,
6559 config.privileged_operator_prompt,
6560 config.modules,
6561 config.reserved_capabilities,
6562 );
6563
6564 let mut restart_required = Vec::new();
6573 for section in RestartRequiredSection::ALL {
6574 let changed = match section {
6575 RestartRequiredSection::Port => configured_port != context.configured_port,
6576 RestartRequiredSection::Storage => storage_config != context.storage_config,
6577 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6578 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6579 }
6580 RestartRequiredSection::AdmissionFactsTargets => {
6581 admission_facts_targets != context.admission_facts_targets
6582 }
6583 RestartRequiredSection::ScopeAuthorityOwners => {
6584 scope_authority_owners != context.scope_authority_owners
6585 }
6586 RestartRequiredSection::PrivilegedOperatorPrompt => {
6587 privileged_operator_prompt != context.privileged_operator_prompt
6588 }
6589 };
6590 if changed {
6591 restart_required.push(section.label().to_string());
6592 }
6593 }
6594 if !restart_required.is_empty() {
6595 warn!(
6596 config_path = %context.config_path.display(),
6597 sections = %restart_required.join(", "),
6598 "daemon config changed outside the modules section; restart the daemon to apply those changes"
6599 );
6600 }
6601
6602 for configured in &modules {
6603 if let Err(err) = validate_spec(&configured.module_spec()) {
6604 return Ok(vec![control_error_frame(
6605 &frame,
6606 "invalid_daemon_config",
6607 format!("supervisor rescan rejected daemon config: {err}"),
6608 )?]);
6609 }
6610 }
6611
6612 let configured_capabilities = modules
6613 .iter()
6614 .map(|module| (module.module_id.clone(), module.enabled))
6615 .collect::<Vec<_>>();
6616 let preview_capability_warnings = if preview {
6617 let (_, registrations) = self.runtime_capability_snapshot()?;
6618 let current_modules = self
6619 .supervisor
6620 .list()
6621 .into_iter()
6622 .map(|module| module.module_id().to_string())
6623 .collect::<BTreeSet<_>>();
6624 let resulting_modules = configured_capabilities.clone();
6625 let removed = current_modules
6626 .into_iter()
6627 .filter(|module_id| {
6628 !resulting_modules
6629 .iter()
6630 .any(|(configured_id, _)| configured_id == module_id)
6631 })
6632 .collect::<Vec<_>>();
6633 let disabled = match &mode {
6636 RescanMode::Candidate(_) => resulting_modules
6637 .iter()
6638 .filter(|(_, enabled)| !enabled)
6639 .map(|(module_id, _)| module_id.clone())
6640 .collect::<Vec<_>>(),
6641 _ => Vec::new(),
6642 };
6643 self.capability_evaluator.preview_warnings(
6644 resulting_modules,
6645 &removed,
6646 &disabled,
6647 ®istrations,
6648 )
6649 } else {
6650 Vec::new()
6651 };
6652 let reconciled = match &mode {
6653 RescanMode::Candidate(candidate) => self
6654 .preview_candidate(&context, modules, candidate)
6655 .map(|result| (result, Vec::new())),
6656 _ => {
6657 self.reconcile_supervised_modules(
6658 &context.supervisor,
6659 modules,
6660 preview,
6661 requester.clone(),
6662 )
6663 .await
6664 }
6665 };
6666 let (result, changed) = match reconciled {
6667 Ok(result) => result,
6668 Err(message) => {
6669 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6670 }
6671 };
6672 if !preview {
6673 if let Some(ledger) = &self.activity {
6674 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6675 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6676 }
6677 if let Err(code) = self.configure_activity(activity_config).await {
6678 warn!(
6679 code,
6680 "activity retention could not be applied during rescan"
6681 );
6682 }
6683 self.capability_evaluator
6684 .configure(configured_capabilities, reserved_capabilities);
6685 self.capability_evaluator.wake_deadline_loop();
6686 self.refresh_capability_requirements();
6687 }
6688 let mut result = result;
6689 result.restart_required = restart_required;
6690 result.capability_warnings = preview_capability_warnings;
6691 let response = ClientControlResponse::SupervisorRescan { result };
6692 Ok(vec![control_response_body_frame(
6693 &frame,
6694 &response,
6695 "ClientControlResponse::SupervisorRescan",
6696 )?])
6697 }
6698
6699 async fn handle_supervisor_release_reserved(
6700 &self,
6701 frame: Frame,
6702 module_id: String,
6703 ) -> Result<Vec<Frame>, RouterError> {
6704 let Some(context) = self.rescan.clone() else {
6705 return Ok(vec![control_error_frame(
6706 &frame,
6707 "release_unavailable",
6708 "reserved-id release requires a daemon started with a reloadable config path",
6709 )?]);
6710 };
6711 let operation_lock = self.supervisor.operation_lock();
6712 let _operation_guard = operation_lock.lock().await;
6713 let loaded = match crate::daemon_config::load(&context.config_path) {
6714 Ok(Some(config)) => config,
6715 Ok(None) => {
6716 return Ok(vec![control_error_frame(
6717 &frame,
6718 "invalid_daemon_config",
6719 format!(
6720 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6721 context.config_path.display()
6722 ),
6723 )?])
6724 }
6725 Err(err) => {
6726 return Ok(vec![control_error_frame(
6727 &frame,
6728 "invalid_daemon_config",
6729 format!("unable to verify reserved-id release against daemon config: {err}"),
6730 )?])
6731 }
6732 };
6733 if loaded
6734 .modules
6735 .iter()
6736 .any(|configured| configured.module_id == module_id)
6737 {
6738 return Ok(vec![control_error_frame(
6739 &frame,
6740 "reserved_module_configured",
6741 format!(
6742 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6743 ),
6744 )?]);
6745 }
6746 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6747 return Ok(vec![control_error_frame(
6748 &frame,
6749 "reserved_gate_not_retained",
6750 format!(
6751 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6752 ),
6753 )?]);
6754 }
6755
6756 let response = ClientControlResponse::SupervisorAck {
6757 module_id,
6758 applied: true,
6759 };
6760 Ok(vec![control_response_body_frame(
6761 &frame,
6762 &response,
6763 "ClientControlResponse::SupervisorAck",
6764 )?])
6765 }
6766
6767 fn plan_rescan(
6771 &self,
6772 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6773 ) -> Result<RescanPlan, String> {
6774 let mut current = BTreeMap::new();
6775 for module in self.supervisor.list() {
6776 let (spec, health) = module.configuration().map_err(|err| {
6777 format!(
6778 "failed to read configuration for module_id '{}': {err}",
6779 module.module_id()
6780 )
6781 })?;
6782 let enabled = module
6783 .status()
6784 .map_err(|err| {
6785 format!(
6786 "failed to read status for module_id '{}': {err}",
6787 module.module_id()
6788 )
6789 })?
6790 .enabled;
6791 current.insert(
6792 module.module_id().to_string(),
6793 (module, spec, health, enabled),
6794 );
6795 }
6796 let configured = configured_modules
6797 .into_iter()
6798 .map(|module| (module.module_id.clone(), module))
6799 .collect::<BTreeMap<_, _>>();
6800
6801 let added = configured
6802 .keys()
6803 .filter(|module_id| !current.contains_key(*module_id))
6804 .cloned()
6805 .collect::<Vec<_>>();
6806 let removed = current
6807 .keys()
6808 .filter(|module_id| !configured.contains_key(*module_id))
6809 .cloned()
6810 .collect::<Vec<_>>();
6811 let mut changed_pending_reload = Vec::new();
6812 let mut configuration_changes = BTreeSet::new();
6813 let mut enabled_changes = BTreeSet::new();
6814 let mut unchanged = 0_u32;
6815
6816 for (module_id, configured_module) in &configured {
6817 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6818 else {
6819 continue;
6820 };
6821 let launch_changed = *current_spec != configured_module.module_spec();
6825 let configuration_changed =
6826 launch_changed || *current_health != configured_module.health;
6827 let enabled_changed = *current_enabled != configured_module.enabled;
6828 if configuration_changed {
6829 configuration_changes.insert(module_id.clone());
6830 }
6831 if launch_changed {
6832 changed_pending_reload.push(module_id.clone());
6833 }
6834 if enabled_changed {
6835 enabled_changes.insert(module_id.clone());
6836 }
6837 if !configuration_changed && !enabled_changed {
6838 unchanged = unchanged.saturating_add(1);
6839 }
6840 }
6841 Ok(RescanPlan {
6842 current,
6843 configured,
6844 added,
6845 removed,
6846 changed_pending_reload,
6847 configuration_changes,
6848 enabled_changes,
6849 unchanged,
6850 })
6851 }
6852
6853 async fn reconcile_supervised_modules(
6862 &self,
6863 supervisor: &Supervisor,
6864 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6865 preview: bool,
6866 requester: Principal,
6867 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6868 let plan = self.plan_rescan(configured_modules)?;
6869
6870 if preview {
6876 return Ok((plan.preview_result(), Vec::new()));
6877 }
6878 let changed = plan
6879 .configuration_changes
6880 .union(&plan.enabled_changes)
6881 .cloned()
6882 .collect::<Vec<_>>();
6883 let RescanPlan {
6884 current,
6885 configured,
6886 added,
6887 removed,
6888 changed_pending_reload,
6889 configuration_changes,
6890 enabled_changes,
6891 unchanged,
6892 } = plan;
6893 if !(added.is_empty()
6898 && removed.is_empty()
6899 && configuration_changes.is_empty()
6900 && enabled_changes.is_empty())
6901 {
6902 if let Some(context) = &self.rescan {
6903 context
6904 .generation
6905 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6906 }
6907 }
6908
6909 for module_id in &removed {
6910 let module = ¤t
6911 .get(module_id)
6912 .expect("removed module came from current supervisor state")
6913 .0;
6914 module
6915 .retire_requested(Some(requester.clone()))
6916 .await
6917 .map_err(|err| {
6918 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6919 })?;
6920 self.supervisor.record_rescan_removal(module_id);
6954 self.supervisor.retire(module_id);
6955 self.route_outages.forget(module_id);
6956 }
6957
6958 for module_id in configured.keys() {
6959 let Some((module, _, _, _)) = current.get(module_id) else {
6960 continue;
6961 };
6962 let configured_module = configured
6963 .get(module_id)
6964 .expect("configured module id came from configured map");
6965 if configuration_changes.contains(module_id) {
6966 module
6967 .update_configuration(
6968 configured_module.module_spec(),
6969 configured_module.health.clone(),
6970 configured_module.drain_timeout_ms,
6971 )
6972 .await
6973 .map_err(|err| {
6974 format!(
6975 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6976 )
6977 })?;
6978 }
6979 if enabled_changes.contains(module_id) {
6980 self.route_outages.mark_operator_action(module_id);
6983 module
6984 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6985 .await
6986 .map_err(|err| {
6987 self.route_outages.operator_action_ended_unrefused(module_id);
6988 format!(
6989 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6990 configured_module.enabled
6991 )
6992 })?;
6993 }
6994 }
6995
6996 for module_id in &added {
6997 let configured_module = configured
6998 .get(module_id)
6999 .expect("added module id came from configured map");
7000 supervisor
7001 .supervise_configured_requested(
7002 configured_module.module_spec(),
7003 configured_module.enabled,
7004 configured_module.health.clone(),
7005 configured_module.drain_timeout_ms,
7006 configured_module.restart,
7007 configured_module.register_timeout,
7008 Some(requester.clone()),
7009 )
7010 .map_err(|err| {
7011 format!("failed to add module_id '{module_id}' during rescan: {err}")
7012 })?;
7013 }
7014
7015 Ok((
7016 SupervisorRescanResult {
7017 added,
7018 removed,
7019 changed_pending_reload,
7020 enabled_changes: enabled_changes.iter().cloned().collect(),
7021 unchanged,
7022 preview: false,
7023 restart_required: Vec::new(),
7026 capability_warnings: Vec::new(),
7027 module_actions: Vec::new(),
7028 earlier_pending: Vec::new(),
7029 expect: None,
7030 },
7031 changed,
7032 ))
7033 }
7034
7035 async fn handle_supervisor_set_enabled(
7036 &self,
7037 frame: Frame,
7038 module_id: String,
7039 enabled: bool,
7040 requester: Principal,
7041 ) -> Result<Vec<Frame>, RouterError> {
7042 let operation_lock = self.supervisor.operation_lock();
7043 let _operation_guard = operation_lock.lock().await;
7044 let Some(module) = self.supervisor.get(&module_id) else {
7045 return Ok(vec![control_error_frame(
7046 &frame,
7047 "unknown_module",
7048 format!("module_id '{module_id}' is not supervised"),
7049 )?]);
7050 };
7051
7052 self.route_outages.mark_operator_action(&module_id);
7055 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
7056 Ok(applied) => applied,
7057 Err(err) => {
7058 self.route_outages
7059 .operator_action_ended_unrefused(&module_id);
7060 return Ok(vec![control_error_frame(
7061 &frame,
7062 "target_unavailable",
7063 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
7064 )?]);
7065 }
7066 };
7067 if !applied {
7068 self.route_outages
7071 .operator_action_ended_unrefused(&module_id);
7072 }
7073
7074 self.capability_evaluator.wake_deadline_loop();
7075 self.refresh_capability_requirements();
7076 let response = ClientControlResponse::SupervisorAck { module_id, applied };
7077 Ok(vec![control_response_body_frame(
7078 &frame,
7079 &response,
7080 "ClientControlResponse::SupervisorAck",
7081 )?])
7082 }
7083
7084 async fn handle_supervisor_health_probe(
7085 &self,
7086 ctx: &RouteCtx,
7087 frame: Frame,
7088 module_id: String,
7089 ) -> Result<Vec<Frame>, RouterError> {
7090 self.refresh_capability_requirements();
7091 let Some(registration) = self
7092 .registry
7093 .get_module(&module_id)
7094 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
7095 else {
7096 return Ok(vec![control_error_frame(
7097 &frame,
7098 "unknown_module",
7099 format!("module_id '{module_id}' is not registered"),
7100 )?]);
7101 };
7102
7103 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
7116 {
7117 return Ok(vec![control_error_frame(
7118 &frame,
7119 "health_not_advertised",
7120 format!("module_id '{module_id}' did not advertise health.check"),
7121 )?]);
7122 }
7123
7124 self.observe_control_wait(
7125 ctx,
7126 Some(®istration.manifest.module_id),
7127 ops::SUPERVISOR_HEALTH_PROBE,
7128 async {
7129 let deadline = Instant::now() + self.health_probe_timeout;
7130 let pending = match self.forwarding.begin_module_control_rpc_for(
7131 &module_id,
7132 MODULE_CONTROL_OP_HEALTH_CHECK,
7133 deadline,
7134 ) {
7135 Ok(pending) => pending,
7136 Err(err) => {
7137 return Ok(vec![control_error_frame(
7138 &frame,
7139 forwarding_error_code(&err),
7140 err.to_string(),
7141 )?])
7142 }
7143 };
7144
7145 let PendingModuleControlRpc {
7146 endpoint,
7147 module_sink,
7148 negotiated_ver,
7149 corr: probe_corr,
7150 receiver,
7151 } = pending;
7152 let mut guard =
7153 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
7154 let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
7155 .map_err(|err| {
7156 RouterError::backend(
7157 0,
7158 frame.header.corr,
7159 format!("failed to encode health.check request: {err}"),
7160 )
7161 })?;
7162 let probe_frame = Frame::build_with_version(
7163 negotiated_ver,
7164 FrameType::Request,
7165 control_flags(),
7166 0,
7167 0,
7168 probe_corr,
7169 probe_body,
7170 )
7171 .map_err(RouterError::FrameBuild)?;
7172
7173 match timeout_at(deadline, module_sink.send(probe_frame)).await {
7176 Ok(Ok(())) => {}
7177 Ok(Err(err)) => {
7178 return Ok(vec![control_error_frame(
7179 &frame,
7180 "target_unavailable",
7181 err.to_string(),
7182 )?]);
7183 }
7184 Err(_) => {
7185 return Ok(vec![control_error_frame(
7186 &frame,
7187 "module_timeout",
7188 format!(
7189 "module_id '{module_id}' did not answer health.check within {:?}",
7190 self.health_probe_timeout
7191 ),
7192 )?]);
7193 }
7194 }
7195
7196 match timeout_at(deadline, receiver).await {
7197 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
7198 guard.disarm();
7199 let Some(report) = response.health_report() else {
7200 return Ok(vec![control_error_frame(
7201 &frame,
7202 "invalid_control_body",
7203 "health.check RPC returned a non-health response",
7204 )?]);
7205 };
7206 let HealthReport {
7211 status,
7212 detail,
7213 metrics,
7214 } = report;
7215 let capability_detail = self
7216 .capability_evaluator
7217 .required_problem_detail(&module_id);
7218 let response = ClientControlResponse::SupervisorHealthProbe {
7219 module_id,
7220 status,
7221 detail: append_capability_problem_detail(detail, capability_detail),
7222 metrics,
7223 };
7224 Ok(vec![control_response_body_frame(
7225 &frame,
7226 &response,
7227 "ClientControlResponse::SupervisorHealthProbe",
7228 )?])
7229 }
7230 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
7231 guard.disarm();
7232 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
7237 }
7238 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
7239 guard.disarm();
7240 Ok(vec![control_error_frame(
7241 &frame,
7242 "target_unavailable",
7243 message,
7244 )?])
7245 }
7246 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
7247 guard.disarm();
7248 Ok(vec![control_error_frame(
7249 &frame,
7250 "invalid_control_body",
7251 message,
7252 )?])
7253 }
7254 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
7255 guard.disarm();
7256 Ok(vec![control_error_frame(
7257 &frame,
7258 "invalid_control_body",
7259 format!("expected module-control op '{expected}', got '{actual}'"),
7260 )?])
7261 }
7262 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
7263 guard.disarm();
7264 Ok(vec![control_error_frame(
7265 &frame,
7266 "module_timeout",
7267 format!(
7268 "module_id '{module_id}' answered health.check after {:?}",
7269 self.health_probe_timeout
7270 ),
7271 )?])
7272 }
7273 Ok(Err(_)) => Ok(vec![control_error_frame(
7274 &frame,
7275 "target_unavailable",
7276 "health.check waiter was canceled before the module responded",
7277 )?]),
7278 Err(_) => Ok(vec![control_error_frame(
7279 &frame,
7280 "module_timeout",
7281 format!(
7282 "module_id '{module_id}' did not answer health.check within {:?}",
7283 self.health_probe_timeout
7284 ),
7285 )?]),
7286 }
7287 },
7288 )
7289 .await
7290 }
7291
7292 fn supervisor_status(
7293 &self,
7294 module_id: &str,
7295 corr: u64,
7296 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
7297 self.supervisor
7298 .get(module_id)
7299 .map(|module| {
7300 let warming = module.is_warming_for_control("status").map_err(|err| {
7301 RouterError::backend(
7302 0,
7303 corr,
7304 format!(
7305 "failed to read supervisor warming state for module_id '{module_id}': {err}"
7306 ),
7307 )
7308 })?;
7309 module.status_for_control("status").map_err(|err| {
7310 RouterError::backend(
7311 0,
7312 corr,
7313 format!(
7314 "failed to read supervisor status for module_id '{module_id}': {err}"
7315 ),
7316 )
7317 }).map(|status| (status, warming))
7318 })
7319 .transpose()
7320 }
7321
7322 fn guard_module_control_op(
7323 &self,
7324 frame: &Frame,
7325 module_id: &str,
7326 op: &str,
7327 ) -> Result<Option<Frame>, RouterError> {
7328 if self.module_grants_op(module_id, op, frame.header.corr)? {
7329 return Ok(None);
7330 }
7331
7332 Ok(Some(control_error_frame(
7333 frame,
7334 "op_not_allowed",
7335 format!("module_id '{module_id}' did not grant control op '{op}'"),
7336 )?))
7337 }
7338
7339 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
7340 let Some(registration) = self
7341 .registry
7342 .get_module(module_id)
7343 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
7344 else {
7345 return Ok(false);
7346 };
7347 Ok(module_registration_grants_op(®istration.control_ops, op))
7348 }
7349
7350 fn handle_status_update(
7351 &self,
7352 endpoint: ModuleEndpointId,
7353 frame: Frame,
7354 ) -> Result<Vec<Frame>, RouterError> {
7355 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
7356 Ok(update) => update,
7357 Err(err) => {
7358 if is_known_module_push_op(&frame.body) {
7363 return Ok(vec![control_error_frame(
7364 &frame,
7365 "invalid_control_body",
7366 format!("malformed module control push body: {err}"),
7367 )?]);
7368 }
7369 return Ok(Vec::new());
7370 }
7371 };
7372
7373 match update {
7374 ModuleControlPush::RouteStatus {
7375 route_channel,
7376 route_epoch,
7377 status,
7378 } => {
7379 self.forwarding
7380 .cache_status(endpoint, route_channel, route_epoch, status)
7381 .map_err(RouterError::Forwarding)?;
7382 }
7383 }
7384 Ok(Vec::new())
7385 }
7386
7387 fn handle_route_poll(
7388 &self,
7389 ctx: &RouteCtx,
7390 frame: Frame,
7391 route_channel: u16,
7392 route_epoch: u32,
7393 kind: PollKind,
7394 ) -> Result<Vec<Frame>, RouterError> {
7395 let snapshot = self
7396 .forwarding
7397 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7398 .map_err(RouterError::Forwarding)?;
7399 let response = match (kind, snapshot) {
7400 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7401 ClientControlResponse::RoutePoll {
7402 route_channel,
7403 route_epoch,
7404 status,
7405 live: None,
7406 }
7407 }
7408 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7409 route_channel,
7410 route_epoch,
7411 status: None,
7412 live: None,
7413 },
7414 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7415 let live = self
7441 .process_liveness
7442 .as_ref()
7443 .and_then(|source| source.process_live(&module_id))
7444 .unwrap_or(true);
7445 ClientControlResponse::RoutePoll {
7446 route_channel,
7447 route_epoch,
7448 status: None,
7449 live: Some(live),
7450 }
7451 }
7452 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7453 route_channel,
7454 route_epoch,
7455 status: None,
7456 live: Some(false),
7457 },
7458 };
7459
7460 Ok(vec![control_response_body_frame(
7461 &frame,
7462 &response,
7463 "ClientControlResponse::RoutePoll",
7464 )?])
7465 }
7466
7467 pub(crate) fn observe_module_control_completion(
7468 &self,
7469 completion: ModuleControlRpcCompletion,
7470 ) -> bool {
7471 match completion {
7472 ModuleControlRpcCompletion::Unknown => false,
7473 ModuleControlRpcCompletion::Settled => true,
7474 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7475 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7476 info!(
7477 module_id = %module_id,
7478 latency_ms,
7479 "late health.check answer proves the module is alive"
7480 );
7481 match self
7482 .supervisor
7483 .record_late_health_answer(&module_id, latency_ms)
7484 {
7485 Ok(true) => {}
7486 Ok(false) => debug!(
7487 module_id = %module_id,
7488 latency_ms,
7489 "late health.check answer has no active supervisor snapshot"
7490 ),
7491 Err(err) => warn!(
7492 module_id = %module_id,
7493 latency_ms,
7494 error = %err,
7495 "failed to record late health.check answer"
7496 ),
7497 }
7498 true
7499 }
7500 }
7501 }
7502
7503 fn refuse_to_end_module_connection_for_a_client(
7524 &self,
7525 module_connection_id: ConnectionId,
7526 corr: u64,
7527 err: ForwardingError,
7528 ) -> Result<(), RouterError> {
7529 if let ForwardingError::ConnectionClosing { connection_id } = err {
7530 if connection_id != module_connection_id {
7531 warn!(
7532 module_connection_id = module_connection_id.get(),
7533 client_connection_id = connection_id.get(),
7534 corr,
7535 "dropping a route.bind response for a closing client; the module connection keeps serving"
7536 );
7537 return Ok(());
7538 }
7539 }
7540 Err(RouterError::Forwarding(err))
7541 }
7542
7543 fn handle_module_relay_response(
7544 &self,
7545 connection_id: ConnectionId,
7546 frame: Frame,
7547 ) -> Result<Vec<Frame>, RouterError> {
7548 let mut secondary_error = None;
7549 let outcome = match frame.header.ty {
7550 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7551 Ok(probe) if probe.op == "route.bind" => {
7552 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7553 Ok(ModuleControlResponse::RouteBindAck {}) => {
7554 RouteBindRelayOutcome::Accepted
7555 }
7556 Ok(other) => {
7557 let message =
7558 format!("route.bind response carried unexpected body: {other:?}");
7559 secondary_error = Some(control_error_frame(
7560 &frame,
7561 "invalid_control_body",
7562 message.clone(),
7563 )?);
7564 RouteBindRelayOutcome::ModuleGone(message)
7565 }
7566 Err(err) => {
7567 let message = format!("malformed route.bind response body: {err}");
7568 secondary_error = Some(control_error_frame(
7569 &frame,
7570 "invalid_control_body",
7571 message.clone(),
7572 )?);
7573 RouteBindRelayOutcome::ModuleGone(message)
7574 }
7575 }
7576 }
7577 Ok(probe) => {
7578 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7579 {
7580 Ok(response) => ModuleControlRpcOutcome::Response(response),
7581 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7582 "malformed {} response body: {err}",
7583 probe.op
7584 )),
7585 };
7586 let completion = self
7587 .forwarding
7588 .complete_module_control_rpc(
7589 connection_id,
7590 frame.header.corr,
7591 Some(&probe.op),
7592 outcome,
7593 )
7594 .map_err(RouterError::Forwarding)?;
7595 if !self.observe_module_control_completion(completion) {
7596 debug!(
7597 connection_id = connection_id.get(),
7598 corr = frame.header.corr,
7599 op = %probe.op,
7600 "dropping late or unknown module-control RPC response"
7601 );
7602 }
7603 return Ok(Vec::new());
7604 }
7605 Err(err) => {
7606 if let Some(expected_op) = self
7607 .forwarding
7608 .pending_module_control_op(connection_id, frame.header.corr)
7609 .map_err(RouterError::Forwarding)?
7610 {
7611 let completion = self
7612 .forwarding
7613 .complete_module_control_rpc(
7614 connection_id,
7615 frame.header.corr,
7616 None,
7617 ModuleControlRpcOutcome::MalformedResponse(format!(
7618 "malformed {expected_op} response body: {err}"
7619 )),
7620 )
7621 .map_err(RouterError::Forwarding)?;
7622 if !self.observe_module_control_completion(completion) {
7623 debug!(
7624 connection_id = connection_id.get(),
7625 corr = frame.header.corr,
7626 "dropping late malformed module-control RPC response"
7627 );
7628 }
7629 return Ok(Vec::new());
7630 }
7631 let message = format!("malformed route.bind response body: {err}");
7632 secondary_error = Some(control_error_frame(
7633 &frame,
7634 "invalid_control_body",
7635 message.clone(),
7636 )?);
7637 RouteBindRelayOutcome::ModuleGone(message)
7638 }
7639 },
7640 FrameType::Error => {
7641 if self
7642 .forwarding
7643 .pending_module_control_op(connection_id, frame.header.corr)
7644 .map_err(RouterError::Forwarding)?
7645 .is_some()
7646 {
7647 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7648 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7649 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7650 "malformed module-control ERROR body: {err}"
7651 )),
7652 };
7653 let completion = self
7654 .forwarding
7655 .complete_module_control_rpc(
7656 connection_id,
7657 frame.header.corr,
7658 None,
7659 outcome,
7660 )
7661 .map_err(RouterError::Forwarding)?;
7662 if !self.observe_module_control_completion(completion) {
7663 debug!(
7664 connection_id = connection_id.get(),
7665 corr = frame.header.corr,
7666 "dropping late or unknown module-control RPC error"
7667 );
7668 }
7669 return Ok(Vec::new());
7670 }
7671 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7672 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7673 Err(err) => {
7674 let message = format!("malformed route.bind ERROR body: {err}");
7675 secondary_error = Some(control_error_frame(
7676 &frame,
7677 "invalid_control_body",
7678 message.clone(),
7679 )?);
7680 RouteBindRelayOutcome::ModuleGone(message)
7681 }
7682 }
7683 }
7684 ty => {
7685 return Ok(vec![control_error_frame(
7686 &frame,
7687 "unsupported_control_frame",
7688 format!("unsupported module channel-0 frame {ty:?}"),
7689 )?])
7690 }
7691 };
7692
7693 let settled =
7694 self.forwarding
7695 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7696 let completion = match settled {
7697 Ok(completion) => completion,
7698 Err(err) => {
7699 self.refuse_to_end_module_connection_for_a_client(
7700 connection_id,
7701 frame.header.corr,
7702 err,
7703 )?;
7704 return Ok(secondary_error.into_iter().collect());
7705 }
7706 };
7707 if let Some(target) = completion.abandoned.as_ref() {
7708 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7709 }
7710 if !completion.settled {
7711 debug!(
7712 connection_id = connection_id.get(),
7713 corr = frame.header.corr,
7714 frame_type = ?frame.header.ty,
7715 "dropping late or unknown route.bind relay response"
7716 );
7717 }
7718 Ok(secondary_error.into_iter().collect())
7719 }
7720
7721 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7722 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7723 self.cleanup_connection_with_end_reason(
7727 connection_id,
7728 RegistrationEndReason::ExplicitGoodbye,
7729 )
7730 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7731 Ok(Vec::new())
7732 }
7733}
7734
7735impl Default for ControlHandler {
7736 fn default() -> Self {
7737 Self::new(Arc::new(Registry::default()))
7738 }
7739}
7740
7741impl crate::supervise::SwapPromotionObserver for ControlHandler {
7742 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7743 self.apply_registration_capabilities(registration);
7744 }
7745}
7746
7747fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7748 CapabilityRequirementStatus {
7749 consumer: status.consumer,
7750 capability: status.capability,
7751 need: match status.need {
7752 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7753 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7754 },
7755 verdict: status.verdict.as_str().to_string(),
7756 episode_seq: status.episode_seq,
7757 config_satisfiable: status.config_satisfiable,
7758 runtime_available: status.runtime_available,
7759 detail: status.detail,
7760 }
7761}
7762
7763fn append_capability_problem_detail(
7764 detail: Option<String>,
7765 capability_detail: Option<String>,
7766) -> Option<String> {
7767 match (detail, capability_detail) {
7768 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7769 (Some(detail), None) => Some(detail),
7770 (None, Some(capability_detail)) => Some(capability_detail),
7771 (None, None) => None,
7772 }
7773}
7774
7775fn subc_ops() -> Vec<String> {
7776 SUBC_CONTROL_OPS
7777 .iter()
7778 .map(|op| (*op).to_string())
7779 .collect()
7780}
7781
7782fn module_subc_ops() -> Vec<String> {
7783 SUBC_CONTROL_OPS
7784 .iter()
7785 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7786 .map(|op| (*op).to_string())
7787 .collect()
7788}
7789
7790#[cfg(test)]
7791fn module_baseline_control_ops() -> Vec<String> {
7792 MODULE_BASELINE_CONTROL_OPS
7793 .iter()
7794 .map(|op| (*op).to_string())
7795 .collect()
7796}
7797
7798fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7799 let mut seen = HashSet::new();
7800 let mut effective = Vec::new();
7801 for op in MODULE_BASELINE_CONTROL_OPS {
7802 if seen.insert((*op).to_string()) {
7803 effective.push((*op).to_string());
7804 }
7805 }
7806 for op in declared.unwrap_or_default() {
7807 if seen.insert(op.clone()) {
7808 effective.push(op);
7809 }
7810 }
7811 effective
7812}
7813
7814fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7815 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7816}
7817
7818fn target_module_id(target: &RouteTarget) -> &str {
7819 match target {
7820 RouteTarget::ToolProvider { module_id }
7821 | RouteTarget::ManagementSurface { module_id }
7822 | RouteTarget::InternalService { module_id, .. } => module_id,
7823 }
7824}
7825
7826fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7827 roles.iter().any(|role| match (target, role) {
7828 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7829 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7830 (
7831 RouteTarget::InternalService { service_id, .. },
7832 ProviderRole::InternalService {
7833 service_id: provided,
7834 ..
7835 },
7836 ) => service_id == provided,
7837 _ => false,
7838 })
7839}
7840
7841fn is_routable_role(role: &ProviderRole) -> bool {
7842 matches!(
7843 role,
7844 ProviderRole::ToolProvider { .. }
7845 | ProviderRole::ManagementSurface { .. }
7846 | ProviderRole::InternalService { .. }
7847 )
7848}
7849
7850#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7851enum ControlRequestBodyError {
7852 UnknownOp,
7853 InvalidBody,
7854}
7855
7856#[derive(Debug, Deserialize)]
7857struct ControlOpProbe {
7858 op: String,
7859}
7860
7861const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7864
7865fn is_known_module_push_op(body: &[u8]) -> bool {
7866 serde_json::from_slice::<ControlOpProbe>(body)
7867 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7868 .unwrap_or(false)
7869}
7870
7871fn is_known_module_request_op(body: &[u8]) -> bool {
7872 serde_json::from_slice::<ControlOpProbe>(body)
7873 .map(|probe| is_module_to_subc_op(&probe.op))
7874 .unwrap_or(false)
7875}
7876
7877fn is_module_to_subc_op(op: &str) -> bool {
7878 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7879}
7880
7881fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7882 debug!(
7883 op = %op,
7884 connection_id = connection_id.get(),
7885 corr,
7886 "control dispatch"
7887 );
7888}
7889
7890fn log_slow_control_dispatch(
7891 dispatch_started_at: Option<StdInstant>,
7892 op: &'static str,
7893 connection_id: ConnectionId,
7894 corr: u64,
7895 route_open_timings: Option<&RouteOpenTimings>,
7896) {
7897 let Some(dispatch_started_at) = dispatch_started_at else {
7898 return;
7899 };
7900 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7902 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7903 if let Some(timings) = route_open_timings {
7904 warn!(
7905 op = %op,
7906 connection_id = connection_id.get(),
7907 corr,
7908 elapsed_ms = elapsed.as_millis() as u64,
7909 module_id = ?timings.module_id,
7910 principal = %timings.principal,
7911 admission_ms = timings.admission.as_millis() as u64,
7912 project_root_ms = timings.project_root.as_millis() as u64,
7913 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7914 bind_send_ms = timings.bind_send.as_millis() as u64,
7915 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7916 "slow control dispatch"
7917 );
7918 return;
7919 }
7920 warn!(
7921 op = %op,
7922 connection_id = connection_id.get(),
7923 corr,
7924 elapsed_ms = elapsed.as_millis() as u64,
7925 "slow control dispatch"
7926 );
7927 }
7928}
7929
7930#[cfg(test)]
7932type TestCallerLookup =
7933 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7934
7935fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7937 match request {
7938 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7939 Some(PrivilegedOperation::Restart { module_id })
7940 }
7941 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7942 Some(PrivilegedOperation::Swap { module_id })
7943 }
7944 ClientControlRequest::SupervisorReload { module_id } => {
7945 Some(PrivilegedOperation::Reload { module_id })
7946 }
7947 ClientControlRequest::SupervisorRescan { preview } => {
7948 if *preview {
7949 None
7950 } else {
7951 Some(PrivilegedOperation::Rescan)
7952 }
7953 }
7954 ClientControlRequest::SupervisorRescanChecked {
7959 preview,
7960 candidate,
7961 expect,
7962 } => (!*preview && candidate.is_none() && expect.is_some())
7963 .then_some(PrivilegedOperation::Rescan),
7964 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7965 Some(PrivilegedOperation::SetEnabled {
7966 module_id,
7967 enabled: *enabled,
7968 })
7969 }
7970 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7971 Some(PrivilegedOperation::ReleaseReserved { module_id })
7972 }
7973 ClientControlRequest::ServerDescribe {}
7974 | ClientControlRequest::DaemonShutdownIntent { .. }
7977 | ClientControlRequest::CatalogList { .. }
7978 | ClientControlRequest::RouteOpen { .. }
7979 | ClientControlRequest::RoutePoll { .. }
7980 | ClientControlRequest::SupervisorList {}
7981 | ClientControlRequest::SupervisorHealth {}
7982 | ClientControlRequest::SupervisorHealthProbe { .. }
7983 | ClientControlRequest::SupervisorRoutes { .. }
7984 | ClientControlRequest::SupervisorProvenance { .. }
7985 | ClientControlRequest::SupervisorStderrTail { .. }
7986 | ClientControlRequest::SupervisorTerminals { .. }
7987 | ClientControlRequest::SupervisorSpawnSnapshot {}
7988 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7989 }
7990}
7991
7992fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7993 match request {
7994 ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
7995 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7996 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7997 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7998 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7999 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
8000 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
8001 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
8002 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
8003 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
8004 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
8005 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
8006 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
8007 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
8008 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
8009 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
8010 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
8011 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
8012 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
8013 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
8014 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
8015 }
8016}
8017
8018struct RouteOpenCaller {
8026 principal: String,
8030 scope: String,
8033 harness: String,
8036}
8037
8038impl RouteOpenCaller {
8039 fn new(
8040 principal: Option<&Principal>,
8041 scope: Option<&ScopeSelector>,
8042 identity: &BindIdentity,
8043 ) -> Self {
8044 Self {
8045 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
8046 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
8047 harness: bounded_log_text(&identity.harness),
8048 }
8049 }
8050
8051 fn unknown() -> Self {
8054 Self {
8055 principal: "unknown".to_string(),
8056 scope: "unknown".to_string(),
8057 harness: "unknown".to_string(),
8058 }
8059 }
8060}
8061
8062const MAX_LOGGED_TEXT_CHARS: usize = 64;
8065
8066fn bounded_log_text(value: &str) -> String {
8070 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
8071 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
8072 bounded.push('…');
8073 }
8074 format!("{bounded:?}")
8075}
8076
8077fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
8083 use subc_protocol::scope::ScopeKind;
8084 let kind = match stamp.kind {
8085 ScopeKind::Head => "head",
8086 ScopeKind::Worker => "worker",
8087 ScopeKind::Ephemeral => "ephemeral",
8088 };
8089 format!(
8090 "{kind}/{}/{}@{}",
8091 principal_label(&stamp.owner),
8092 bounded_log_text(&stamp.scope_ref),
8093 stamp.scope_epoch
8094 )
8095}
8096
8097fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
8104 let epoch = selector
8105 .scope_epoch
8106 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
8107 format!(
8108 "requested/{}/{}@{epoch}",
8109 bounded_log_text(&principal_label(&selector.owner)),
8110 bounded_log_text(&selector.scope_ref),
8111 )
8112}
8113
8114fn principal_label(principal: &Principal) -> String {
8115 match principal {
8116 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
8117 Principal::Direct => "direct".to_string(),
8118 other => format!("{other:?}"),
8119 }
8120}
8121
8122fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
8123 match request {
8124 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
8125 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
8126 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
8127 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
8128 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
8129 }
8130}
8131
8132fn parse_client_control_request(
8133 body: &[u8],
8134) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
8135 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
8136 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8137 Ok(probe)
8138 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
8139 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
8140 {
8141 ControlRequestBodyError::InvalidBody
8142 }
8143 Ok(_) => ControlRequestBodyError::UnknownOp,
8144 Err(_) => ControlRequestBodyError::InvalidBody,
8145 };
8146 (err, classification)
8147 })
8148}
8149
8150fn parse_module_control_request_from_module(
8151 body: &[u8],
8152) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
8153 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
8154 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8155 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
8156 Ok(_) => ControlRequestBodyError::UnknownOp,
8157 Err(_) => ControlRequestBodyError::InvalidBody,
8158 };
8159 (err, classification)
8160 })
8161}
8162
8163#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
8164enum ProviderRoleKind {
8165 ToolProvider,
8166 PipelineStage,
8167 ManagementSurface,
8168 InternalService,
8169}
8170
8171fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
8172 match role {
8173 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
8174 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
8175 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
8176 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
8177 }
8178}
8179
8180fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
8181 roles.iter().map(provider_role_kind).collect()
8182}
8183
8184const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
8187
8188#[derive(Debug, Default, PartialEq, Eq)]
8190struct ScopeOutcomeCounts {
8191 created: usize,
8192 replaced: usize,
8193 updated: usize,
8194 unchanged: usize,
8195 refused: usize,
8196}
8197
8198impl ScopeOutcomeCounts {
8199 fn of(results: &[ScopeRecordResult]) -> Self {
8200 let mut counts = Self::default();
8201 for result in results {
8202 let slot = match result.outcome {
8203 ScopeRecordOutcome::Created => &mut counts.created,
8204 ScopeRecordOutcome::Replaced => &mut counts.replaced,
8205 ScopeRecordOutcome::Updated => &mut counts.updated,
8206 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
8207 ScopeRecordOutcome::Refused => &mut counts.refused,
8208 };
8209 *slot += 1;
8210 }
8211 counts
8212 }
8213}
8214
8215#[cfg(test)]
8216mod scope_outcome_count_tests {
8217 use super::*;
8218
8219 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
8220 ScopeRecordResult {
8221 scope_ref: "r".to_string(),
8222 scope_epoch: 1,
8223 outcome,
8224 code: None,
8225 message: None,
8226 version: None,
8227 parent_state: None,
8228 }
8229 }
8230
8231 #[test]
8234 fn every_outcome_is_counted_in_its_own_field() {
8235 let results = [
8236 result(ScopeRecordOutcome::Created),
8237 result(ScopeRecordOutcome::Created),
8238 result(ScopeRecordOutcome::Replaced),
8239 result(ScopeRecordOutcome::Updated),
8240 result(ScopeRecordOutcome::Unchanged),
8241 result(ScopeRecordOutcome::Refused),
8242 result(ScopeRecordOutcome::Refused),
8243 result(ScopeRecordOutcome::Refused),
8244 ];
8245 assert_eq!(
8246 ScopeOutcomeCounts::of(&results),
8247 ScopeOutcomeCounts {
8248 created: 2,
8249 replaced: 1,
8250 updated: 1,
8251 unchanged: 1,
8252 refused: 3,
8253 }
8254 );
8255 }
8256}
8257
8258fn capability_census_trigger(
8262 old: Option<&CapabilityDeclarations>,
8263 new: Option<&CapabilityDeclarations>,
8264) -> bool {
8265 let old_provides = old
8266 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
8267 .unwrap_or_default();
8268 let old_denies = old
8269 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
8270 .unwrap_or_default();
8271 let new = new.cloned().unwrap_or(CapabilityDeclarations {
8272 provides: Vec::new(),
8273 requires: Vec::new(),
8274 must_never_reach: Vec::new(),
8275 });
8276
8277 new.provides
8278 .iter()
8279 .any(|capability| !old_provides.contains(capability))
8280 || new
8281 .must_never_reach
8282 .iter()
8283 .any(|capability| !old_denies.contains(capability))
8284}
8285
8286fn denied_capability<'a>(
8289 opening_manifest: &'a ModuleManifest,
8290 target_manifest: &ModuleManifest,
8291) -> Option<&'a str> {
8292 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
8293 let target_capabilities = target_manifest.capabilities.as_ref()?;
8294 opening_capabilities
8295 .must_never_reach
8296 .iter()
8297 .find(|denied| {
8298 target_capabilities
8299 .provides
8300 .iter()
8301 .any(|provided| provided == *denied)
8302 })
8303 .map(String::as_str)
8304}
8305
8306fn catalog_update_frozen_field_message(
8307 registered: &ModuleManifest,
8308 provides: &[ProviderRole],
8309) -> Option<String> {
8310 let old_has_provides = !registered.provides.is_empty();
8311 let new_has_provides = !provides.is_empty();
8312 if old_has_provides != new_has_provides {
8313 return Some(format!(
8314 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
8315 registered.module_id
8316 ));
8317 }
8318
8319 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
8320 return Some(format!(
8321 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
8322 registered.module_id
8323 ));
8324 }
8325
8326 let registered_concurrency = manifest_concurrency(registered);
8327 let mut candidate = registered.clone();
8328 candidate.provides = provides.to_vec();
8329 let candidate_concurrency = manifest_concurrency(&candidate);
8330 if candidate_concurrency != registered_concurrency {
8331 return Some(format!(
8332 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
8333 registered.module_id, registered_concurrency, candidate_concurrency
8334 ));
8335 }
8336
8337 None
8340}
8341
8342fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
8343 manifest.provides.iter().any(is_routable_role)
8344}
8345
8346fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
8352 manifest
8353 .provides
8354 .iter()
8355 .find_map(|provider| match provider {
8356 ProviderRole::ToolProvider { concurrency, .. }
8357 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
8358 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
8359 })
8360 .unwrap_or(Concurrency::ModuleManaged)
8361}
8362
8363fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
8370 let has_management_surface = manifest
8371 .provides
8372 .iter()
8373 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
8374 if !has_management_surface {
8375 return false;
8376 }
8377 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
8378 return false;
8379 };
8380 let Some(provides) = raw
8381 .get("manifest")
8382 .and_then(|manifest| manifest.get("provides"))
8383 .and_then(serde_json::Value::as_array)
8384 else {
8385 return false;
8386 };
8387 provides.iter().any(|role| {
8392 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8393 && role.get("concurrency").is_none()
8394 })
8395}
8396
8397fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8398 if peer_version != PROTOCOL_VERSION {
8399 return Err(format!(
8400 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8401 ));
8402 }
8403 Ok(PROTOCOL_VERSION)
8404}
8405
8406fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8407 Frame::build_with_version(
8408 response_version(frame),
8409 FrameType::Pong,
8410 frame.header.flags,
8411 0,
8412 0,
8413 frame.header.corr,
8414 Vec::new(),
8415 )
8416 .map_err(RouterError::FrameBuild)
8417}
8418
8419pub(crate) fn control_error_frame(
8420 frame: &Frame,
8421 code: &'static str,
8422 message: impl Into<String>,
8423) -> Result<Frame, RouterError> {
8424 control_error_body_frame(
8425 frame,
8426 ErrorBody {
8427 code: code.to_string(),
8428 message: message.into(),
8429 detail: None,
8430 },
8431 )
8432}
8433
8434fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8442 control_error_frame_with_flags(frame, error, daemon_error_flags())
8443}
8444
8445fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8451 control_error_frame_with_flags(frame, error, control_flags())
8452}
8453
8454fn control_error_frame_with_flags(
8455 frame: &Frame,
8456 error: ErrorBody,
8457 flags: Flags,
8458) -> Result<Frame, RouterError> {
8459 let body = serde_json::to_vec(&error).map_err(|err| {
8460 RouterError::backend(
8461 0,
8462 frame.header.corr,
8463 format!("failed to encode control ERROR: {err}"),
8464 )
8465 })?;
8466
8467 Frame::build_with_version(
8468 response_version(frame),
8469 FrameType::Error,
8470 flags,
8471 0,
8472 0,
8473 frame.header.corr,
8474 body,
8475 )
8476 .map_err(RouterError::FrameBuild)
8477}
8478
8479pub(crate) fn control_response_body_frame<T: Serialize>(
8480 frame: &Frame,
8481 reply: &T,
8482 label: &'static str,
8483) -> Result<Frame, RouterError> {
8484 let body = serde_json::to_vec(reply).map_err(|err| {
8485 RouterError::backend(
8486 0,
8487 frame.header.corr,
8488 format!("failed to encode {label}: {err}"),
8489 )
8490 })?;
8491
8492 Frame::build_with_version(
8493 response_version(frame),
8494 FrameType::Response,
8495 control_flags(),
8496 0,
8497 0,
8498 frame.header.corr,
8499 body,
8500 )
8501 .map_err(RouterError::FrameBuild)
8502}
8503
8504fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8534 match err {
8535 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8536 ForwardingError::NoModuleConnection => "target_unavailable",
8537 ForwardingError::ModuleReloading { .. } => "module_reloading",
8538 ForwardingError::ClientRouteChannelExhausted { .. }
8539 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8540 ForwardingError::StaleModuleEndpoint
8541 | ForwardingError::UnknownReservation { .. }
8542 | ForwardingError::ConnectionClosing { .. }
8543 | ForwardingError::ClientEgressClosed { .. }
8544 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8545 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8548 ForwardingError::RelayCorrelationExhausted
8549 | ForwardingError::RouteOpenBuild(_)
8550 | ForwardingError::Poisoned => "forwarding_error",
8551 }
8552}
8553
8554fn response_version(frame: &Frame) -> u8 {
8555 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8556 frame.header.ver
8557 } else {
8558 PROTOCOL_VERSION
8559 }
8560}
8561
8562fn control_flags() -> Flags {
8563 Flags::new(false, Priority::Passive, false)
8564}
8565
8566fn send_goodbye_target_best_effort(
8571 counters: &DaemonCounters,
8572 target: &GoodbyeTarget,
8573 context: &'static str,
8574) {
8575 let Ok(frame) = Frame::build_with_version(
8576 target.negotiated_ver,
8577 FrameType::Goodbye,
8578 control_flags(),
8579 target.channel,
8580 target.epoch,
8581 0,
8582 Vec::new(),
8583 ) else {
8584 return;
8585 };
8586 crate::forwarding::send_module_route_goodbye(
8587 counters,
8588 &target.sink,
8589 frame,
8590 target.module_id.as_deref(),
8591 context,
8592 );
8593}
8594
8595pub(crate) fn send_route_control_pushes(
8596 forwarding: &ForwardingTable,
8597 routes: Vec<EndpointRoute>,
8598 push: ClientControlPush,
8599) {
8600 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8601 for route in routes {
8602 let target = route.goodbye_target;
8603 if let Some((existing, channels)) = targets
8604 .iter_mut()
8605 .find(|(existing, _)| existing.connection_id == target.connection_id)
8606 {
8607 debug_assert_eq!(
8608 existing.negotiated_ver, target.negotiated_ver,
8609 "one connection cannot negotiate multiple frame versions"
8610 );
8611 if !channels.contains(&target.channel) {
8612 channels.push(target.channel);
8613 }
8614 continue;
8615 }
8616 let channel = target.channel;
8617 targets.push((target, vec![channel]));
8618 }
8619 for (target, mut channels) in targets {
8620 channels.sort_unstable();
8621 let mut push = push.clone();
8622 match &mut push {
8623 ClientControlPush::RouteClosing {
8624 channels: covered, ..
8625 }
8626 | ClientControlPush::RouteClosed {
8627 channels: covered, ..
8628 } => *covered = channels,
8629 }
8630 let body = match serde_json::to_vec(&push) {
8631 Ok(body) => body,
8632 Err(err) => {
8633 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8634 continue;
8635 }
8636 };
8637 let frame = match Frame::build_with_version(
8638 target.negotiated_ver,
8639 FrameType::Push,
8640 control_flags(),
8641 0,
8642 0,
8643 0,
8644 body.clone(),
8645 ) {
8646 Ok(frame) => frame,
8647 Err(err) => {
8648 warn!(
8649 route_channel = target.channel,
8650 error = %err,
8651 "failed to build route lifecycle control PUSH frame"
8652 );
8653 continue;
8654 }
8655 };
8656 if let Err(err) = target.sink.try_send(frame) {
8657 if target.close_on_delivery_failure() {
8658 warn!(
8659 target_connection_id = target.connection_id.get(),
8660 route_channel = target.channel,
8661 error = %err,
8662 "route lifecycle control PUSH was not delivered to client; closing target connection"
8663 );
8664 let _ = forwarding.escalate_client_delivery_failure(
8665 target.connection_id,
8666 target.channel,
8667 target.epoch,
8668 CloseReason::new(
8669 "route_lifecycle_push_delivery_failed",
8670 format!(
8671 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8672 target.channel
8673 ),
8674 ),
8675 crate::forwarding::UndeliveredFrame {
8676 module_id: target.module_id.as_deref(),
8677 sink: &target.sink,
8678 },
8679 );
8680 }
8681 }
8682 }
8683}
8684
8685#[cfg(test)]
8686mod tests {
8687 #[tokio::test]
8688 async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
8689 let registry = Arc::new(Registry::default());
8690 let handler = ControlHandler::new(registry);
8691 assert_eq!(
8692 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8693 None
8694 );
8695 let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
8696 hello_via_sink(
8697 &handler,
8698 &ctx,
8699 &mut rx,
8700 hello_frame("daemon", PROTOCOL_VERSION, 1),
8701 )
8702 .await;
8703 assert_eq!(
8704 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8705 Some("daemon".into())
8706 );
8707 assert_eq!(
8708 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
8709 None
8710 );
8711 assert_eq!(
8712 handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
8713 None,
8714 "a registered module is not necessarily supervised"
8715 );
8716 assert_eq!(
8717 handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
8718 None,
8719 "aggregate provenance has no single target"
8720 );
8721 assert_eq!(
8722 handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
8723 None
8724 );
8725 }
8726
8727 mod peer_manifest {
8728 include!("peer_manifest_tests.rs");
8729 }
8730 mod activity_read {
8731 include!("activity_read_tests.rs");
8732 }
8733
8734 mod lifecycle_activity {
8735 include!("control_lifecycle_tests.rs");
8736 }
8737 #[cfg(unix)]
8738 #[tokio::test]
8739 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8740 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8741 let path = dir.join("subc.jsonc");
8742 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8743 "program":"/bin/sleep","args":["60"],"protocol":"none",
8744 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8745 }}}).to_string()).unwrap();
8746 let mut configured = crate::daemon_config::load(&path)
8747 .unwrap()
8748 .unwrap()
8749 .modules
8750 .pop()
8751 .unwrap();
8752 let registry = std::sync::Arc::new(crate::Registry::default());
8753 let handle = crate::SupervisorHandle::new();
8754 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8755 .with_handle(handle.clone());
8756 let module = supervisor
8757 .supervise_configured_with_health(
8758 configured.module_spec(),
8759 true,
8760 configured.health.clone(),
8761 None,
8762 configured.restart,
8763 )
8764 .unwrap();
8765 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8766 let before = module.status().unwrap().pid;
8767 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8768 configured.health.cadence = std::time::Duration::from_secs(3600);
8769 let health_only = handler
8770 .reconcile_supervised_modules(
8771 &supervisor,
8772 vec![configured.clone()],
8773 false,
8774 Principal::Direct,
8775 )
8776 .await
8777 .unwrap();
8778 assert!(
8779 health_only.0.changed_pending_reload.is_empty(),
8780 "health policy is already applied live"
8781 );
8782 assert_eq!(module.status().unwrap().pid, before);
8783 assert_eq!(
8784 module.configuration().unwrap().1.http,
8785 configured.health.http
8786 );
8787 configured.args = vec!["61".into()];
8788 let launch = handler
8789 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8790 .await
8791 .unwrap();
8792 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8793 assert_eq!(
8794 module.status().unwrap().pid,
8795 before,
8796 "a launch edit is stored until reload"
8797 );
8798 module.drain().await.unwrap();
8799 }
8800 use cortexkit_test_support::ScratchDir;
8801 use std::{
8802 collections::BTreeMap,
8803 fmt,
8804 path::PathBuf,
8805 sync::{Arc, Mutex},
8806 time::Duration,
8807 };
8808
8809 use serde_json::{json, Value};
8810 use subc_protocol::{
8811 manifest::{
8812 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8813 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8814 },
8815 session::HealthStatus,
8816 FrameType,
8817 };
8818
8819 use super::*;
8820 use crate::{
8821 forwarding::{DataRoute, DataRouteState},
8822 registry::ChannelState,
8823 router::FrameSink,
8824 stderr_tail::DEFAULT_MAX_LINE_BYTES,
8825 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8826 RouteCtx, Router,
8827 };
8828 use tokio::{
8829 sync::mpsc,
8830 time::{sleep, Instant},
8831 };
8832 use tracing::{
8833 field::{Field, Visit},
8834 Event, Subscriber,
8835 };
8836 use tracing_subscriber::{layer::Context, prelude::*, Layer};
8837
8838 fn fake_aft_stub_path() -> PathBuf {
8857 let mut path = std::env::current_exe().expect("current_exe available in tests");
8858 path.pop(); path.pop(); path.push(if cfg!(windows) {
8861 "fake-aft-stub.exe"
8862 } else {
8863 "fake-aft-stub"
8864 });
8865 assert!(
8866 path.exists(),
8867 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8868 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8869 path.display()
8870 );
8871 path
8872 }
8873
8874 fn client_retries(code: &str) -> bool {
8879 subc_protocol::error_codes::is_retryable_route_open(code)
8880 }
8881
8882 #[test]
8893 fn retryability_of_forwarding_codes_matches_the_failure() {
8894 let transient = [
8897 ForwardingError::NoModuleConnection,
8898 ForwardingError::ModuleReloading {
8899 module_id: "m".into(),
8900 },
8901 ForwardingError::StaleModuleEndpoint,
8902 ForwardingError::UnknownReservation {
8903 client_channel: 1,
8904 module_channel: 1,
8905 },
8906 ForwardingError::ConnectionClosing {
8907 connection_id: ConnectionId::new(1),
8908 },
8909 ForwardingError::ClientEgressClosed {
8910 connection_id: ConnectionId::new(1),
8911 },
8912 ForwardingError::ModuleEgressUnavailable {
8913 connection_id: ConnectionId::new(1),
8914 },
8915 ];
8916 for err in transient {
8917 let code = forwarding_error_code(&err);
8918 assert!(
8919 client_retries(code),
8920 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8921 );
8922 }
8923
8924 let permanent = [
8929 ForwardingError::ConnectionRoleConflict {
8930 connection_id: ConnectionId::new(1),
8931 },
8932 ForwardingError::ClientRouteChannelExhausted {
8933 connection_id: ConnectionId::new(1),
8934 },
8935 ForwardingError::ModuleRouteChannelExhausted {
8936 endpoint: ModuleEndpointId {
8937 connection_id: ConnectionId::new(1),
8938 generation: 1,
8939 },
8940 },
8941 ForwardingError::RelayCorrelationExhausted,
8942 ForwardingError::RouteOpenBuild("x".into()),
8943 ForwardingError::Poisoned,
8944 ];
8945 for err in permanent {
8946 let code = forwarding_error_code(&err);
8947 assert!(
8948 !client_retries(code),
8949 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8950 );
8951 }
8952 }
8953
8954 #[tokio::test]
8982 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8983 let handler = ControlHandler::default();
8984 let frame =
8985 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8986
8987 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8991 assert_eq!(
8992 stamped,
8993 Principal::Direct,
8994 "a caller that proved nothing must not be stamped as a supervised module"
8995 );
8996
8997 let forged = handler
9002 .route_open_principal(
9003 &frame,
9004 Some(ConsumerIdentity {
9005 module_id: "aft".to_string(),
9006 launch_nonce: "not-a-real-nonce".to_string(),
9007 }),
9008 )
9009 .unwrap();
9010 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
9011 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
9012 }
9013
9014 #[test]
9031 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
9032 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
9033 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
9034 let ClientControlRequest::RouteOpen {
9035 consumer_identity, ..
9036 } = parsed
9037 else {
9038 panic!("route.open body must parse as RouteOpen");
9039 };
9040 assert_eq!(
9041 consumer_identity,
9042 Some(ConsumerIdentity {
9043 module_id: "aft".to_string(),
9044 launch_nonce: "n".to_string(),
9045 }),
9046 "the wire field name must reach the value route_open_principal reads"
9047 );
9048 }
9049
9050 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
9051 ModuleManifest::builder(module_id, "0.1.0")
9052 .protocol_ver(protocol_ver)
9053 .provides(vec![ProviderRole::ToolProvider {
9054 tools: vec![Tool {
9055 name: "read".to_string(),
9056 description: None,
9057 execution_mode: ExecutionMode::Pure,
9058 schema: json!({"type": "object"}),
9059 }],
9060 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
9061 concurrency: Concurrency::ModuleManaged,
9062 emits_push: true,
9063 sub_supervises: true,
9064 }])
9065 .build()
9066 }
9067
9068 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
9069 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
9070 }
9071
9072 fn hello_frame_with_control_ops(
9073 module_id: &str,
9074 protocol_ver: u8,
9075 corr: u64,
9076 control_ops: Option<Vec<String>>,
9077 ) -> Frame {
9078 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
9079 }
9080
9081 fn hello_frame_with_nonce(
9082 module_id: &str,
9083 protocol_ver: u8,
9084 corr: u64,
9085 launch_nonce: Option<&str>,
9086 ) -> Frame {
9087 hello_frame_full(
9088 module_id,
9089 protocol_ver,
9090 corr,
9091 None,
9092 launch_nonce.map(ToOwned::to_owned),
9093 )
9094 }
9095
9096 fn hello_frame_full(
9097 module_id: &str,
9098 protocol_ver: u8,
9099 corr: u64,
9100 control_ops: Option<Vec<String>>,
9101 launch_nonce: Option<String>,
9102 ) -> Frame {
9103 let body = serde_json::to_vec(&ModuleHelloBody {
9104 manifest: manifest(module_id, protocol_ver),
9105 protocol_ver,
9106 control_ops,
9107 launch_nonce,
9108 })
9109 .unwrap();
9110 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9111 }
9112
9113 fn hello_frame_with_rich_ui(
9114 module_id: &str,
9115 protocol_ver: u8,
9116 corr: u64,
9117 launch_nonce: Option<&str>,
9118 rich_ui: Value,
9119 ) -> Frame {
9120 let mut body = serde_json::to_value(ModuleHelloBody {
9121 manifest: manifest(module_id, protocol_ver),
9122 protocol_ver,
9123 control_ops: None,
9124 launch_nonce: launch_nonce.map(str::to_string),
9125 })
9126 .unwrap();
9127 body["manifest"]["rich_ui"] = rich_ui;
9128 Frame::build(
9129 FrameType::Hello,
9130 control_flags(),
9131 0,
9132 0,
9133 corr,
9134 serde_json::to_vec(&body).unwrap(),
9135 )
9136 .unwrap()
9137 }
9138
9139 fn non_routable_hello_frame_with_control_ops(
9140 module_id: &str,
9141 corr: u64,
9142 control_ops: Option<Vec<String>>,
9143 ) -> Frame {
9144 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9145 manifest.provides.clear();
9146 let body = serde_json::to_vec(&ModuleHelloBody {
9147 manifest,
9148 protocol_ver: PROTOCOL_VERSION,
9149 control_ops,
9150 launch_nonce: None,
9151 })
9152 .unwrap();
9153 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9154 }
9155
9156 fn capability_grammar_hello_frame(
9157 capabilities: Value,
9158 runtime_computed: Option<Value>,
9159 corr: u64,
9160 ) -> Frame {
9161 let mut body = serde_json::to_value(ModuleHelloBody {
9162 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
9163 protocol_ver: PROTOCOL_VERSION,
9164 control_ops: None,
9165 launch_nonce: None,
9166 })
9167 .expect("HELLO body serializes");
9168 body["manifest"]["capabilities"] = capabilities;
9169 if let Some(runtime_computed) = runtime_computed {
9170 body["runtime_computed"] = runtime_computed;
9171 }
9172 Frame::build(
9173 FrameType::Hello,
9174 control_flags(),
9175 0,
9176 0,
9177 corr,
9178 serde_json::to_vec(&body).expect("HELLO body reserializes"),
9179 )
9180 .expect("HELLO frame builds")
9181 }
9182
9183 fn channel_request(channel: u16, corr: u64) -> Frame {
9184 Frame::build(
9185 FrameType::Request,
9186 Flags::new(true, Priority::Interactive, false),
9187 channel,
9188 0,
9189 corr,
9190 b"opaque".to_vec(),
9191 )
9192 .unwrap()
9193 }
9194
9195 fn route_ctx(
9196 connection_id: ConnectionId,
9197 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
9198 let (tx, rx) = mpsc::channel(8);
9199 (
9200 RouteCtx {
9201 connection_id,
9202 egress: FrameSink::new(tx),
9203 },
9204 rx,
9205 )
9206 }
9207
9208 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
9209 serde_json::from_slice(&frame.body).unwrap()
9210 }
9211
9212 async fn hello_via_sink(
9217 handler: &ControlHandler,
9218 ctx: &RouteCtx,
9219 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9220 hello: Frame,
9221 ) -> Frame {
9222 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
9223 assert!(
9224 replies.is_empty(),
9225 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
9226 );
9227 let ack = rx
9228 .try_recv()
9229 .expect("HELLO_ACK is queued on the module sink")
9230 .frame;
9231 assert_eq!(ack.header.ty, FrameType::HelloAck);
9232 ack
9233 }
9234
9235 fn parse_error(frame: &Frame) -> Value {
9236 serde_json::from_slice(&frame.body).unwrap()
9237 }
9238
9239 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
9240 serde_json::from_slice(&frame.body).unwrap()
9241 }
9242
9243 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
9244 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
9245 route_channel,
9246 route_epoch: 0,
9247 kind,
9248 })
9249 .unwrap();
9250 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9251 }
9252
9253 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
9254 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
9255 module_id: module_id.to_string(),
9256 })
9257 .unwrap();
9258 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9259 }
9260
9261 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
9262 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
9263 }
9264
9265 fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
9267 let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
9268 let mut frame = route_open_frame(corr, module_id, root);
9269 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
9270 body["identity"]["project_root"] = serde_json::json!(path.as_path());
9271 frame.body = serde_json::to_vec(&body).unwrap();
9272 frame
9273 }
9274
9275 fn route_open_frame_with_consumer_capabilities(
9276 corr: u64,
9277 module_id: &str,
9278 _project_root: ScratchDir,
9279 consumer_capabilities: Option<Vec<String>>,
9280 ) -> Frame {
9281 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9282 busy_refusals: None,
9283 target: RouteTarget::ToolProvider {
9284 module_id: module_id.to_string(),
9285 },
9286 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
9287 consumer_identity: None,
9288 consumer_capabilities,
9289 role_versions: None,
9290 admission_facts: None,
9291 scope: None,
9292 })
9293 .unwrap();
9294 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9295 }
9296
9297 fn route_open_frame_with_role_versions(
9298 corr: u64,
9299 module_id: &str,
9300 _project_root: ScratchDir,
9301 role_versions: Option<BTreeMap<String, String>>,
9302 ) -> Frame {
9303 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9304 busy_refusals: None,
9305 target: RouteTarget::ToolProvider {
9306 module_id: module_id.to_string(),
9307 },
9308 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9309 consumer_identity: None,
9310 consumer_capabilities: None,
9311 role_versions,
9312 admission_facts: None,
9313 scope: None,
9314 })
9315 .unwrap();
9316 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9317 }
9318
9319 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
9320 entries
9321 .iter()
9322 .map(|(role, version)| (role.to_string(), version.to_string()))
9323 .collect()
9324 }
9325
9326 fn route_open_frame_with_admission_facts(
9327 corr: u64,
9328 module_id: &str,
9329 _project_root: ScratchDir,
9330 consumer_identity: Option<subc_control::ConsumerIdentity>,
9331 facts: Option<Value>,
9332 ) -> Frame {
9333 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9334 busy_refusals: None,
9335 target: RouteTarget::ToolProvider {
9336 module_id: module_id.to_string(),
9337 },
9338 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9339 consumer_identity,
9340 consumer_capabilities: None,
9341 role_versions: None,
9342 admission_facts: facts,
9343 scope: None,
9344 })
9345 .unwrap();
9346 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9347 }
9348
9349 #[derive(Clone, Default)]
9350 struct EventCapture {
9351 events: Arc<Mutex<Vec<CapturedEvent>>>,
9352 }
9353
9354 #[derive(Clone, Debug)]
9355 struct CapturedEvent {
9356 target: String,
9357 level: tracing::Level,
9358 fields: BTreeMap<String, String>,
9359 }
9360
9361 impl EventCapture {
9362 fn events(&self) -> Vec<CapturedEvent> {
9363 self.events.lock().unwrap().clone()
9364 }
9365 }
9366
9367 impl<S> Layer<S> for EventCapture
9368 where
9369 S: Subscriber,
9370 {
9371 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
9372 let mut visitor = EventFieldVisitor::default();
9373 event.record(&mut visitor);
9374 self.events.lock().unwrap().push(CapturedEvent {
9375 target: event.metadata().target().to_string(),
9376 level: *event.metadata().level(),
9377 fields: visitor.fields,
9378 });
9379 }
9380 }
9381
9382 #[derive(Default)]
9383 struct EventFieldVisitor {
9384 fields: BTreeMap<String, String>,
9385 }
9386
9387 impl Visit for EventFieldVisitor {
9388 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
9389 self.fields
9390 .insert(field.name().to_string(), format!("{value:?}"));
9391 }
9392 }
9393
9394 fn health_response(corr: u64, status: HealthStatus) -> Frame {
9395 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
9396 status,
9397 detail: Some("warming".to_string()),
9398 metrics: Some(json!({"queue_depth": 3})),
9399 })
9400 .unwrap();
9401 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9402 }
9403
9404 fn route_bind_ack(corr: u64) -> Frame {
9405 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
9406 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9407 }
9408
9409 fn unique_project_root(label: &str) -> ScratchDir {
9410 ScratchDir::new(label)
9411 }
9412
9413 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
9414 match parse_route_poll(frame) {
9415 ClientControlResponse::RoutePoll {
9416 status: None,
9417 live: Some(live),
9418 ..
9419 } => assert_eq!(live, expected_live),
9420 other => panic!("unexpected route.poll response: {other:?}"),
9421 }
9422 }
9423
9424 fn bind_liveness_route(
9425 registry: &Registry,
9426 forwarding: &ForwardingTable,
9427 module_id: &str,
9428 ) -> (RouteCtx, u16, u32) {
9429 let module_connection = ConnectionId::new(101);
9430 let client_connection = ConnectionId::new(202);
9431 let registration = registry
9432 .register_with_control_ops(
9433 manifest(module_id, PROTOCOL_VERSION),
9434 PROTOCOL_VERSION,
9435 module_connection,
9436 module_baseline_control_ops(),
9437 )
9438 .unwrap();
9439 let (module_tx, _module_rx) = mpsc::channel(8);
9440 let endpoint = forwarding
9441 .register_module_connection(
9442 module_connection,
9443 module_id.to_string(),
9444 PROTOCOL_VERSION,
9445 manifest_concurrency(®istration.manifest),
9446 FrameSink::new(module_tx),
9447 )
9448 .unwrap();
9449 let (client_ctx, _client_rx) = route_ctx(client_connection);
9450 let pending = forwarding
9451 .begin_route_bind_relay_for_test(
9452 client_connection,
9453 client_ctx.egress.clone(),
9454 1,
9455 module_id,
9456 )
9457 .unwrap();
9458 assert_eq!(pending.endpoint, endpoint);
9459 let route_channel = pending.client_channel;
9460 let route_epoch = pending.client_epoch;
9461 forwarding
9462 .complete_pending_relay(
9463 module_connection,
9464 pending.corr,
9465 RouteBindRelayOutcome::Accepted,
9466 )
9467 .unwrap();
9468 (client_ctx, route_channel, route_epoch)
9469 }
9470
9471 struct FakeProcessLiveness {
9472 live: Option<bool>,
9473 }
9474
9475 impl ModuleProcessLiveness for FakeProcessLiveness {
9476 fn process_live(&self, _module_id: &str) -> Option<bool> {
9477 self.live
9478 }
9479 }
9480
9481 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9482 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9483 {
9484 let registry = Arc::new(Registry::default());
9485 let supervisor_handle = SupervisorHandle::new();
9486 let supervisor = Supervisor::new_for_test(
9487 Arc::clone(®istry),
9488 RestartPolicy::new(1, Duration::from_millis(10)),
9489 )
9490 .with_handle(supervisor_handle.clone());
9491 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9492 let module = supervisor
9493 .spawn(ModuleSpec {
9494 carry_containment: false,
9495 module_id: "stderr-tail-wire".to_string(),
9496 program: fake_aft_stub_path(),
9497 args: Vec::new(),
9498 env: vec![
9499 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9500 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9501 ],
9502 reserved: false,
9503 reserved_prefixes: Vec::new(),
9504 protocol: ModuleProtocol::Subc,
9505 overlap: Default::default(),
9506 })
9507 .unwrap();
9508
9509 let deadline = Instant::now() + Duration::from_secs(5);
9510 loop {
9511 let tail = module.stderr_tail(None, None);
9512 if tail
9513 .entries
9514 .iter()
9515 .any(|entry| matches!(entry, TailEntry::ProcessStart))
9516 && tail.entries.iter().any(|entry| {
9517 matches!(
9518 entry,
9519 TailEntry::Line {
9520 truncated: true,
9521 ..
9522 }
9523 )
9524 })
9525 {
9526 break;
9527 }
9528 assert!(
9529 Instant::now() < deadline,
9530 "module did not produce a truncated line and restart boundary: {tail:?}"
9531 );
9532 sleep(Duration::from_millis(10)).await;
9533 }
9534
9535 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9536 let request = ClientControlRequest::SupervisorStderrTail {
9537 module_id: "stderr-tail-wire".to_string(),
9538 max_lines: None,
9539 max_bytes: None,
9540 };
9541 let frame = Frame::build(
9542 FrameType::Request,
9543 control_flags(),
9544 0,
9545 0,
9546 1,
9547 serde_json::to_vec(&request).unwrap(),
9548 )
9549 .unwrap();
9550 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9551 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9552 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9553 serde_json::from_slice(&responses[0].body).unwrap()
9554 else {
9555 panic!("expected supervisor.stderr_tail response");
9556 };
9557
9558 assert!(
9559 tail.entries
9560 .iter()
9561 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9562 "the control response lost the restart boundary"
9563 );
9564 let Some(StderrTailEntry::Line {
9565 text,
9566 truncated,
9567 at_ms,
9568 }) = tail.entries.iter().find(|entry| {
9569 matches!(
9570 entry,
9571 StderrTailEntry::Line {
9572 truncated: true,
9573 ..
9574 }
9575 )
9576 })
9577 else {
9578 panic!("the control response lost the truncated line");
9579 };
9580 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9581 assert!(*truncated);
9582 assert!(
9583 at_ms.is_some(),
9584 "the control response lost the line's capture time"
9585 );
9586 }
9587
9588 #[tokio::test(flavor = "current_thread")]
9593 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9594 let dir = ScratchDir::new("terminals-off-worker");
9595 let journal_path = dir.join("terminals.jsonl");
9596 let registry = Arc::new(Registry::default());
9597 let supervisor_handle = SupervisorHandle::new();
9598 let supervisor =
9599 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9600 .with_handle(supervisor_handle.clone())
9601 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9602 let module = supervisor
9603 .spawn(ModuleSpec {
9604 carry_containment: false,
9605 module_id: "terminal-off-worker".to_string(),
9606 program: fake_aft_stub_path(),
9607 args: Vec::new(),
9608 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9609 reserved: false,
9610 reserved_prefixes: Vec::new(),
9611 protocol: ModuleProtocol::Subc,
9612 overlap: Default::default(),
9613 })
9614 .unwrap();
9615 let deadline = Instant::now() + Duration::from_secs(5);
9616 while module.terminal_history().entries.len() != 2 {
9617 assert!(Instant::now() < deadline, "module did not record two exits");
9618 sleep(Duration::from_millis(10)).await;
9619 }
9620
9621 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9622 let handler =
9623 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
9624 let frame = Frame::build(
9625 FrameType::Request,
9626 control_flags(),
9627 0,
9628 0,
9629 1,
9630 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9631 module_id: "terminal-off-worker".to_string(),
9632 })
9633 .unwrap(),
9634 )
9635 .unwrap();
9636 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9637 let spawned_at = std::time::Instant::now();
9638 let read = tokio::spawn({
9639 let handler = Arc::clone(&handler);
9640 async move { handler.handle_control_frame(&ctx, frame).await }
9641 });
9642 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9645 .await
9646 .unwrap()
9647 .expect("the history read reached its pause");
9648 let elapsed = spawned_at.elapsed();
9649 assert!(
9650 elapsed < Duration::from_secs(2) && !read.is_finished(),
9651 "this task could not run while the history read was paused \
9652 (resumed after {elapsed:?}, read finished: {})",
9653 read.is_finished()
9654 );
9655
9656 drop(release);
9657 let responses = read.await.unwrap().unwrap();
9658 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9659 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9660 panic!("expected supervisor.terminals response");
9661 };
9662 assert_eq!(terminals.entries.len(), 2);
9663 assert_eq!(terminals.journal_skipped_lines, 0);
9664 assert_eq!(terminals.journal_read_errors, 0);
9665 }
9666
9667 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9668 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9669 let registry = Arc::new(Registry::default());
9670 let supervisor_handle = SupervisorHandle::new();
9671 let supervisor =
9672 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9673 .with_handle(supervisor_handle.clone());
9674 let module = supervisor
9675 .spawn(ModuleSpec {
9676 carry_containment: false,
9677 module_id: "terminal-golden".to_string(),
9678 program: fake_aft_stub_path(),
9679 args: Vec::new(),
9680 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9681 reserved: false,
9682 reserved_prefixes: Vec::new(),
9683 protocol: ModuleProtocol::Subc,
9684 overlap: Default::default(),
9685 })
9686 .unwrap();
9687
9688 let deadline = Instant::now() + Duration::from_secs(5);
9689 while module.terminal_history().entries.len() != 2 {
9690 assert!(
9691 Instant::now() < deadline,
9692 "module did not retain two terminal exits: {:?}",
9693 module.terminal_history()
9694 );
9695 sleep(Duration::from_millis(10)).await;
9696 }
9697
9698 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9699 let request = ClientControlRequest::SupervisorTerminals {
9700 module_id: "terminal-golden".to_string(),
9701 };
9702 let frame = Frame::build(
9703 FrameType::Request,
9704 control_flags(),
9705 0,
9706 0,
9707 1,
9708 serde_json::to_vec(&request).unwrap(),
9709 )
9710 .unwrap();
9711 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9712 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9713 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9714 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9715 panic!("expected supervisor.terminals response");
9716 };
9717 assert_eq!(terminals.entries.len(), 2);
9718 assert_eq!(terminals.dropped, 0);
9719
9720 let mut rendered = serde_json::to_value(response).unwrap();
9721 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9724 for (index, entry) in rendered["entries"]
9725 .as_array_mut()
9726 .expect("terminal response entries array")
9727 .iter_mut()
9728 .enumerate()
9729 {
9730 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9731 }
9732
9733 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9734 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9735 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9736 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9737 std::fs::write(&golden_path, &serialized).unwrap();
9738 }
9739 let expected: Value =
9740 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9741 assert_eq!(rendered, expected);
9742 }
9743
9744 #[test]
9745 fn hello_registers_manifest_and_returns_ack() {
9746 let registry = Arc::new(Registry::default());
9747 let handler = ControlHandler::new(Arc::clone(®istry));
9748 let conn = ConnectionId::new(1);
9749
9750 let responses = handler
9751 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9752 .unwrap();
9753
9754 assert_eq!(responses.len(), 1);
9755 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9756 assert_eq!(responses[0].header.channel, 0);
9757 assert_eq!(responses[0].header.corr, 7);
9758 let ack = parse_ack(&responses[0]);
9759 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9760 assert!(ack
9761 .subc_capabilities
9762 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9763 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9764 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9765 assert!(ack
9766 .subc_ops
9767 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9768 assert!(ack
9769 .subc_ops
9770 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9771
9772 let registration = registry.get_module("aft").unwrap().unwrap();
9773 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9774 assert_eq!(registration.state, ChannelState::Active);
9775 assert_eq!(registration.connection_id, conn);
9776 assert_eq!(registration.control_ops, module_baseline_control_ops());
9777 }
9778
9779 #[test]
9780 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9781 let invalid_identifiers = [
9782 ("case_change", "credentials-Provider/v1"),
9783 ("leading_zero", "credentials-provider/v01"),
9784 ("trailing_hyphen", "credentials-provider-/v1"),
9785 ("consecutive_hyphens", "credentials--provider/v1"),
9786 ("uppercase", "Credentials-provider/v1"),
9787 ("missing_v", "credentials-provider/1"),
9788 ("whitespace", "credentials provider/v1"),
9789 ("zero_version", "credentials-provider/v0"),
9790 ("out_of_range_version", "credentials-provider/v4294967296"),
9791 (
9792 "overlength_name",
9793 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9794 ),
9795 ];
9796 let mut cases = invalid_identifiers
9797 .into_iter()
9798 .map(|(name, identifier)| {
9799 (
9800 format!("identifier_{name}"),
9801 "capabilities.provides[0]".to_string(),
9802 identifier.to_string(),
9803 json!({ "provides": [identifier] }),
9804 None,
9805 )
9806 })
9807 .collect::<Vec<_>>();
9808 cases.extend([
9809 (
9810 "unknown_need".to_string(),
9811 "capabilities.requires[0].need".to_string(),
9812 "deferred".to_string(),
9813 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9814 None,
9815 ),
9816 (
9817 "duplicate_provides".to_string(),
9818 "capabilities.provides[1]".to_string(),
9819 "credentials-provider/v1".to_string(),
9820 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9821 None,
9822 ),
9823 (
9824 "duplicate_must_never_reach".to_string(),
9825 "capabilities.must_never_reach[1]".to_string(),
9826 "credentials-provider/v1".to_string(),
9827 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9828 None,
9829 ),
9830 (
9831 "duplicate_requires_same_need".to_string(),
9832 "capabilities.requires[1]".to_string(),
9833 "credentials-provider/v1".to_string(),
9834 json!({ "requires": [
9835 { "capability": "credentials-provider/v1", "need": "required" },
9836 { "capability": "credentials-provider/v1", "need": "required" }
9837 ] }),
9838 None,
9839 ),
9840 (
9841 "duplicate_requires_conflicting_need".to_string(),
9842 "capabilities.requires[1]".to_string(),
9843 "credentials-provider/v1".to_string(),
9844 json!({ "requires": [
9845 { "capability": "credentials-provider/v1", "need": "required" },
9846 { "capability": "credentials-provider/v1", "need": "optional" }
9847 ] }),
9848 None,
9849 ),
9850 (
9851 "capabilities_root_pointer".to_string(),
9852 "runtime_computed[0]".to_string(),
9853 "/capabilities".to_string(),
9854 json!({}),
9855 Some(json!(["/capabilities"])),
9856 ),
9857 (
9858 "capabilities_descendant_pointer".to_string(),
9859 "runtime_computed[0]".to_string(),
9860 "/capabilities/provides".to_string(),
9861 json!({}),
9862 Some(json!(["/capabilities/provides"])),
9863 ),
9864 (
9865 "malformed_pointer_without_leading_slash".to_string(),
9866 "runtime_computed[0]".to_string(),
9867 "capabilities".to_string(),
9868 json!({}),
9869 Some(json!(["capabilities"])),
9870 ),
9871 (
9872 "malformed_pointer_escape".to_string(),
9873 "runtime_computed[0]".to_string(),
9874 "/roles/~2/tools".to_string(),
9875 json!({}),
9876 Some(json!(["/roles/~2/tools"])),
9877 ),
9878 (
9879 "unknown_capabilities_field".to_string(),
9880 "capabilities.future".to_string(),
9881 "<array>".to_string(),
9882 json!({ "future": [] }),
9883 None,
9884 ),
9885 ]);
9886
9887 for (index, (name, field, value, capabilities, runtime_computed)) in
9888 cases.into_iter().enumerate()
9889 {
9890 let registry = Arc::new(Registry::default());
9891 let handler = ControlHandler::new(Arc::clone(®istry));
9892 let response = handler
9893 .handle_control(
9894 ConnectionId::new((index + 1) as u64),
9895 capability_grammar_hello_frame(
9896 capabilities,
9897 runtime_computed,
9898 index as u64 + 1,
9899 ),
9900 )
9901 .expect("invalid HELLO returns a refusal");
9902
9903 assert_eq!(response.len(), 1, "{name} must emit one refusal");
9904 let error = parse_error(&response[0]);
9905 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9906 let message = error["message"]
9907 .as_str()
9908 .expect("error message is a string");
9909 assert!(
9910 message.contains(&field),
9911 "{name}: field missing from {message}"
9912 );
9913 assert!(
9914 message.contains(&value),
9915 "{name}: value missing from {message}"
9916 );
9917 assert_eq!(
9918 registry
9919 .active_registration_count()
9920 .expect("registry reads"),
9921 0,
9922 "{name}: refused HELLO must not create a catalog entry"
9923 );
9924 }
9925 }
9926
9927 #[test]
9928 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9929 let registry = Arc::new(Registry::default());
9930 let handler = ControlHandler::new(Arc::clone(®istry));
9931 let capabilities = json!({
9932 "provides": ["credentials-provider/v1"],
9933 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9934 "must_never_reach": ["federation-transport/v1"]
9935 });
9936 let response = handler
9937 .handle_control(
9938 ConnectionId::new(99),
9939 capability_grammar_hello_frame(
9940 capabilities.clone(),
9941 Some(json!(["/roles/0/tools"])),
9942 99,
9943 ),
9944 )
9945 .expect("valid HELLO registers");
9946 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9947
9948 let request = Frame::build(
9949 FrameType::Request,
9950 control_flags(),
9951 0,
9952 0,
9953 100,
9954 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9955 .expect("catalog request serializes"),
9956 )
9957 .expect("catalog request frame builds");
9958 let response = handler
9959 .handle_catalog_list(request, None)
9960 .expect("catalog list succeeds");
9961 let ClientControlResponse::CatalogList { modules, .. } =
9962 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9963 else {
9964 panic!("catalog request must return catalog.list");
9965 };
9966 assert_eq!(modules.len(), 1);
9967 assert_eq!(
9968 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9969 capabilities
9970 );
9971 }
9972
9973 #[test]
9974 fn catalog_list_mirrors_management_operation_description() {
9975 let registry = Arc::new(Registry::default());
9976 let handler = ControlHandler::new(Arc::clone(®istry));
9977 let description = "List managed records and return their identifiers and metadata.";
9978 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9979 manifest.provides = vec![ProviderRole::ManagementSurface {
9980 operations: vec![ManagementOperation {
9981 name: "records.list".to_string(),
9982 kind: ManagementOperationKind::Query,
9983 description: Some(description.to_string()),
9984 }],
9985 config_schema: json!({"type": "object"}),
9986 observability: vec![ObservabilitySurface {
9987 name: "records.stats".to_string(),
9988 kind: ObservabilityKind::Snapshot,
9989 }],
9990 identity_scope: vec![IdentityScope::Project],
9991 concurrency: Concurrency::ModuleManaged,
9992 }];
9993 registry
9994 .register_with_control_ops(
9995 manifest,
9996 PROTOCOL_VERSION,
9997 ConnectionId::new(99),
9998 Vec::new(),
9999 )
10000 .expect("described management manifest registers");
10001
10002 let request = Frame::build(
10003 FrameType::Request,
10004 control_flags(),
10005 0,
10006 0,
10007 100,
10008 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10009 .expect("catalog request serializes"),
10010 )
10011 .expect("catalog request frame builds");
10012 let response = handler
10013 .handle_catalog_list(request, None)
10014 .expect("catalog list succeeds");
10015 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10016 assert_eq!(
10017 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
10018 "catalog.list must preserve the declared operation description verbatim"
10019 );
10020 }
10021
10022 #[test]
10023 fn catalog_list_mirrors_registered_operation_delivery() {
10024 let registry = Arc::new(Registry::default());
10025 let handler = ControlHandler::new(Arc::clone(®istry));
10026 let delivery =
10027 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
10028 let mut body = serde_json::to_value(ModuleHelloBody {
10029 manifest: manifest("watch-management", PROTOCOL_VERSION),
10030 protocol_ver: PROTOCOL_VERSION,
10031 control_ops: None,
10032 launch_nonce: None,
10033 })
10034 .unwrap();
10035 body["manifest"]["provides"] = json!([{
10036 "role": "management_surface",
10037 "operations": [
10038 {"name": "status", "kind": "query"},
10039 {"name": "future", "kind": "query"},
10040 {"name": "reset", "kind": "mutate"}
10041 ],
10042 "config_schema": {}, "observability": [], "identity_scope": []
10043 }]);
10044 body["manifest"]["operation_delivery"] = delivery.clone();
10045 let hello = Frame::build(
10046 FrameType::Hello,
10047 control_flags(),
10048 0,
10049 0,
10050 99,
10051 serde_json::to_vec(&body).unwrap(),
10052 )
10053 .unwrap();
10054 let response = handler
10055 .handle_control(ConnectionId::new(99), hello)
10056 .unwrap();
10057 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10058
10059 let (_, registrations) = registry.list_modules().unwrap();
10060 let registered = ®istrations[0].manifest;
10061 assert!(registered.is_watch_operation("status"));
10062 assert!(!registered.is_watch_operation("future"));
10063 assert!(!registered.is_watch_operation("reset"));
10064 assert!(!registered.is_watch_operation("missing"));
10065
10066 let request = Frame::build(
10067 FrameType::Request,
10068 control_flags(),
10069 0,
10070 0,
10071 100,
10072 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10073 )
10074 .unwrap();
10075 let response = handler.handle_catalog_list(request, None).unwrap();
10076 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
10077 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
10078 let ClientControlResponse::CatalogList { modules, .. } =
10079 serde_json::from_slice(&response[0].body).unwrap()
10080 else {
10081 panic!("expected catalog.list");
10082 };
10083 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
10084 }
10085
10086 #[test]
10087 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
10088 let registry = Arc::new(Registry::default());
10089 let supervisor = SupervisorHandle::new();
10090 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10091 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10092 let descriptor = json!({
10093 "schema": "ckios.richui.manifest/1",
10094 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10095 "bytes": 4096,
10096 "chunks": 4
10097 });
10098 let connection = ConnectionId::new(201);
10099 let hello = handler
10100 .handle_control(
10101 connection,
10102 hello_frame_with_rich_ui(
10103 "vault",
10104 PROTOCOL_VERSION,
10105 201,
10106 Some("the-real-nonce"),
10107 descriptor.clone(),
10108 ),
10109 )
10110 .unwrap();
10111 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10112
10113 let request =
10114 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
10115 let provides = registry
10116 .get_module_by_connection(connection)
10117 .unwrap()
10118 .unwrap()
10119 .manifest
10120 .provides;
10121 handler
10122 .handle_catalog_update(connection, request, provides, None, Some(false))
10123 .unwrap();
10124
10125 let request = Frame::build(
10126 FrameType::Request,
10127 control_flags(),
10128 0,
10129 0,
10130 203,
10131 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10132 )
10133 .unwrap();
10134 let response = handler.handle_catalog_list(request, None).unwrap();
10135 let ClientControlResponse::CatalogList { modules, .. } =
10136 serde_json::from_slice(&response[0].body).unwrap()
10137 else {
10138 panic!("expected catalog.list");
10139 };
10140 assert_eq!(
10141 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
10142 descriptor
10143 );
10144 assert!(modules[0].nonce_verified);
10145 }
10146
10147 fn hello_with_role_declarations(declarations: Value) -> Frame {
10148 let mut body = serde_json::to_value(ModuleHelloBody {
10149 manifest: manifest("role-test", PROTOCOL_VERSION),
10150 protocol_ver: PROTOCOL_VERSION,
10151 control_ops: None,
10152 launch_nonce: None,
10153 })
10154 .unwrap();
10155 body["manifest"]["role_declarations"] = declarations;
10156 Frame::build(
10157 FrameType::Hello,
10158 control_flags(),
10159 0,
10160 0,
10161 301,
10162 serde_json::to_vec(&body).unwrap(),
10163 )
10164 .unwrap()
10165 }
10166
10167 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
10168 let request = Frame::build(
10169 FrameType::Request,
10170 control_flags(),
10171 0,
10172 0,
10173 302,
10174 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10175 )
10176 .unwrap();
10177 let response = handler.handle_catalog_list(request, None).unwrap();
10178 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
10179 let ClientControlResponse::CatalogList { mut modules, .. } =
10180 serde_json::from_slice(&response[0].body).unwrap()
10181 else {
10182 panic!("expected catalog.list");
10183 };
10184 (raw["modules"][0].clone(), modules.remove(0))
10185 }
10186
10187 #[test]
10188 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
10189 let registry = Arc::new(Registry::default());
10190 let handler = ControlHandler::new(Arc::clone(®istry));
10191 let declarations = json!({
10192 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
10193 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
10194 });
10195 let connection = ConnectionId::new(301);
10196 let response = handler
10197 .handle_control(
10198 connection,
10199 hello_with_role_declarations(declarations.clone()),
10200 )
10201 .unwrap();
10202 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10203 let registered = registry
10204 .get_module_by_connection(connection)
10205 .unwrap()
10206 .unwrap()
10207 .manifest;
10208 assert_eq!(
10212 serde_json::to_value(®istered.role_declarations).unwrap(),
10213 declarations
10214 );
10215 let (raw, entry) = role_declarations_catalog(&handler);
10216 assert_eq!(raw["role_declarations"], declarations);
10217 assert_eq!(entry.role_declarations, registered.role_declarations);
10218
10219 let request =
10220 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
10221 handler
10222 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
10223 .unwrap();
10224 let (raw, entry) = role_declarations_catalog(&handler);
10225 assert_eq!(raw["role_declarations"], declarations);
10226 assert_eq!(entry.role_declarations, registered.role_declarations);
10227 }
10228
10229 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
10230 let registry = Arc::new(Registry::default());
10231 let handler = ControlHandler::new(Arc::clone(®istry));
10232 let capture = EventCapture::default();
10233 let _subscriber =
10234 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10235 let response = handler
10236 .handle_control(
10237 ConnectionId::new(301),
10238 hello_with_role_declarations(declarations),
10239 )
10240 .unwrap();
10241 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10242 assert!(registry
10243 .get_module_by_connection(ConnectionId::new(301))
10244 .unwrap()
10245 .unwrap()
10246 .manifest
10247 .role_declarations
10248 .is_empty());
10249 let (raw, entry) = role_declarations_catalog(&handler);
10250 assert!(raw.get("role_declarations").is_none());
10251 assert!(entry.role_declarations.is_empty());
10252 let warnings: Vec<_> = capture
10253 .events()
10254 .into_iter()
10255 .filter(|event| {
10256 event.fields.get("message").map(String::as_str)
10257 == Some("dropping whole role_declarations map from HELLO")
10258 })
10259 .collect();
10260 assert_eq!(warnings.len(), 1);
10261 assert_eq!(warnings[0].level, tracing::Level::WARN);
10262 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
10263 assert_eq!(warnings[0].fields["reason"], reason);
10264 }
10265
10266 #[test]
10267 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
10268 for value in [
10269 json!(null),
10270 json!(false),
10271 json!(0),
10272 json!("opaque"),
10273 json!([]),
10274 ] {
10275 assert_hello_drops_role_declarations_with_warning(
10276 json!({"routing/v1": {}, "broken/v1": value}),
10277 "role_declarations values must be objects",
10278 );
10279 }
10280 assert_hello_drops_role_declarations_with_warning(
10281 json!([]),
10282 "role_declarations must be an object",
10283 );
10284 }
10285
10286 #[test]
10287 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
10288 assert_hello_drops_role_declarations_with_warning(
10289 json!({"routing/v1": {}, "bad key": {}}),
10290 "role_declarations key must use capability-name grammar",
10291 );
10292 }
10293
10294 #[test]
10295 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
10296 assert_hello_drops_role_declarations_with_warning(
10297 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
10298 "role_declarations exceeds 16384 compact JSON bytes",
10299 );
10300 }
10301
10302 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
10303 let mut body = serde_json::to_value(ModuleHelloBody {
10304 manifest: manifest("flow-ops", PROTOCOL_VERSION),
10305 protocol_ver: PROTOCOL_VERSION,
10306 control_ops: None,
10307 launch_nonce: None,
10308 })
10309 .unwrap();
10310 body["manifest"]["provides"] = json!([{
10311 "role": "management_surface",
10312 "operations": [
10313 {"name": "diagnostics.findings", "kind": "query"},
10314 {"name": "reset", "kind": "mutate"}
10315 ],
10316 "config_schema": {}, "observability": [], "identity_scope": []
10317 }]);
10318 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
10319 if let Some(flow_operations) = flow_operations {
10320 body["manifest"]["flow_operations"] = flow_operations;
10321 }
10322 Frame::build(
10323 FrameType::Hello,
10324 control_flags(),
10325 0,
10326 0,
10327 401,
10328 serde_json::to_vec(&body).unwrap(),
10329 )
10330 .unwrap()
10331 }
10332
10333 #[test]
10334 fn catalog_list_mirrors_flow_operations_unchanged() {
10335 for (capability, flow_operations) in [
10338 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
10339 ("call-modes/v1", json!(["diagnostics.findings"])),
10340 ("flow-scopes/v1", json!([])),
10341 ] {
10342 let registry = Arc::new(Registry::default());
10343 let handler = ControlHandler::new(Arc::clone(®istry));
10344 let connection = ConnectionId::new(401);
10345 let response = handler
10346 .handle_control(
10347 connection,
10348 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
10349 )
10350 .unwrap();
10351 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10352 let registered = registry
10353 .get_module_by_connection(connection)
10354 .unwrap()
10355 .unwrap()
10356 .manifest;
10357 assert_eq!(
10358 serde_json::to_value(®istered.flow_operations).unwrap(),
10359 flow_operations
10360 );
10361 let (raw, entry) = role_declarations_catalog(&handler);
10362 assert_eq!(raw["flow_operations"], flow_operations);
10363 assert_eq!(entry.flow_operations, registered.flow_operations);
10364 }
10365
10366 let registry = Arc::new(Registry::default());
10367 let handler = ControlHandler::new(Arc::clone(®istry));
10368 let response = handler
10369 .handle_control(
10370 ConnectionId::new(401),
10371 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
10372 )
10373 .unwrap();
10374 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10375 let (raw, entry) = role_declarations_catalog(&handler);
10376 assert!(raw.get("flow_operations").is_none());
10377 assert_eq!(entry.flow_operations, None);
10378 }
10379
10380 fn assert_hello_drops_flow_operations_with_warning(
10381 capabilities: Value,
10382 flow_operations: Value,
10383 reason: &str,
10384 ) {
10385 let registry = Arc::new(Registry::default());
10386 let handler = ControlHandler::new(Arc::clone(®istry));
10387 let capture = EventCapture::default();
10388 let _subscriber =
10389 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10390 let response = handler
10391 .handle_control(
10392 ConnectionId::new(401),
10393 hello_with_flow_operations(capabilities, Some(flow_operations)),
10394 )
10395 .unwrap();
10396 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10397 let registered = registry
10398 .get_module_by_connection(ConnectionId::new(401))
10399 .unwrap()
10400 .expect("the module still registers")
10401 .manifest;
10402 assert_eq!(registered.module_id, "flow-ops");
10403 assert_eq!(registered.flow_operations, None);
10404 let (raw, entry) = role_declarations_catalog(&handler);
10405 assert_eq!(raw["module_id"], "flow-ops");
10406 assert!(raw.get("flow_operations").is_none());
10407 assert_eq!(entry.flow_operations, None);
10408 let warnings: Vec<_> = capture
10409 .events()
10410 .into_iter()
10411 .filter(|event| {
10412 event.fields.get("message").map(String::as_str)
10413 == Some("dropping flow_operations from HELLO")
10414 })
10415 .collect();
10416 assert_eq!(warnings.len(), 1);
10417 assert_eq!(warnings[0].level, tracing::Level::WARN);
10418 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
10419 assert_eq!(warnings[0].fields["reason"], reason);
10420 }
10421
10422 #[test]
10423 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
10424 assert_hello_drops_flow_operations_with_warning(
10425 json!(["flow-scopes/v1"]),
10426 json!(["diagnostics.findings", "missing"]),
10427 "flow_operations names 'missing', which is not a declared management operation",
10428 );
10429 }
10430
10431 #[test]
10432 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
10433 assert_hello_drops_flow_operations_with_warning(
10434 json!([]),
10435 json!(["diagnostics.findings"]),
10436 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10437 );
10438 assert_hello_drops_flow_operations_with_warning(
10440 json!(["agent-run-scopes/v1"]),
10441 json!([]),
10442 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10443 );
10444 }
10445
10446 #[test]
10447 fn hello_malformed_flow_operations_is_dropped_with_warning() {
10448 assert_hello_drops_flow_operations_with_warning(
10449 json!(["flow-scopes/v1"]),
10450 json!(["reset", "reset"]),
10451 "flow_operations names 'reset' more than once",
10452 );
10453 assert_hello_drops_flow_operations_with_warning(
10454 json!(["flow-scopes/v1"]),
10455 json!(vec!["reset"; 257]),
10456 "flow_operations lists 257 names, more than 256",
10457 );
10458 assert_hello_drops_flow_operations_with_warning(
10459 json!(["flow-scopes/v1"]),
10460 json!("reset"),
10461 "flow_operations must be an array of operation names",
10462 );
10463 }
10464
10465 #[test]
10466 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
10467 let registry = Arc::new(Registry::default());
10468 let handler = ControlHandler::new(registry.clone());
10469 let descriptor = json!({
10470 "schema": "ckios.richui.manifest/1",
10471 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10472 "bytes": 128,
10473 "chunks": 2
10474 });
10475 let malformed = [
10476 ("schema", json!("")),
10477 ("schema", json!("a".repeat(65))),
10478 ("schema", json!("Bad schema")),
10479 (
10480 "manifest_digest",
10481 json!(format!("sha256:{}", "A".repeat(64))),
10482 ),
10483 ("manifest_digest", json!("sha256:abc")),
10484 ("bytes", json!(0)),
10485 ("bytes", json!(16_777_217)),
10486 ("bytes", json!("128")),
10487 ("chunks", json!(0)),
10488 ("chunks", json!(1025)),
10489 ("chunks", json!("2")),
10490 ];
10491 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
10492 let mut invalid = descriptor.clone();
10493 invalid[field] = invalid_value;
10494 let response = handler
10495 .handle_control(
10496 ConnectionId::new(210 + index as u64),
10497 hello_frame_with_rich_ui(
10498 &format!("bad-rich-ui-{index}"),
10499 PROTOCOL_VERSION,
10500 210 + index as u64,
10501 None,
10502 invalid,
10503 ),
10504 )
10505 .unwrap();
10506 assert_eq!(response[0].header.ty, FrameType::Error);
10507 let error = parse_error(&response[0]);
10508 assert_eq!(error["code"], "invalid_manifest");
10509 assert!(error["message"]
10510 .as_str()
10511 .unwrap()
10512 .contains(&format!("rich_ui.{field}")));
10513 assert!(registry
10514 .get_module(&format!("bad-rich-ui-{index}"))
10515 .unwrap()
10516 .is_none());
10517 }
10518 }
10519
10520 #[test]
10521 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
10522 let registry = Arc::new(Registry::default());
10523 let handler = ControlHandler::new(Arc::clone(®istry));
10524 let connection = ConnectionId::new(219);
10525 registry
10526 .register_with_control_ops(
10527 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
10528 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
10529 "bad schema",
10530 format!("sha256:{}", "a".repeat(64)),
10531 128,
10532 2,
10533 )))
10534 .build(),
10535 PROTOCOL_VERSION,
10536 connection,
10537 Vec::new(),
10538 )
10539 .unwrap();
10540 let request =
10541 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
10542 let response = handler
10543 .handle_catalog_update(connection, request, Vec::new(), None, None)
10544 .unwrap();
10545 assert_eq!(response[0].header.ty, FrameType::Error);
10546 let error = parse_error(&response[0]);
10547 assert_eq!(error["code"], "invalid_manifest");
10548 assert!(error["message"]
10549 .as_str()
10550 .unwrap()
10551 .contains("rich_ui.schema"));
10552 }
10553
10554 #[test]
10555 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
10556 let registry = Arc::new(Registry::default());
10557 let supervisor = SupervisorHandle::new();
10558 supervisor.open_swap("provider", "candidate-nonce".to_string());
10559 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10560 let connection = ConnectionId::new(218);
10561 let hello = handler
10562 .handle_control(
10563 connection,
10564 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
10565 )
10566 .unwrap();
10567 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10568 assert!(registry.list_modules().unwrap().1.is_empty());
10569
10570 registry.promote_candidate("provider").unwrap().unwrap();
10571 let request = Frame::build(
10572 FrameType::Request,
10573 control_flags(),
10574 0,
10575 0,
10576 219,
10577 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10578 )
10579 .unwrap();
10580 let response = handler.handle_catalog_list(request, None).unwrap();
10581 let ClientControlResponse::CatalogList { modules, .. } =
10582 serde_json::from_slice(&response[0].body).unwrap()
10583 else {
10584 panic!("expected catalog.list");
10585 };
10586 assert_eq!(modules.len(), 1);
10587 assert!(modules[0].nonce_verified);
10588 }
10589
10590 #[test]
10591 fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
10592 for provenance_json in [
10593 r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
10594 r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
10595 ] {
10596 let registry = Arc::new(Registry::default());
10597 let handler = ControlHandler::new(registry);
10598 let hello = Frame::build(
10599 FrameType::Hello,
10600 control_flags(),
10601 0,
10602 0,
10603 220,
10604 serde_json::to_vec(&ModuleHelloBody {
10605 manifest: ModuleManifest::builder("built", "0.1.0")
10606 .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
10607 .build(),
10608 protocol_ver: PROTOCOL_VERSION,
10609 control_ops: None,
10610 launch_nonce: None,
10611 })
10612 .unwrap(),
10613 )
10614 .unwrap();
10615 assert_eq!(
10616 handler
10617 .handle_control(ConnectionId::new(220), hello)
10618 .unwrap()[0]
10619 .header
10620 .ty,
10621 FrameType::HelloAck
10622 );
10623 let request = Frame::build(
10624 FrameType::Request,
10625 control_flags(),
10626 0,
10627 0,
10628 221,
10629 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10630 )
10631 .unwrap();
10632 let response = handler.handle_catalog_list(request, None).unwrap();
10633 let ClientControlResponse::CatalogList { modules, .. } =
10634 serde_json::from_slice(&response[0].body).unwrap()
10635 else {
10636 panic!("expected catalog.list");
10637 };
10638 assert_eq!(modules.len(), 1);
10639 assert_eq!(
10640 serde_json::to_vec(&modules[0]).unwrap(),
10641 format!(
10642 r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
10643 )
10644 .as_bytes()
10645 );
10646 }
10647 }
10648
10649 #[test]
10650 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
10651 let registry = Arc::new(Registry::default());
10652 let handler = ControlHandler::new(Arc::clone(®istry));
10653 let hello = Frame::build(
10654 FrameType::Hello,
10655 control_flags(),
10656 0,
10657 0,
10658 220,
10659 serde_json::to_vec(&ModuleHelloBody {
10660 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
10661 protocol_ver: PROTOCOL_VERSION,
10662 control_ops: None,
10663 launch_nonce: Some("unverified".to_string()),
10664 })
10665 .unwrap(),
10666 )
10667 .unwrap();
10668 assert_eq!(
10669 handler
10670 .handle_control(ConnectionId::new(220), hello)
10671 .unwrap()[0]
10672 .header
10673 .ty,
10674 FrameType::HelloAck
10675 );
10676 let request = Frame::build(
10677 FrameType::Request,
10678 control_flags(),
10679 0,
10680 0,
10681 221,
10682 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10683 )
10684 .unwrap();
10685 let response = handler.handle_catalog_list(request, None).unwrap();
10686 let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
10687 assert!(wire["modules"][0].get("provenance").is_none());
10688 let ClientControlResponse::CatalogList { modules, .. } =
10689 serde_json::from_slice(&response[0].body).unwrap()
10690 else {
10691 panic!("expected catalog.list");
10692 };
10693 assert_eq!(
10694 serde_json::to_vec(&modules[0]).unwrap(),
10695 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
10696 );
10697 assert!(!modules[0].nonce_verified);
10698 assert!(modules[0].provenance.is_none());
10699 }
10700
10701 #[test]
10702 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
10703 let registry = Arc::new(Registry::default());
10704 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10705 [("vault".to_string(), true), ("squatter".to_string(), true)],
10706 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10707 );
10708 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
10709 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10710 provides: vec!["credentials-provider/v1".to_string()],
10711 requires: Vec::new(),
10712 must_never_reach: Vec::new(),
10713 });
10714 let frame = Frame::build(
10715 FrameType::Hello,
10716 control_flags(),
10717 0,
10718 0,
10719 77,
10720 serde_json::to_vec(&ModuleHelloBody {
10721 manifest: squatter,
10722 protocol_ver: PROTOCOL_VERSION,
10723 control_ops: None,
10724 launch_nonce: None,
10725 })
10726 .expect("HELLO serializes"),
10727 )
10728 .expect("HELLO frame builds");
10729 let response = handler
10730 .handle_control(ConnectionId::new(77), frame)
10731 .expect("reserved claim receives a typed refusal");
10732 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10733 assert_eq!(
10734 registry
10735 .active_registration_count()
10736 .expect("registry reads"),
10737 0,
10738 "a reserved capability refusal must not leave a catalog entry"
10739 );
10740 }
10741
10742 #[test]
10743 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10744 for settlement in ["timeout", "inconclusive", "drop"] {
10745 let breakers = RouteBindBreakers::default();
10746 let RouteBindAdmission::Admitted {
10747 guard: mut old,
10748 probe: false,
10749 } = breakers.admit("prov")
10750 else {
10751 panic!("ordinary relay admitted")
10752 };
10753 let RouteBindAdmission::Admitted {
10754 guard: mut opener, ..
10755 } = breakers.admit("prov")
10756 else {
10757 panic!("second relay admitted")
10758 };
10759 assert!(
10760 !opener
10761 .record_timeout(1, Duration::ZERO)
10762 .unwrap()
10763 .reopened_after_probe
10764 );
10765 let RouteBindAdmission::Admitted {
10766 guard: mut probe,
10767 probe: true,
10768 } = breakers.admit("prov")
10769 else {
10770 panic!("one cooldown probe admitted")
10771 };
10772 match settlement {
10773 "timeout" => assert!(
10774 !old.record_timeout(1, Duration::ZERO)
10775 .unwrap()
10776 .reopened_after_probe
10777 ),
10778 "inconclusive" => old.record_inconclusive(),
10779 "drop" => drop(old),
10780 _ => unreachable!(),
10781 }
10782 assert!(
10783 matches!(
10784 breakers.admit("prov"),
10785 RouteBindAdmission::Refused {
10786 probe_in_flight: true,
10787 ..
10788 }
10789 ),
10790 "{settlement} of a pre-open relay cannot release the real probe"
10791 );
10792 assert!(
10793 probe
10794 .record_timeout(1, Duration::ZERO)
10795 .unwrap()
10796 .reopened_after_probe
10797 );
10798 assert!(matches!(
10799 breakers.admit("prov"),
10800 RouteBindAdmission::Admitted { probe: true, .. }
10801 ));
10802 }
10803 let breakers = RouteBindBreakers::default();
10804 let admit = || match breakers.admit("prov") {
10805 RouteBindAdmission::Admitted { guard, .. } => guard,
10806 _ => panic!("relay admitted"),
10807 };
10808 admit().record_timeout(1, Duration::ZERO);
10809 let mut old_probe = admit();
10810 breakers.reset_for_new_module_connection("prov");
10811 admit().record_timeout(1, Duration::ZERO);
10812 let _new_probe = admit();
10813 old_probe.record_inconclusive();
10814 assert!(matches!(
10815 breakers.admit("prov"),
10816 RouteBindAdmission::Refused {
10817 probe_in_flight: true,
10818 ..
10819 }
10820 ));
10821 }
10822
10823 #[tokio::test]
10824 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
10825 for candidate in [false, true] {
10826 let registry = Arc::new(Registry::default());
10827 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10828 [("vault".to_string(), true), ("squatter".to_string(), true)],
10829 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10830 );
10831 let conn = ConnectionId::new(77);
10832 let (ctx, mut rx) = route_ctx(conn);
10833 let initial = capability_manifest("squatter", &[], &[]);
10834 if candidate {
10835 registry
10836 .register_candidate_with_control_ops(
10837 initial.clone(),
10838 PROTOCOL_VERSION,
10839 conn,
10840 module_baseline_control_ops(),
10841 )
10842 .unwrap();
10843 handler
10844 .forwarding
10845 .register_candidate_module_connection(
10846 conn,
10847 "squatter".to_string(),
10848 PROTOCOL_VERSION,
10849 manifest_concurrency(&initial),
10850 ctx.egress.clone(),
10851 )
10852 .unwrap();
10853 } else {
10854 hello_via_sink(
10855 &handler,
10856 &ctx,
10857 &mut rx,
10858 hello_frame_with_manifest(initial.clone(), 1),
10859 )
10860 .await;
10861 }
10862 let response = handler
10863 .handle_control_frame(
10864 &ctx,
10865 catalog_update_with_capabilities_frame(
10866 2,
10867 capability_manifest("squatter", &["credentials-provider/v1"], &[])
10868 .capabilities
10869 .unwrap(),
10870 ),
10871 )
10872 .await
10873 .unwrap();
10874 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10875 assert_eq!(
10876 registry
10877 .get_module_by_connection(conn)
10878 .unwrap()
10879 .unwrap()
10880 .manifest,
10881 initial
10882 );
10883 }
10884 }
10885
10886 #[test]
10887 fn server_describe_surfaces_required_capability_verdict_fields() {
10888 let registry = Arc::new(Registry::default());
10889 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10890 [
10891 ("consumer".to_string(), true),
10892 ("provider".to_string(), false),
10893 ],
10894 BTreeMap::new(),
10895 );
10896 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10897 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10898 provides: Vec::new(),
10899 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10900 capability: "credentials-provider/v1".to_string(),
10901 need: subc_protocol::manifest::CapabilityNeed::Required,
10902 }],
10903 must_never_reach: Vec::new(),
10904 });
10905 let hello = Frame::build(
10906 FrameType::Hello,
10907 control_flags(),
10908 0,
10909 0,
10910 78,
10911 serde_json::to_vec(&ModuleHelloBody {
10912 manifest: consumer,
10913 protocol_ver: PROTOCOL_VERSION,
10914 control_ops: None,
10915 launch_nonce: None,
10916 })
10917 .expect("HELLO serializes"),
10918 )
10919 .expect("HELLO frame builds");
10920 handler
10921 .handle_control(ConnectionId::new(78), hello)
10922 .expect("consumer registers");
10923 let describe = Frame::build(
10924 FrameType::Request,
10925 control_flags(),
10926 0,
10927 0,
10928 79,
10929 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10930 .expect("request serializes"),
10931 )
10932 .expect("describe frame builds");
10933 let response = handler
10934 .handle_server_describe(describe)
10935 .expect("server.describe succeeds");
10936 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10937 let requirement = &rendered["capability_requirements"][0];
10938 assert_eq!(requirement["consumer"], "consumer");
10939 assert_eq!(requirement["verdict"], "never_provided");
10940 assert_eq!(requirement["episode_seq"], 1);
10941 assert_eq!(requirement["config_satisfiable"], false);
10942 assert_eq!(requirement["runtime_available"], false);
10943 assert!(requirement["detail"]
10944 .as_str()
10945 .expect("detail string")
10946 .contains("credentials-provider/v1"));
10947 }
10948
10949 fn deregister_required_provider(
10955 daemon_shutdown_begun: bool,
10956 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10957 const CAPABILITY: &str = "project-identity/v1";
10958 let registry = Arc::new(Registry::default());
10959 let daemon_shutdown = DaemonShutdownFlag::default();
10960 let handler = ControlHandler::new(Arc::clone(®istry))
10961 .with_capability_config(
10962 [
10963 ("consumer".to_string(), true),
10964 ("provider".to_string(), true),
10965 ],
10966 BTreeMap::new(),
10967 )
10968 .with_daemon_shutdown_flag(daemon_shutdown.clone());
10969 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10970 consumer.capabilities = Some(CapabilityDeclarations {
10971 provides: Vec::new(),
10972 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10973 capability: CAPABILITY.to_string(),
10974 need: subc_protocol::manifest::CapabilityNeed::Required,
10975 }],
10976 must_never_reach: Vec::new(),
10977 });
10978 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10979 let provider_conn = ConnectionId::new(182);
10980 for (conn, manifest, corr) in [
10981 (ConnectionId::new(181), consumer, 181),
10982 (provider_conn, provider, 182),
10983 ] {
10984 let hello = handler
10985 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10986 .expect("module registers");
10987 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10988 }
10989 assert_eq!(
10990 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10991 Some(CapabilityVerdict::Provided),
10992 "the registered provider must satisfy the requirement first"
10993 );
10994 if daemon_shutdown_begun {
10995 daemon_shutdown.set_for_test();
10996 }
10997
10998 let capture = EventCapture::default();
10999 let _subscriber =
11000 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11001 handler
11002 .cleanup_connection(provider_conn)
11003 .expect("provider deregisters");
11004 let events = capture
11005 .events()
11006 .into_iter()
11007 .filter(|event| {
11008 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
11009 })
11010 .collect();
11011 (
11012 events,
11013 handler.capability_evaluator.verdict("consumer", CAPABILITY),
11014 )
11015 }
11016
11017 #[test]
11018 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
11019 let (events, verdict) = deregister_required_provider(true);
11020 assert_eq!(
11021 verdict,
11022 Some(CapabilityVerdict::NeverProvided),
11023 "shutdown must not change the verdict route.open refusals read"
11024 );
11025 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11026 let event = &events[0];
11027 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
11028 assert_eq!(
11029 event.fields.get("cause").map(String::as_str),
11030 Some("\"daemon_shutdown\""),
11031 "{event:?}"
11032 );
11033 assert_eq!(
11034 event.fields.get("verdict").map(String::as_str),
11035 Some("\"never_provided\""),
11036 "{event:?}"
11037 );
11038 }
11039
11040 #[test]
11041 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
11042 let (events, verdict) = deregister_required_provider(false);
11043 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
11044 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11045 let event = &events[0];
11046 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
11047 assert_eq!(event.fields.get("cause"), None, "{event:?}");
11048 assert_eq!(
11049 event.fields.get("verdict").map(String::as_str),
11050 Some("\"never_provided\""),
11051 "{event:?}"
11052 );
11053 }
11054
11055 #[test]
11056 fn catalog_list_omits_capabilities_for_legacy_manifest() {
11057 let registry = Arc::new(Registry::default());
11058 let handler = ControlHandler::new(Arc::clone(®istry));
11059 let hello = handler
11060 .handle_control(
11061 ConnectionId::new(101),
11062 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
11063 )
11064 .expect("legacy HELLO registers");
11065 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11066
11067 let request = Frame::build(
11068 FrameType::Request,
11069 control_flags(),
11070 0,
11071 0,
11072 102,
11073 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
11074 .expect("catalog request serializes"),
11075 )
11076 .expect("catalog request frame builds");
11077 let response = handler
11078 .handle_catalog_list(request, None)
11079 .expect("catalog list succeeds");
11080 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
11081 assert!(
11082 body["modules"][0].get("capabilities").is_none(),
11083 "legacy manifest must retain an absent capabilities field on catalog.list"
11084 );
11085 }
11086
11087 #[test]
11088 fn hello_ack_omits_storage_when_no_storage_config() {
11089 let registry = Arc::new(Registry::default());
11090 let handler = ControlHandler::new(Arc::clone(®istry));
11091 let responses = handler
11092 .handle_control(
11093 ConnectionId::new(1),
11094 hello_frame("aft", PROTOCOL_VERSION, 7),
11095 )
11096 .unwrap();
11097 let ack = parse_ack(&responses[0]);
11098 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
11099 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
11100 }
11101
11102 #[tokio::test]
11103 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
11104 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
11105 let registry = Arc::new(Registry::default());
11106 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
11107 let responses = handler
11108 .handle_control(
11109 ConnectionId::new(1),
11110 hello_frame("aft", PROTOCOL_VERSION, 7),
11111 )
11112 .unwrap();
11113 let ack = parse_ack(&responses[0]);
11114 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
11115
11116 let described = handler
11117 .handle_control_frame(
11118 &route_ctx(ConnectionId::new(2)).0,
11119 Frame::build(
11120 FrameType::Request,
11121 control_flags(),
11122 0,
11123 0,
11124 9,
11125 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11126 )
11127 .unwrap(),
11128 )
11129 .await
11130 .unwrap();
11131 let ClientControlResponse::ServerDescribe { machine_id, .. } =
11132 serde_json::from_slice(&described[0].body).unwrap()
11133 else {
11134 panic!("server.describe answered with another shape");
11135 };
11136 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
11137 }
11138
11139 #[test]
11140 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
11141 let registry = Arc::new(Registry::default());
11144 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
11145 crate::daemon_config::StorageConfig::Sqlite {
11146 data_home: std::path::PathBuf::from("/data"),
11147 },
11148 ));
11149
11150 let responses = handler
11151 .handle_control(
11152 ConnectionId::new(1),
11153 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
11154 )
11155 .unwrap();
11156 let ack = parse_ack(&responses[0]);
11157 assert_eq!(
11158 ack.storage,
11159 Some(serde_json::json!({
11160 "module_id": "alfonso-routing",
11161 "storage_namespace": "default",
11162 "isolation": { "kind": "module" },
11163 "backend": {
11164 "backend": "sqlite",
11165 "path": "/data/cortexkit/alfonso-routing/store.db"
11166 }
11167 })),
11168 "the delivered descriptor is the module's own sqlite store path"
11169 );
11170 }
11171
11172 #[test]
11173 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
11174 let registry = Arc::new(Registry::default());
11175 let handler = ControlHandler::new(Arc::clone(®istry));
11176 let conn = ConnectionId::new(1);
11177 let responses = handler
11178 .handle_control(
11179 conn,
11180 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11181 )
11182 .unwrap();
11183 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11184 let registration = registry.get_module("aft").unwrap().unwrap();
11185 assert_eq!(registration.control_ops, module_baseline_control_ops());
11186
11187 let frame =
11188 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
11189 assert!(handler
11190 .guard_module_control_op(&frame, "aft", "route.bind")
11191 .unwrap()
11192 .is_none());
11193 let error = handler
11194 .guard_module_control_op(&frame, "aft", "test.synthetic")
11195 .unwrap()
11196 .expect("synthetic ungranted op should be rejected");
11197 assert_eq!(error.header.ty, FrameType::Error);
11198 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
11199 }
11200
11201 #[test]
11202 fn hello_control_ops_some_adds_optional_grants() {
11203 let registry = Arc::new(Registry::default());
11204 let handler = ControlHandler::new(Arc::clone(®istry));
11205 handler
11206 .handle_control(
11207 ConnectionId::new(1),
11208 hello_frame_with_control_ops(
11209 "aft",
11210 PROTOCOL_VERSION,
11211 7,
11212 Some(vec![
11213 "future.synthetic".to_string(),
11214 "route.bind".to_string(),
11215 ]),
11216 ),
11217 )
11218 .unwrap();
11219 let registration = registry.get_module("aft").unwrap().unwrap();
11220 assert_eq!(
11221 registration.control_ops,
11222 vec![
11223 "route.bind".to_string(),
11224 "route.status".to_string(),
11225 "future.synthetic".to_string(),
11226 ]
11227 );
11228 let frame =
11229 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
11230 assert!(handler
11231 .guard_module_control_op(&frame, "aft", "future.synthetic")
11232 .unwrap()
11233 .is_none());
11234 }
11235
11236 #[tokio::test]
11237 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
11238 let registry = Arc::new(Registry::default());
11239 let forwarding = Arc::new(ForwardingTable::default());
11240 let handler =
11241 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11242 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
11243 hello_via_sink(
11244 &handler,
11245 &module_ctx,
11246 &mut module_rx,
11247 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11248 )
11249 .await;
11250
11251 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
11252 let responses = handler
11253 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
11254 .await
11255 .unwrap();
11256 assert_eq!(responses.len(), 1);
11257 assert_eq!(responses[0].header.ty, FrameType::Error);
11258 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
11259 assert!(module_rx.try_recv().is_err());
11260 }
11261
11262 #[tokio::test]
11263 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
11264 let registry = Arc::new(Registry::default());
11265 let forwarding = Arc::new(ForwardingTable::default());
11266 let handler =
11267 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11268 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
11269 hello_via_sink(
11270 &handler,
11271 &module_ctx,
11272 &mut module_rx,
11273 hello_frame_with_control_ops(
11274 "aft",
11275 PROTOCOL_VERSION,
11276 7,
11277 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11278 ),
11279 )
11280 .await;
11281
11282 let project_root = unique_project_root("demux");
11283 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
11284 let route_handler = handler.clone();
11285 let route_task = tokio::spawn(async move {
11286 route_handler
11287 .handle_control_frame(
11288 &route_client_ctx,
11289 route_open_frame(100, "aft", project_root),
11290 )
11291 .await
11292 .unwrap()
11293 });
11294 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11295 .await
11296 .unwrap()
11297 .unwrap();
11298 assert!(matches!(
11299 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
11300 ModuleControlRequest::RouteBind { .. }
11301 ));
11302
11303 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
11304 let health_handler = handler.clone();
11305 let health_task = tokio::spawn(async move {
11306 health_handler
11307 .handle_control_frame(
11308 &health_client_ctx,
11309 supervisor_health_probe_frame(101, "aft"),
11310 )
11311 .await
11312 .unwrap()
11313 });
11314 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11315 .await
11316 .unwrap()
11317 .unwrap();
11318 assert_eq!(
11319 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11320 ModuleControlRequest::HealthCheck {}
11321 );
11322
11323 handler
11324 .handle_control_frame(
11325 &module_ctx,
11326 health_response(health_frame.header.corr, HealthStatus::Degraded),
11327 )
11328 .await
11329 .unwrap();
11330 let health_response = health_task.await.unwrap();
11331 assert_eq!(health_response.len(), 1);
11332 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11333 ClientControlResponse::SupervisorHealthProbe {
11334 module_id,
11335 status,
11336 detail,
11337 metrics,
11338 } => {
11339 assert_eq!(module_id, "aft");
11340 assert_eq!(status, HealthStatus::Degraded);
11341 assert_eq!(detail.as_deref(), Some("warming"));
11342 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
11343 }
11344 other => panic!("unexpected health response: {other:?}"),
11345 }
11346
11347 handler
11348 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11349 .await
11350 .unwrap();
11351 let route_response = route_task.await.unwrap();
11352 assert!(route_response.is_empty());
11353 let published = route_client_rx.recv().await.unwrap();
11354 assert!(matches!(
11355 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11356 ClientControlResponse::RouteOpen { .. }
11357 ));
11358 }
11359
11360 async fn relay_route_open(
11365 handler: &ControlHandler,
11366 client_connection: ConnectionId,
11367 client_egress: &FrameSink,
11368 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11369 corr: u64,
11370 module_id: &str,
11371 project_root_label: &str,
11372 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
11373 let ctx = RouteCtx {
11374 connection_id: client_connection,
11375 egress: client_egress.clone(),
11376 };
11377 let handler = handler.clone();
11378 let project_root = unique_project_root(project_root_label);
11379 let module_id = module_id.to_string();
11380 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
11381 let task = tokio::spawn(async move {
11382 let _guard = tracing::dispatcher::set_default(&dispatch);
11383 handler
11384 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
11385 .await
11386 .unwrap()
11387 });
11388 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
11389 .await
11390 .expect("module receives the relayed route.bind")
11391 .expect("module egress is open");
11392 (task, bind.frame)
11393 }
11394
11395 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
11396 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
11397 ModuleControlRequest::RouteBind {
11398 route_channel,
11399 epoch,
11400 ..
11401 } => (route_channel, epoch),
11402 other => panic!("expected a route.bind request, got {other:?}"),
11403 }
11404 }
11405
11406 fn published_route(frame: &Frame) -> (u16, u32) {
11407 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
11408 ClientControlResponse::RouteOpen {
11409 route_channel,
11410 route_epoch,
11411 ..
11412 } => (route_channel, route_epoch),
11413 other => panic!("expected a route.open response, got {other:?}"),
11414 }
11415 }
11416
11417 #[tokio::test]
11437 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
11438 let registry = Arc::new(Registry::default());
11439 let forwarding = Arc::new(ForwardingTable::default());
11440 let handler =
11441 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11442
11443 let module_connection = ConnectionId::new(30);
11444 let (module_ctx, mut module_rx) = route_ctx(module_connection);
11445 hello_via_sink(
11446 &handler,
11447 &module_ctx,
11448 &mut module_rx,
11449 hello_frame("aft", PROTOCOL_VERSION, 7),
11450 )
11451 .await;
11452
11453 let dying_client = ConnectionId::new(31);
11454 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
11455
11456 let (first_task, first_bind) = relay_route_open(
11459 &handler,
11460 dying_client,
11461 &dying_ctx.egress,
11462 &mut module_rx,
11463 100,
11464 "aft",
11465 "closing-first",
11466 )
11467 .await;
11468 handler
11469 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
11470 .await
11471 .unwrap();
11472 assert!(first_task.await.unwrap().is_empty());
11473 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
11474
11475 let (second_task, second_bind) = relay_route_open(
11477 &handler,
11478 dying_client,
11479 &dying_ctx.egress,
11480 &mut module_rx,
11481 101,
11482 "aft",
11483 "closing-second",
11484 )
11485 .await;
11486 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
11487
11488 assert!(forwarding
11491 .escalate_client_delivery_failure(
11492 dying_client,
11493 first_channel,
11494 first_epoch,
11495 CloseReason::new(
11496 "module_to_client_delivery_failed",
11497 "client egress refused a module frame",
11498 ),
11499 crate::forwarding::UndeliveredFrame {
11500 module_id: None,
11501 sink: &dying_ctx.egress,
11502 },
11503 )
11504 .unwrap());
11505 assert!(!dying_ctx.egress.is_closed());
11506
11507 let ack = handler
11509 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
11510 .await;
11511 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11512 if module_loop_error.is_some() {
11513 handler.cleanup_connection(module_connection).unwrap();
11517 }
11518 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11522 .await
11523 .ok()
11524 .flatten();
11525
11526 assert!(
11528 registry
11529 .get_module_by_connection(module_connection)
11530 .unwrap()
11531 .is_some(),
11532 "one client's closing connection ended the shared module connection: \
11533 {module_loop_error:?}"
11534 );
11535 let cotenant = ConnectionId::new(32);
11537 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11538 let (cotenant_task, cotenant_bind) = relay_route_open(
11539 &handler,
11540 cotenant,
11541 &cotenant_ctx.egress,
11542 &mut module_rx,
11543 102,
11544 "aft",
11545 "closing-cotenant",
11546 )
11547 .await;
11548 handler
11549 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
11550 .await
11551 .unwrap();
11552 assert!(cotenant_task.await.unwrap().is_empty());
11553 let (cotenant_channel, cotenant_epoch) =
11554 published_route(&cotenant_rx.recv().await.unwrap());
11555 assert!(matches!(
11556 forwarding
11557 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11558 .unwrap(),
11559 DataRoute::Client(DataRouteState::Bound(_))
11560 ));
11561
11562 let goodbye = post_ack_module_frame
11565 .expect("module receives a GOODBYE for the abandoned route channel");
11566 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11567 assert_eq!(goodbye.header.channel, abandoned_channel);
11568 assert_eq!(goodbye.header.epoch, abandoned_epoch);
11569
11570 assert!(dying_rx.try_recv().is_err());
11574 let second_response = second_task.await.unwrap();
11575 assert_eq!(second_response.len(), 1);
11576 assert_eq!(
11577 parse_error(&second_response[0])["code"],
11578 "target_unavailable"
11579 );
11580 }
11581
11582 async fn catalog_snapshot(
11585 handler: &ControlHandler,
11586 client_ctx: &RouteCtx,
11587 corr: u64,
11588 ) -> (u64, Vec<(String, bool)>) {
11589 let body =
11590 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
11591 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
11592 let reply = handler
11593 .handle_control_frame(client_ctx, request)
11594 .await
11595 .unwrap()
11596 .pop()
11597 .unwrap();
11598 let ClientControlResponse::CatalogList {
11599 generation,
11600 modules,
11601 ..
11602 } = serde_json::from_slice(&reply.body).unwrap()
11603 else {
11604 panic!("not a catalog.list reply");
11605 };
11606 (
11607 generation,
11608 modules
11609 .into_iter()
11610 .map(|entry| (entry.module_id, entry.ready))
11611 .collect(),
11612 )
11613 }
11614
11615 fn hello_held_before_forwarding() -> (
11621 ControlHandler,
11622 Arc<std::sync::Barrier>,
11623 std::thread::JoinHandle<Vec<Frame>>,
11624 RouteCtx,
11625 mpsc::Receiver<crate::router::OutboundFrame>,
11626 ) {
11627 let hold = Arc::new(std::sync::Barrier::new(2));
11628 let handler = ControlHandler::with_forwarding(
11629 Arc::new(Registry::default()),
11630 Arc::new(ForwardingTable::default()),
11631 )
11632 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
11633 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11634 let hello = {
11635 let handler = handler.clone();
11636 let module_ctx = module_ctx.clone();
11637 std::thread::spawn(move || {
11638 handler
11639 .handle_hello(
11640 module_ctx.connection_id,
11641 Some(module_ctx.egress.clone()),
11642 hello_frame("aft", PROTOCOL_VERSION, 7),
11643 )
11644 .unwrap()
11645 })
11646 };
11647 (handler, hold, hello, module_ctx, module_rx)
11648 }
11649
11650 #[tokio::test]
11659 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
11660 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
11661 let client = ConnectionId::new(41);
11662 let (client_ctx, mut client_rx) = route_ctx(client);
11663
11664 hold.wait();
11665 assert!(handler.registry.get_module("aft").unwrap().is_some());
11667 assert!(!handler
11668 .forwarding
11669 .has_live_module_connection("aft")
11670 .unwrap());
11671 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
11672 assert!(
11673 listed.is_empty(),
11674 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
11675 );
11676 let refused = handler
11677 .handle_control_frame(
11678 &client_ctx,
11679 route_open_frame(301, "aft", unique_project_root("listed-window")),
11680 )
11681 .await
11682 .unwrap();
11683 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
11684 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
11685 let duplicate = handler
11686 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
11687 .await
11688 .unwrap();
11689 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11690
11691 hold.wait();
11692 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11693 let ack = module_rx.try_recv().unwrap().frame;
11694 assert_eq!(ack.header.ty, FrameType::HelloAck);
11695 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11696 assert_eq!(listed, vec![("aft".to_string(), true)]);
11697
11698 let (open_task, bind) = relay_route_open(
11699 &handler,
11700 client,
11701 &client_ctx.egress,
11702 &mut module_rx,
11703 303,
11704 "aft",
11705 "listed-after",
11706 )
11707 .await;
11708 assert!(handler
11709 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11710 .await
11711 .unwrap()
11712 .is_empty());
11713 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
11714 published_route(&client_rx.try_recv().unwrap().frame);
11715 }
11716
11717 #[tokio::test]
11724 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11725 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11726 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11727 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11728
11729 hold.wait();
11730 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11731 assert!(listed.is_empty());
11732 assert_eq!(
11733 during, before,
11734 "the generation moved while the module was not listed"
11735 );
11736
11737 hold.wait();
11738 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11739 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11740 assert_eq!(listed, vec![("aft".to_string(), true)]);
11741 assert_eq!(
11742 after,
11743 before + 1,
11744 "the module appeared without exactly one generation bump"
11745 );
11746 }
11747
11748 #[tokio::test]
11753 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11754 let registry = Arc::new(Registry::default());
11755 let handler = ControlHandler::with_forwarding(
11756 Arc::clone(®istry),
11757 Arc::new(ForwardingTable::default()),
11758 );
11759 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11760 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11761 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11762 drop(module_rx);
11763
11764 let replies = handler
11765 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
11766 .await
11767 .unwrap();
11768 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
11769 assert!(registry.get_module("aft").unwrap().is_none());
11770 assert!(registry
11771 .get_module_by_connection(module_ctx.connection_id)
11772 .unwrap()
11773 .is_none());
11774 assert_eq!(
11775 catalog_snapshot(&handler, &client_ctx, 301).await,
11776 (before, Vec::new())
11777 );
11778 }
11779
11780 async fn route_open_held_after_bind_send() -> (
11784 ControlHandler,
11785 Arc<tokio::sync::Barrier>,
11786 RouteCtx,
11787 mpsc::Receiver<crate::router::OutboundFrame>,
11788 ) {
11789 let hold = Arc::new(tokio::sync::Barrier::new(2));
11790 let handler = ControlHandler::with_forwarding(
11791 Arc::new(Registry::default()),
11792 Arc::new(ForwardingTable::default()),
11793 )
11794 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
11795 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11796 hello_via_sink(
11797 &handler,
11798 &module_ctx,
11799 &mut module_rx,
11800 hello_frame("aft", PROTOCOL_VERSION, 7),
11801 )
11802 .await;
11803 (handler, hold, module_ctx, module_rx)
11804 }
11805
11806 #[tokio::test]
11813 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
11814 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11815 let client = ConnectionId::new(41);
11816 let (client_ctx, mut client_rx) = route_ctx(client);
11817
11818 let (open_task, bind) = relay_route_open(
11819 &handler,
11820 client,
11821 &client_ctx.egress,
11822 &mut module_rx,
11823 200,
11824 "aft",
11825 "acked-before-mark",
11826 )
11827 .await;
11828 let (module_channel, module_epoch) = route_bind_channel(&bind);
11829 hold.wait().await;
11831
11832 assert!(handler
11834 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11835 .await
11836 .unwrap()
11837 .is_empty());
11838 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
11839
11840 hold.wait().await;
11841 assert!(
11842 open_task.await.unwrap().is_empty(),
11843 "route.open answers accepted"
11844 );
11845
11846 match module_rx.try_recv() {
11849 Err(mpsc::error::TryRecvError::Empty) => {}
11850 Ok(outbound) => panic!(
11851 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
11852 outbound.frame.header.ty,
11853 outbound.frame.header.channel,
11854 outbound.frame.header.epoch
11855 ),
11856 Err(err) => panic!("module egress closed: {err}"),
11857 }
11858
11859 let router = Router::with_control_handler(Arc::new(handler.clone()));
11862 let request = Frame::build(
11863 FrameType::Request,
11864 Flags::new(true, Priority::Interactive, false),
11865 client_channel,
11866 client_epoch,
11867 201,
11868 b"first request".to_vec(),
11869 )
11870 .unwrap();
11871 router
11872 .route_for_connection(&client_ctx, request)
11873 .await
11874 .unwrap();
11875 let delivered = module_rx.try_recv().unwrap().frame;
11876 assert_eq!(delivered.header.ty, FrameType::Request);
11877 assert_eq!(
11878 (delivered.header.channel, delivered.header.epoch),
11879 (module_channel, module_epoch)
11880 );
11881 assert_eq!(delivered.body, b"first request");
11882 }
11883
11884 #[tokio::test]
11890 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
11891 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11892 let client = ConnectionId::new(42);
11893 let (client_ctx, _client_rx) = route_ctx(client);
11894
11895 let (open_task, bind) = relay_route_open(
11896 &handler,
11897 client,
11898 &client_ctx.egress,
11899 &mut module_rx,
11900 300,
11901 "aft",
11902 "released-before-mark",
11903 )
11904 .await;
11905 let (module_channel, module_epoch) = route_bind_channel(&bind);
11906 hold.wait().await;
11907
11908 handler.cleanup_connection(client).unwrap();
11909 assert!(
11910 module_rx.try_recv().is_err(),
11911 "client teardown must not GOODBYE a bind route.open is still queueing"
11912 );
11913
11914 hold.wait().await;
11915 let response = open_task.await.unwrap();
11916 assert_eq!(response.len(), 1);
11917 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11918
11919 let goodbye = module_rx
11920 .try_recv()
11921 .expect("module is told to drop the binding the daemon abandoned")
11922 .frame;
11923 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11924 assert_eq!(
11925 (goodbye.header.channel, goodbye.header.epoch),
11926 (module_channel, module_epoch)
11927 );
11928 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11929 }
11930
11931 #[tokio::test]
11936 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11937 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11938 let client = ConnectionId::new(43);
11939 let (client_ctx, client_rx) = route_ctx(client);
11940
11941 let (open_task, bind) = relay_route_open(
11942 &handler,
11943 client,
11944 &client_ctx.egress,
11945 &mut module_rx,
11946 400,
11947 "aft",
11948 "gone-client-before-mark",
11949 )
11950 .await;
11951 let (module_channel, module_epoch) = route_bind_channel(&bind);
11952 hold.wait().await;
11953
11954 drop(client_rx);
11955 assert!(client_ctx.egress.is_closed());
11956 handler
11957 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11958 .await
11959 .unwrap();
11960
11961 hold.wait().await;
11962 let response = open_task.await.unwrap();
11963 assert_eq!(response.len(), 1);
11964 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11965
11966 let goodbye = module_rx
11967 .try_recv()
11968 .expect("module is told to drop the binding it acked")
11969 .frame;
11970 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11971 assert_eq!(
11972 (goodbye.header.channel, goodbye.header.epoch),
11973 (module_channel, module_epoch)
11974 );
11975 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11976 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11977 }
11978
11979 #[test]
11986 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11987 let handler = ControlHandler::default();
11988 let module_connection = ConnectionId::new(30);
11989 let client_connection = ConnectionId::new(31);
11990
11991 handler
11992 .refuse_to_end_module_connection_for_a_client(
11993 module_connection,
11994 77,
11995 ForwardingError::ConnectionClosing {
11996 connection_id: client_connection,
11997 },
11998 )
11999 .expect("a closing client must never end the module connection");
12000
12001 assert!(matches!(
12002 handler.refuse_to_end_module_connection_for_a_client(
12003 module_connection,
12004 78,
12005 ForwardingError::ConnectionClosing {
12006 connection_id: module_connection,
12007 },
12008 ),
12009 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
12010 connection_id
12011 })) if connection_id == module_connection
12012 ));
12013 assert!(matches!(
12014 handler.refuse_to_end_module_connection_for_a_client(
12015 module_connection,
12016 79,
12017 ForwardingError::Poisoned,
12018 ),
12019 Err(RouterError::Forwarding(ForwardingError::Poisoned))
12020 ));
12021 assert!(matches!(
12022 handler.refuse_to_end_module_connection_for_a_client(
12023 module_connection,
12024 80,
12025 ForwardingError::StaleModuleEndpoint,
12026 ),
12027 Err(RouterError::Forwarding(
12028 ForwardingError::StaleModuleEndpoint
12029 ))
12030 ));
12031 }
12032
12033 #[tokio::test]
12046 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
12047 let registry = Arc::new(Registry::default());
12048 let forwarding = Arc::new(ForwardingTable::default());
12049 let supervisor = SupervisorHandle::new();
12050 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12051 let handler =
12052 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12053 .with_supervisor(supervisor);
12054
12055 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12056 hello_via_sink(
12057 &handler,
12058 &target_ctx,
12059 &mut target_rx,
12060 hello_frame("target", PROTOCOL_VERSION, 1),
12061 )
12062 .await;
12063
12064 let wrong_nonce = handler
12068 .handle_control_frame(
12069 &route_ctx(ConnectionId::new(91)).0,
12070 route_open_frame_with_admission_facts(
12071 20,
12072 "target",
12073 unique_project_root("admission-facts"),
12074 Some(subc_control::ConsumerIdentity {
12075 module_id: "fed".to_string(),
12076 launch_nonce: "not-the-real-nonce".to_string(),
12077 }),
12078 None,
12079 ),
12080 )
12081 .await
12082 .unwrap();
12083 assert_eq!(
12084 parse_error(&wrong_nonce[0])["code"],
12085 "bad_consumer_identity",
12086 "a mismatched launch nonce must be refused, not stamped Reserved"
12087 );
12088
12089 let never_spawned = handler
12093 .handle_control_frame(
12094 &route_ctx(ConnectionId::new(92)).0,
12095 route_open_frame_with_admission_facts(
12096 21,
12097 "target",
12098 unique_project_root("admission-facts"),
12099 Some(subc_control::ConsumerIdentity {
12100 module_id: "never-spawned".to_string(),
12101 launch_nonce: "any-nonce".to_string(),
12102 }),
12103 None,
12104 ),
12105 )
12106 .await
12107 .unwrap();
12108 assert_eq!(
12109 parse_error(&never_spawned[0])["code"],
12110 "bad_consumer_identity",
12111 "an unspawned module_id must be refused rather than accepted for lack of a record"
12112 );
12113 }
12114
12115 #[tokio::test]
12138 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
12139 let registry = Arc::new(Registry::default());
12140 let forwarding = Arc::new(ForwardingTable::default());
12141 let supervisor = SupervisorHandle::new();
12142 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12143 let handler =
12144 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12145 .with_supervisor(supervisor);
12146
12147 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
12148 hello_via_sink(
12149 &handler,
12150 &target_ctx,
12151 &mut target_rx,
12152 hello_frame("target", PROTOCOL_VERSION, 1),
12153 )
12154 .await;
12155
12156 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
12157 let route_handler = handler.clone();
12158 let route_task = tokio::spawn(async move {
12159 route_handler
12160 .handle_control_frame(
12161 &client_ctx,
12162 route_open_frame_with_admission_facts(
12163 30,
12164 "target",
12165 unique_project_root("admission-facts"),
12166 Some(subc_control::ConsumerIdentity {
12167 module_id: "fed".to_string(),
12168 launch_nonce: "fed-nonce".to_string(),
12169 }),
12170 None,
12171 ),
12172 )
12173 .await
12174 .unwrap()
12175 });
12176
12177 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12184 .await
12185 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
12186 .expect("module control channel closed before route.bind");
12187 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12188 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12189 panic!("expected route.bind")
12190 };
12191 assert_eq!(
12192 principal,
12193 Some(Principal::Reserved {
12194 module_id: "fed".to_string()
12195 }),
12196 "a correctly attested consumer must be stamped Reserved for its own id"
12197 );
12198
12199 handler
12200 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12201 .await
12202 .unwrap();
12203 assert!(route_task.await.unwrap().is_empty());
12204 assert!(
12205 matches!(
12206 serde_json::from_slice::<ClientControlResponse>(
12207 &client_rx.recv().await.unwrap().body
12208 )
12209 .unwrap(),
12210 ClientControlResponse::RouteOpen { .. }
12211 ),
12212 "the route must actually open, not merely avoid an error"
12213 );
12214 }
12215
12216 #[tokio::test(start_paused = true)]
12217 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
12218 let registry = Arc::new(Registry::default());
12219 let forwarding = Arc::new(ForwardingTable::default());
12220 let supervisor = SupervisorHandle::new();
12221 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12222 let handler =
12223 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12224 .with_supervisor(supervisor);
12225
12226 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
12227 hello_via_sink(
12228 &handler,
12229 &target_ctx,
12230 &mut target_rx,
12231 hello_frame("target", PROTOCOL_VERSION, 1),
12232 )
12233 .await;
12234
12235 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
12236 let direct_handler = handler.clone();
12237 let direct_open = tokio::spawn(async move {
12238 direct_handler
12239 .handle_control_frame(
12240 &direct_ctx,
12241 route_open_frame(2, "target", unique_project_root("route-census-direct")),
12242 )
12243 .await
12244 .unwrap()
12245 });
12246 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12247 .await
12248 .expect("no direct route.bind within 5s")
12249 .expect("target control channel closed before direct route.bind");
12250 handler
12251 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
12252 .await
12253 .unwrap();
12254 assert!(direct_open.await.unwrap().is_empty());
12255 let _ = direct_rx.recv().await.unwrap();
12256
12257 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
12258 let reserved_handler = handler.clone();
12259 let reserved_open = tokio::spawn(async move {
12260 reserved_handler
12261 .handle_control_frame(
12262 &reserved_ctx,
12263 route_open_frame_with_admission_facts(
12264 3,
12265 "target",
12266 unique_project_root("admission-facts"),
12267 Some(ConsumerIdentity {
12268 module_id: "fed".to_string(),
12269 launch_nonce: "fed-nonce".to_string(),
12270 }),
12271 None,
12272 ),
12273 )
12274 .await
12275 .unwrap()
12276 });
12277 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12278 .await
12279 .expect("no reserved route.bind within 5s")
12280 .expect("target control channel closed before reserved route.bind");
12281 handler
12282 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
12283 .await
12284 .unwrap();
12285 assert!(reserved_open.await.unwrap().is_empty());
12286 let _ = reserved_rx.recv().await.unwrap();
12287
12288 forwarding
12289 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
12290 .unwrap();
12291 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
12292 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
12293 module_id: Some("target".to_string()),
12294 })
12295 .unwrap();
12296 let census_frame =
12297 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
12298 let response = handler
12299 .handle_control_frame(&census_ctx, census_frame)
12300 .await
12301 .unwrap()
12302 .pop()
12303 .unwrap();
12304 let actual: Value = serde_json::from_slice(&response.body).unwrap();
12305 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
12306 assert!(matches!(
12307 decoded,
12308 ClientControlResponse::SupervisorRoutes { .. }
12309 ));
12310 let routes = actual["modules"][0]["routes"].as_array().unwrap();
12311 assert_eq!(routes.len(), 2);
12312 assert!(routes.iter().all(|route| route["draining"] == true));
12313 assert!(
12316 routes.iter().all(|route| route["drain_reason"] == "reload"),
12317 "draining routes must name the drain's reason: {routes:?}"
12318 );
12319 assert!(routes.iter().any(|route| {
12320 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
12321 }));
12322 assert!(routes.iter().any(|route| {
12323 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
12324 }));
12325
12326 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12327 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
12328 if std::env::var_os("UPDATE_GOLDEN").is_some() {
12329 std::fs::write(
12330 &golden_path,
12331 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
12332 )
12333 .unwrap();
12334 }
12335 let expected: Value =
12336 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
12337 assert_eq!(actual, expected);
12338 }
12339
12340 async fn query_live_roots(
12341 handler: &ControlHandler,
12342 module_ctx: &RouteCtx,
12343 ) -> ModuleControlResponseToModule {
12344 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
12345 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
12346 let response = handler
12347 .handle_control_frame(module_ctx, frame)
12348 .await
12349 .unwrap()
12350 .pop()
12351 .unwrap();
12352 serde_json::from_slice(&response.body).unwrap()
12353 }
12354
12355 #[tokio::test(start_paused = true)]
12356 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
12357 let registry = Arc::new(Registry::default());
12358 let forwarding = Arc::new(ForwardingTable::default());
12359 let handler = ControlHandler::with_forwarding(registry, forwarding);
12360 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
12361 hello_via_sink(
12362 &handler,
12363 &target_ctx,
12364 &mut target_rx,
12365 hello_frame("target", PROTOCOL_VERSION, 1),
12366 )
12367 .await;
12368 let root = unique_project_root("live-roots-known");
12369 let path = ProjectRootId::from_path_allowing_missing(root.path())
12370 .unwrap()
12371 .as_path()
12372 .to_path_buf();
12373 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
12374 let open_handler = handler.clone();
12375 let opened = tokio::spawn(async move {
12376 open_handler
12377 .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
12378 .await
12379 .unwrap()
12380 });
12381 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12382 .await
12383 .unwrap()
12384 .unwrap();
12385 handler
12386 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
12387 .await
12388 .unwrap();
12389 assert!(opened.await.unwrap().is_empty());
12390 let _ = client_rx.recv().await.unwrap();
12391
12392 let root = unique_project_root("live-roots-pending");
12393 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
12394 .unwrap()
12395 .as_path()
12396 .to_path_buf();
12397 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
12398 let open_handler = handler.clone();
12399 let pending = tokio::spawn(async move {
12400 open_handler
12401 .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
12402 .await
12403 .unwrap()
12404 });
12405 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12406 .await
12407 .unwrap()
12408 .unwrap();
12409 let actual = query_live_roots(&handler, &target_ctx).await;
12410 let ModuleControlResponseToModule::LiveRoots {
12411 roots,
12412 unknown_root_bindings,
12413 total_bindings,
12414 } = actual
12415 else {
12416 panic!("expected live roots")
12417 };
12418 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
12419 assert_eq!(unknown_root_bindings, 0);
12420 assert_eq!(
12421 roots.len(),
12422 2,
12423 "root-known arm must retain each canonical root"
12424 );
12425 assert_eq!(
12426 total_bindings,
12427 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12428 );
12429 let counts = roots
12430 .iter()
12431 .map(|root| (root.project_root.clone(), root.bound, root.pending))
12432 .collect::<Vec<_>>();
12433 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
12434 expected.sort_by(|a, b| a.0.cmp(&b.0));
12435 assert_eq!(
12436 counts, expected,
12437 "roots must sort by path and count pending separately"
12438 );
12439 handler
12440 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
12441 .await
12442 .unwrap();
12443 assert!(pending.await.unwrap().is_empty());
12444 }
12445
12446 #[tokio::test(start_paused = true)]
12447 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
12448 let forwarding = Arc::new(ForwardingTable::default());
12449 let handler =
12450 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12451 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
12452 hello_via_sink(
12453 &handler,
12454 &target_ctx,
12455 &mut target_rx,
12456 hello_frame("target", PROTOCOL_VERSION, 1),
12457 )
12458 .await;
12459 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
12460 let pending = forwarding
12461 .begin_route_bind_relay_for_test(
12462 client_ctx.connection_id,
12463 client_ctx.egress.clone(),
12464 2,
12465 "target",
12466 )
12467 .unwrap();
12468 forwarding
12469 .complete_pending_relay(
12470 target_ctx.connection_id,
12471 pending.corr,
12472 RouteBindRelayOutcome::Accepted,
12473 )
12474 .unwrap();
12475 let actual = query_live_roots(&handler, &target_ctx).await;
12476 let ModuleControlResponseToModule::LiveRoots {
12477 roots,
12478 unknown_root_bindings,
12479 total_bindings,
12480 } = actual
12481 else {
12482 panic!("expected live roots")
12483 };
12484 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
12485 assert_eq!(
12486 unknown_root_bindings, 1,
12487 "unknown-root arm must not read as no bindings"
12488 );
12489 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
12490 assert_eq!(
12491 total_bindings,
12492 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12493 );
12494 }
12495
12496 #[tokio::test(start_paused = true)]
12504 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
12505 let forwarding = Arc::new(ForwardingTable::default());
12506 let handler =
12507 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12508 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
12509 let replies = handler
12510 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
12511 .await
12512 .unwrap();
12513 let queued_by_hello = module_rx.len();
12514
12515 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
12516 let open_handler = handler.clone();
12517 let open = tokio::spawn(async move {
12518 open_handler
12519 .handle_control_frame(
12520 &client_ctx,
12521 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
12522 )
12523 .await
12524 .unwrap()
12525 });
12526 let mut spins = 0;
12528 while module_rx.len() == queued_by_hello {
12529 spins += 1;
12530 assert!(spins < 10_000, "route.open never queued a route.bind");
12531 tokio::task::yield_now().await;
12532 }
12533
12534 for reply in replies {
12536 module_ctx.egress.send(reply).await.unwrap();
12537 }
12538
12539 let first = module_rx.recv().await.unwrap().frame;
12540 assert_eq!(
12541 first.header.ty,
12542 FrameType::HelloAck,
12543 "the first frame a registering module reads must be its HELLO_ACK"
12544 );
12545 assert_eq!(first.header.corr, 7);
12546 let second = module_rx.recv().await.unwrap().frame;
12547 assert_eq!(second.header.ty, FrameType::Request);
12548 assert!(
12549 matches!(
12550 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
12551 ModuleControlRequest::RouteBind { .. }
12552 ),
12553 "the route.bind follows the ack"
12554 );
12555 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
12556
12557 handler
12558 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
12559 .await
12560 .unwrap();
12561 assert!(open.await.unwrap().is_empty());
12562 let _ = client_rx.recv().await.unwrap();
12563 }
12564
12565 #[tokio::test(start_paused = true)]
12566 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
12567 let handler = ControlHandler::with_forwarding(
12568 Arc::new(Registry::default()),
12569 Arc::new(ForwardingTable::default()),
12570 );
12571 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
12572 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
12573 hello_via_sink(
12574 &handler,
12575 &first_ctx,
12576 &mut first_rx,
12577 hello_frame("first", PROTOCOL_VERSION, 1),
12578 )
12579 .await;
12580 hello_via_sink(
12581 &handler,
12582 &second_ctx,
12583 &mut second_rx,
12584 hello_frame("second", PROTOCOL_VERSION, 2),
12585 )
12586 .await;
12587 let root = unique_project_root("second-only");
12588 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
12589 let cloned = handler.clone();
12590 let open = tokio::spawn(async move {
12591 cloned
12592 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
12593 .await
12594 .unwrap()
12595 });
12596 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
12597 .await
12598 .unwrap()
12599 .unwrap();
12600 let first = query_live_roots(&handler, &first_ctx).await;
12601 let second = query_live_roots(&handler, &second_ctx).await;
12602 assert!(
12603 matches!(
12604 first,
12605 ModuleControlResponseToModule::LiveRoots {
12606 total_bindings: 0,
12607 ..
12608 }
12609 ),
12610 "cross-module scope must not expose another module's roots"
12611 );
12612 assert!(
12613 matches!(
12614 second,
12615 ModuleControlResponseToModule::LiveRoots {
12616 total_bindings: 1,
12617 ..
12618 }
12619 ),
12620 "second module must see its pending route"
12621 );
12622 handler
12623 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
12624 .await
12625 .unwrap();
12626 assert!(open.await.unwrap().is_empty());
12627 }
12628
12629 #[tokio::test(start_paused = true)]
12630 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
12631 let handler = ControlHandler::with_forwarding(
12632 Arc::new(Registry::default()),
12633 Arc::new(ForwardingTable::default()),
12634 );
12635 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
12636 hello_via_sink(
12637 &handler,
12638 &target_ctx,
12639 &mut target_rx,
12640 hello_frame("target", PROTOCOL_VERSION, 1),
12641 )
12642 .await;
12643 let actual = query_live_roots(&handler, &target_ctx).await;
12644 let ModuleControlResponseToModule::LiveRoots {
12645 roots,
12646 unknown_root_bindings,
12647 total_bindings,
12648 } = actual
12649 else {
12650 panic!("expected live roots")
12651 };
12652 assert!(roots.is_empty());
12653 assert_eq!(unknown_root_bindings, 0);
12654 assert_eq!(total_bindings, 0);
12655 assert_eq!(
12656 total_bindings,
12657 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12658 );
12659 }
12660
12661 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
12668 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12669 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
12670 let text = std::fs::read_to_string(&path)
12671 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
12672 let vectors: Vec<(String, Value)> = text
12673 .lines()
12674 .filter(|line| !line.trim().is_empty())
12675 .map(|line| {
12676 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
12677 let id = entry["corpus_id"]
12678 .as_str()
12679 .expect("every vector carries a corpus_id")
12680 .to_string();
12681 (id, entry["package"].clone())
12682 })
12683 .collect();
12684 assert_eq!(
12688 vectors.len(),
12689 3,
12690 "vendored fed corpus changed size; re-sync from subc-federation"
12691 );
12692
12693 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
12708 let richest = vectors
12709 .iter()
12710 .filter_map(|(_, package)| package.as_object())
12711 .map(|object| {
12712 object
12713 .keys()
12714 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
12715 .count()
12716 })
12717 .max()
12718 .unwrap_or(0);
12719 assert!(
12720 richest >= 2,
12721 "vendored corpus no longer carries a package with unmodelled fields, \
12722 so the relay test can no longer distinguish a verbatim relay from a lossy one"
12723 );
12724
12725 vectors
12726 }
12727
12728 #[tokio::test]
12736 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12737 for (corpus_id, package) in fed_admission_facts_vectors() {
12738 let registry = Arc::new(Registry::default());
12739 let forwarding = Arc::new(ForwardingTable::default());
12740 let supervisor = SupervisorHandle::new();
12741 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12742 let handler =
12743 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12744 .with_supervisor(supervisor)
12745 .with_admission_facts_config(
12746 Some("fed".to_string()),
12747 Some(vec!["target".to_string()]),
12748 );
12749
12750 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12751 hello_via_sink(
12752 &handler,
12753 &target_ctx,
12754 &mut target_rx,
12755 hello_frame("target", PROTOCOL_VERSION, 1),
12756 )
12757 .await;
12758
12759 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12760 let route_handler = handler.clone();
12761 let expected = package.clone();
12762 let route_task = tokio::spawn(async move {
12763 route_handler
12764 .handle_control_frame(
12765 &client_ctx,
12766 route_open_frame_with_admission_facts(
12767 20,
12768 "target",
12769 unique_project_root("admission-facts"),
12770 Some(subc_control::ConsumerIdentity {
12771 module_id: "fed".to_string(),
12772 launch_nonce: "fed-nonce".to_string(),
12773 }),
12774 Some(package),
12775 ),
12776 )
12777 .await
12778 .unwrap()
12779 });
12780
12781 let bind_frame = target_rx.recv().await.unwrap();
12782 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12783 let ModuleControlRequest::RouteBind {
12784 admission_facts, ..
12785 } = bind
12786 else {
12787 panic!("{corpus_id}: expected route.bind")
12788 };
12789 assert_eq!(
12790 admission_facts,
12791 Some(expected),
12792 "{corpus_id}: relay must not add, drop or reshape any field"
12793 );
12794
12795 handler
12796 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12797 .await
12798 .unwrap();
12799 route_task.await.unwrap();
12800 }
12801 }
12802
12803 #[tokio::test]
12804 async fn route_bind_never_relays_the_openers_build_class() {
12805 let registry = Arc::new(Registry::default());
12806 let forwarding = Arc::new(ForwardingTable::default());
12807 let handler =
12808 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12809 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
12810 hello_via_sink(
12811 &handler,
12812 &target_ctx,
12813 &mut target_rx,
12814 hello_frame("target", PROTOCOL_VERSION, 1),
12815 )
12816 .await;
12817
12818 let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
12819 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
12820 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12821 busy_refusals: None,
12822 target: RouteTarget::ToolProvider {
12823 module_id: "target".to_string(),
12824 },
12825 identity,
12826 consumer_identity: None,
12827 consumer_capabilities: None,
12828 role_versions: None,
12829 admission_facts: None,
12830 scope: None,
12831 })
12832 .unwrap();
12833 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
12834
12835 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
12836 let route_handler = handler.clone();
12837 let route_task = tokio::spawn(async move {
12838 route_handler
12839 .handle_control_frame(&client_ctx, open)
12840 .await
12841 .unwrap()
12842 });
12843 let bind_frame = target_rx.recv().await.unwrap();
12844 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12845 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
12846 panic!("expected route.bind")
12847 };
12848 assert_eq!(
12849 identity.build_class, None,
12850 "route.bind carried the build class"
12851 );
12852 assert!(
12853 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
12854 "the route.bind bytes still name build_class"
12855 );
12856 handler
12857 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12858 .await
12859 .unwrap();
12860 route_task.await.unwrap();
12861 }
12862
12863 #[tokio::test]
12864 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
12865 let registry = Arc::new(Registry::default());
12866 let forwarding = Arc::new(ForwardingTable::default());
12867 let supervisor = SupervisorHandle::new();
12868 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12869 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
12870 let handler =
12871 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12872 .with_supervisor(supervisor)
12873 .with_admission_facts_config(
12874 Some("fed".to_string()),
12875 Some(vec!["target".to_string()]),
12876 );
12877
12878 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
12879 hello_via_sink(
12880 &handler,
12881 &target_ctx,
12882 &mut target_rx,
12883 hello_frame("target", PROTOCOL_VERSION, 1),
12884 )
12885 .await;
12886 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
12887 hello_via_sink(
12888 &handler,
12889 &other_ctx,
12890 &mut other_rx,
12891 hello_frame("other", PROTOCOL_VERSION, 2),
12892 )
12893 .await;
12894
12895 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
12896 let expected_facts = facts.clone();
12897 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
12898 let route_handler = handler.clone();
12899 let route_task = tokio::spawn(async move {
12900 route_handler
12901 .handle_control_frame(
12902 &client_ctx,
12903 route_open_frame_with_admission_facts(
12904 10,
12905 "target",
12906 unique_project_root("admission-facts"),
12907 Some(subc_control::ConsumerIdentity {
12908 module_id: "fed".to_string(),
12909 launch_nonce: "fed-nonce".to_string(),
12910 }),
12911 Some(facts.clone()),
12912 ),
12913 )
12914 .await
12915 .unwrap()
12916 });
12917 let bind_frame = target_rx.recv().await.unwrap();
12918 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12919 let ModuleControlRequest::RouteBind {
12920 admission_facts, ..
12921 } = bind
12922 else {
12923 panic!("expected route.bind")
12924 };
12925 assert_eq!(admission_facts, Some(expected_facts));
12926 handler
12927 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12928 .await
12929 .unwrap();
12930 assert!(route_task.await.unwrap().is_empty());
12931 assert!(matches!(
12932 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
12933 .unwrap(),
12934 ClientControlResponse::RouteOpen { .. }
12935 ));
12936
12937 let direct = handler
12938 .handle_control_frame(
12939 &route_ctx(ConnectionId::new(73)).0,
12940 route_open_frame_with_admission_facts(
12941 11,
12942 "target",
12943 unique_project_root("admission-facts"),
12944 None,
12945 Some(json!({"x": 1})),
12946 ),
12947 )
12948 .await
12949 .unwrap();
12950 assert_eq!(
12951 parse_error(&direct[0])["code"],
12952 "admission_facts_not_permitted"
12953 );
12954
12955 let different_reserved = handler
12956 .handle_control_frame(
12957 &route_ctx(ConnectionId::new(77)).0,
12958 route_open_frame_with_admission_facts(
12959 15,
12960 "target",
12961 unique_project_root("admission-facts"),
12962 Some(subc_control::ConsumerIdentity {
12963 module_id: "other".to_string(),
12964 launch_nonce: "other-nonce".to_string(),
12965 }),
12966 Some(json!({"x": 1})),
12967 ),
12968 )
12969 .await
12970 .unwrap();
12971 assert_eq!(
12972 parse_error(&different_reserved[0])["code"],
12973 "admission_facts_not_permitted"
12974 );
12975
12976 let other_target = handler
12977 .handle_control_frame(
12978 &route_ctx(ConnectionId::new(74)).0,
12979 route_open_frame_with_admission_facts(
12980 12,
12981 "other",
12982 unique_project_root("admission-facts"),
12983 Some(subc_control::ConsumerIdentity {
12984 module_id: "fed".to_string(),
12985 launch_nonce: "fed-nonce".to_string(),
12986 }),
12987 Some(json!({"x": 1})),
12988 ),
12989 )
12990 .await
12991 .unwrap();
12992 assert_eq!(
12993 parse_error(&other_target[0])["code"],
12994 "admission_facts_target_not_allowed"
12995 );
12996
12997 let nonexistent = handler
12998 .handle_control_frame(
12999 &route_ctx(ConnectionId::new(75)).0,
13000 route_open_frame_with_admission_facts(
13001 13,
13002 "missing",
13003 unique_project_root("admission-facts"),
13004 None,
13005 Some(json!({"x": 1})),
13006 ),
13007 )
13008 .await
13009 .unwrap();
13010 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
13011
13012 let described = handler
13013 .handle_control_frame(
13014 &route_ctx(ConnectionId::new(76)).0,
13015 Frame::build(
13016 FrameType::Request,
13017 control_flags(),
13018 0,
13019 0,
13020 14,
13021 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13022 )
13023 .unwrap(),
13024 )
13025 .await
13026 .unwrap();
13027 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13028 serde_json::from_slice(&described[0].body).unwrap()
13029 else {
13030 panic!("expected server.describe response")
13031 };
13032 assert!(capabilities
13033 .iter()
13034 .any(|cap| cap == "admission_facts_relay_v1"));
13035 }
13036
13037 #[tokio::test]
13038 async fn admission_facts_without_configured_carrier_are_rejected() {
13039 let registry = Arc::new(Registry::default());
13040 let forwarding = Arc::new(ForwardingTable::default());
13041 let handler = ControlHandler::with_forwarding(registry, forwarding);
13042 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
13043 hello_via_sink(
13044 &handler,
13045 &target_ctx,
13046 &mut target_rx,
13047 hello_frame("target", PROTOCOL_VERSION, 1),
13048 )
13049 .await;
13050
13051 let responses = handler
13052 .handle_control_frame(
13053 &route_ctx(ConnectionId::new(79)).0,
13054 route_open_frame_with_admission_facts(
13055 16,
13056 "target",
13057 unique_project_root("admission-facts"),
13058 None,
13059 Some(json!({"x": 1})),
13060 ),
13061 )
13062 .await
13063 .unwrap();
13064 assert_eq!(
13065 parse_error(&responses[0])["code"],
13066 "admission_facts_not_permitted"
13067 );
13068 }
13069
13070 #[tokio::test]
13071 async fn route_open_relays_consumer_capabilities_verbatim() {
13072 let registry = Arc::new(Registry::default());
13073 let forwarding = Arc::new(ForwardingTable::default());
13074 let handler =
13075 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13076 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
13077 hello_via_sink(
13078 &handler,
13079 &module_ctx,
13080 &mut module_rx,
13081 hello_frame("aft", PROTOCOL_VERSION, 7),
13082 )
13083 .await;
13084
13085 let expected = vec!["elicitation".to_string(), "roots".to_string()];
13086 let expected_for_request = expected.clone();
13087 let project_root = unique_project_root("consumer-capabilities-present");
13088 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
13089 let route_handler = handler.clone();
13090 let route_task = tokio::spawn(async move {
13091 route_handler
13092 .handle_control_frame(
13093 &client_ctx,
13094 route_open_frame_with_consumer_capabilities(
13095 401,
13096 "aft",
13097 project_root,
13098 Some(expected_for_request),
13099 ),
13100 )
13101 .await
13102 .unwrap()
13103 });
13104 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13105 .await
13106 .unwrap()
13107 .unwrap();
13108 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13109 let ModuleControlRequest::RouteBind {
13110 consumer_capabilities,
13111 ..
13112 } = bind
13113 else {
13114 panic!("expected route.bind request, got {bind:?}");
13115 };
13116 assert_eq!(consumer_capabilities, Some(expected.clone()));
13117
13118 handler
13119 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13120 .await
13121 .unwrap();
13122 let route_response = route_task.await.unwrap();
13123 assert!(route_response.is_empty());
13124 let published = client_rx.recv().await.unwrap();
13125 assert!(matches!(
13126 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13127 ClientControlResponse::RouteOpen { .. }
13128 ));
13129 }
13130
13131 #[tokio::test]
13132 async fn route_open_without_consumer_capabilities_relays_none() {
13133 let registry = Arc::new(Registry::default());
13134 let forwarding = Arc::new(ForwardingTable::default());
13135 let handler =
13136 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13137 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
13138 hello_via_sink(
13139 &handler,
13140 &module_ctx,
13141 &mut module_rx,
13142 hello_frame("aft", PROTOCOL_VERSION, 7),
13143 )
13144 .await;
13145
13146 let project_root = unique_project_root("consumer-capabilities-absent");
13147 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
13148 let route_handler = handler.clone();
13149 let route_task = tokio::spawn(async move {
13150 route_handler
13151 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
13152 .await
13153 .unwrap()
13154 });
13155 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13156 .await
13157 .unwrap()
13158 .unwrap();
13159 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13160 let ModuleControlRequest::RouteBind {
13161 consumer_capabilities,
13162 ..
13163 } = bind
13164 else {
13165 panic!("expected route.bind request, got {bind:?}");
13166 };
13167 assert_eq!(consumer_capabilities, None);
13168
13169 handler
13170 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13171 .await
13172 .unwrap();
13173 let route_response = route_task.await.unwrap();
13174 assert!(route_response.is_empty());
13175 let published = client_rx.recv().await.unwrap();
13176 assert!(matches!(
13177 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13178 ClientControlResponse::RouteOpen { .. }
13179 ));
13180 }
13181
13182 async fn bind_role_versions_for(
13186 sent: Option<BTreeMap<String, String>>,
13187 connection: u64,
13188 ) -> Option<BTreeMap<String, String>> {
13189 let registry = Arc::new(Registry::default());
13190 let forwarding = Arc::new(ForwardingTable::default());
13191 let handler =
13192 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13193 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
13194 hello_via_sink(
13195 &handler,
13196 &module_ctx,
13197 &mut module_rx,
13198 hello_frame("aft", PROTOCOL_VERSION, 7),
13199 )
13200 .await;
13201 let project_root = unique_project_root("role-versions");
13202 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
13203 let route_handler = handler.clone();
13204 let route_task = tokio::spawn(async move {
13205 route_handler
13206 .handle_control_frame(
13207 &client_ctx,
13208 route_open_frame_with_role_versions(403, "aft", project_root, sent),
13209 )
13210 .await
13211 .unwrap()
13212 });
13213 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13214 .await
13215 .expect("a well-formed route.open reaches the module as a bind")
13216 .unwrap();
13217 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13218 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
13219 panic!("expected route.bind request, got {bind:?}");
13220 };
13221 handler
13222 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13223 .await
13224 .unwrap();
13225 assert!(route_task.await.unwrap().is_empty());
13226 let published = client_rx.recv().await.unwrap();
13227 assert!(matches!(
13228 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13229 ClientControlResponse::RouteOpen { .. }
13230 ));
13231 role_versions
13232 }
13233
13234 #[tokio::test]
13235 async fn route_open_relays_role_versions_verbatim() {
13236 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
13237 assert_eq!(
13238 bind_role_versions_for(Some(sent.clone()), 141).await,
13239 Some(sent)
13240 );
13241 }
13242
13243 #[tokio::test]
13246 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
13247 assert_eq!(bind_role_versions_for(None, 143).await, None);
13248 assert_eq!(
13249 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
13250 None
13251 );
13252 }
13253
13254 #[tokio::test]
13255 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
13256 let registry = Arc::new(Registry::default());
13257 let forwarding = Arc::new(ForwardingTable::default());
13258 let handler =
13259 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13260 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
13261 hello_via_sink(
13262 &handler,
13263 &module_ctx,
13264 &mut module_rx,
13265 hello_frame("aft", PROTOCOL_VERSION, 7),
13266 )
13267 .await;
13268
13269 let nine: BTreeMap<String, String> = (0..9)
13270 .map(|index| (format!("role-{index}"), "v1".to_string()))
13271 .collect();
13272 for (label, malformed) in [
13273 (
13274 "invalid role name",
13275 role_versions(&[("Tool_Provider", "v1")]),
13276 ),
13277 ("invalid version", role_versions(&[("tool-provider", "v0")])),
13278 ("nine entries", nine),
13279 ] {
13280 let responses = tokio::time::timeout(
13283 Duration::from_secs(1),
13284 handler.handle_control_frame(
13285 &route_ctx(ConnectionId::new(148)).0,
13286 route_open_frame_with_role_versions(
13287 404,
13288 "aft",
13289 unique_project_root("role-versions-malformed"),
13290 Some(malformed),
13291 ),
13292 ),
13293 )
13294 .await
13295 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
13296 .unwrap();
13297 assert_eq!(responses.len(), 1, "{label}");
13298 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
13299 let error = parse_error(&responses[0]);
13300 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
13301 assert_eq!(
13302 error["detail"]["field"], "role_versions",
13303 "{label}: {error}"
13304 );
13305 assert!(
13306 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
13307 "{label}: a malformed declaration is terminal"
13308 );
13309 assert!(
13310 module_rx.try_recv().is_err(),
13311 "{label}: the module must never see a bind"
13312 );
13313 }
13314 }
13315
13316 #[tokio::test]
13320 async fn route_role_versions_capability_is_advertised() {
13321 let handler = ControlHandler::new(Arc::new(Registry::default()));
13322 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
13323 let ack = hello_via_sink(
13324 &handler,
13325 &ctx,
13326 &mut rx,
13327 hello_frame("m", PROTOCOL_VERSION, 1),
13328 )
13329 .await;
13330 let ack = parse_ack(&ack);
13331 assert!(
13332 ack.subc_capabilities
13333 .iter()
13334 .any(|c| c == "route-role-versions/v1"),
13335 "{:?}",
13336 ack.subc_capabilities
13337 );
13338
13339 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13340 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13341 let reply = handler
13342 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
13343 .await
13344 .unwrap()
13345 .pop()
13346 .unwrap();
13347 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13348 serde_json::from_slice(&reply.body).unwrap()
13349 else {
13350 panic!("not a server.describe reply");
13351 };
13352 assert!(
13353 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
13354 "{capabilities:?}"
13355 );
13356 }
13357
13358 #[tokio::test]
13362 async fn catalog_events_capability_is_advertised() {
13363 let handler = ControlHandler::new(Arc::new(Registry::default()));
13364 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13365 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13366 let reply = handler
13367 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
13368 .await
13369 .unwrap()
13370 .pop()
13371 .unwrap();
13372 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13373 serde_json::from_slice(&reply.body).unwrap()
13374 else {
13375 panic!("not a server.describe reply");
13376 };
13377 assert!(
13378 capabilities
13379 .iter()
13380 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
13381 "{capabilities:?}"
13382 );
13383 }
13384
13385 #[tokio::test]
13386 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
13387 let handler = ControlHandler::new(Arc::new(Registry::default()));
13388 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
13389 let ack = hello_via_sink(
13390 &handler,
13391 &module,
13392 &mut module_rx,
13393 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
13394 )
13395 .await;
13396 let ack = parse_ack(&ack);
13397 assert!(
13398 ack.subc_capabilities
13399 .iter()
13400 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13401 "{:?}",
13402 ack.subc_capabilities
13403 );
13404 assert!(ack
13405 .subc_capabilities
13406 .iter()
13407 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13408
13409 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
13410 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13411 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13412 let reply = handler
13413 .handle_control_frame(&client, frame)
13414 .await
13415 .unwrap()
13416 .pop()
13417 .unwrap();
13418 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13419 serde_json::from_slice(&reply.body).unwrap()
13420 else {
13421 panic!("not a server.describe reply");
13422 };
13423 assert!(
13424 capabilities
13425 .iter()
13426 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13427 "{capabilities:?}"
13428 );
13429 assert!(capabilities
13430 .iter()
13431 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13432 }
13433
13434 #[tokio::test]
13435 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
13436 let handler = ControlHandler::new(Arc::new(Registry::default()));
13437 let (module, mut rx) = route_ctx(ConnectionId::new(156));
13438 let ack = hello_via_sink(
13439 &handler,
13440 &module,
13441 &mut rx,
13442 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
13443 )
13444 .await;
13445 assert!(parse_ack(&ack)
13446 .subc_capabilities
13447 .iter()
13448 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13449 assert!(parse_ack(&ack)
13450 .subc_capabilities
13451 .iter()
13452 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13453 let (client, _rx) = route_ctx(ConnectionId::new(157));
13454 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13455 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13456 let reply = handler
13457 .handle_control_frame(&client, frame)
13458 .await
13459 .unwrap()
13460 .pop()
13461 .unwrap();
13462 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13463 serde_json::from_slice(&reply.body).unwrap()
13464 else {
13465 panic!("server.describe reply")
13466 };
13467 assert!(capabilities
13468 .iter()
13469 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13470 assert!(capabilities
13471 .iter()
13472 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13473 }
13474
13475 #[tokio::test]
13476 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
13477 let handler = ControlHandler::new(Arc::new(Registry::default()));
13478 let (module, mut rx) = route_ctx(ConnectionId::new(158));
13479 let ack = hello_via_sink(
13480 &handler,
13481 &module,
13482 &mut rx,
13483 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
13484 )
13485 .await;
13486 assert!(parse_ack(&ack)
13487 .subc_capabilities
13488 .iter()
13489 .any(|cap| cap == "scope-workdir/v1"));
13490 let (client, _rx) = route_ctx(ConnectionId::new(159));
13491 let frame = Frame::build(
13492 FrameType::Request,
13493 control_flags(),
13494 0,
13495 0,
13496 5,
13497 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13498 )
13499 .unwrap();
13500 let reply = handler
13501 .handle_control_frame(&client, frame)
13502 .await
13503 .unwrap()
13504 .remove(0);
13505 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13506 serde_json::from_slice(&reply.body).unwrap()
13507 else {
13508 panic!("server.describe reply")
13509 };
13510 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
13511 }
13512
13513 #[tokio::test]
13514 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
13515 let handler = ControlHandler::new(Arc::new(Registry::default()));
13516 let (module, mut rx) = route_ctx(ConnectionId::new(160));
13517 let ack = hello_via_sink(
13518 &handler,
13519 &module,
13520 &mut rx,
13521 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
13522 )
13523 .await;
13524 assert!(parse_ack(&ack)
13525 .subc_capabilities
13526 .iter()
13527 .any(|cap| cap == "worker-runs/v1"));
13528 let (client, _rx) = route_ctx(ConnectionId::new(161));
13529 let frame = Frame::build(
13530 FrameType::Request,
13531 control_flags(),
13532 0,
13533 0,
13534 5,
13535 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13536 )
13537 .unwrap();
13538 let reply = handler
13539 .handle_control_frame(&client, frame)
13540 .await
13541 .unwrap()
13542 .remove(0);
13543 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13544 serde_json::from_slice(&reply.body).unwrap()
13545 else {
13546 panic!("server.describe reply")
13547 };
13548 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
13549 }
13550
13551 #[tokio::test]
13552 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
13553 let handler = ControlHandler::new(Arc::new(Registry::default()));
13554 let (module, mut rx) = route_ctx(ConnectionId::new(154));
13555 let ack = hello_via_sink(
13556 &handler,
13557 &module,
13558 &mut rx,
13559 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
13560 )
13561 .await;
13562 assert!(parse_ack(&ack)
13563 .subc_capabilities
13564 .iter()
13565 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13566 let (client, _rx) = route_ctx(ConnectionId::new(155));
13567 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13568 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13569 let reply = handler
13570 .handle_control_frame(&client, frame)
13571 .await
13572 .unwrap()
13573 .pop()
13574 .unwrap();
13575 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13576 serde_json::from_slice(&reply.body).unwrap()
13577 else {
13578 panic!("not a server.describe reply")
13579 };
13580 assert!(capabilities
13581 .iter()
13582 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13583 }
13584
13585 #[tokio::test]
13586 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
13587 let registry = Arc::new(Registry::default());
13588 let forwarding = Arc::new(ForwardingTable::default());
13589 let handler =
13590 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13591 .with_health_probe_timeout(Duration::from_secs(5));
13592 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
13593 hello_via_sink(
13594 &handler,
13595 &module_ctx,
13596 &mut module_rx,
13597 non_routable_hello_frame_with_control_ops(
13598 "mcp",
13599 300,
13600 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13601 ),
13602 )
13603 .await;
13604 assert!(registry
13605 .get_module("mcp")
13606 .unwrap()
13607 .unwrap()
13608 .manifest
13609 .provides
13610 .is_empty());
13611
13612 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
13613 let route_response = handler
13614 .handle_control_frame(
13615 &route_client_ctx,
13616 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
13617 )
13618 .await
13619 .unwrap();
13620 assert_eq!(route_response[0].header.ty, FrameType::Error);
13621 assert_eq!(
13622 parse_error(&route_response[0])["code"],
13623 "target_unavailable"
13624 );
13625 assert!(parse_error(&route_response[0])["message"]
13626 .as_str()
13627 .unwrap()
13628 .contains("does not provide the requested target"));
13629 assert!(module_rx.try_recv().is_err());
13630
13631 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
13632 let health_handler = handler.clone();
13633 let health_task = tokio::spawn(async move {
13634 health_handler
13635 .handle_control_frame(
13636 &health_client_ctx,
13637 supervisor_health_probe_frame(302, "mcp"),
13638 )
13639 .await
13640 .unwrap()
13641 });
13642 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13643 .await
13644 .unwrap()
13645 .unwrap();
13646 assert_eq!(
13647 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13648 ModuleControlRequest::HealthCheck {}
13649 );
13650 handler
13651 .handle_control_frame(
13652 &module_ctx,
13653 health_response(health_frame.header.corr, HealthStatus::Ok),
13654 )
13655 .await
13656 .unwrap();
13657 let health_response = health_task.await.unwrap();
13658 assert_eq!(health_response[0].header.ty, FrameType::Response);
13659 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13660 ClientControlResponse::SupervisorHealthProbe {
13661 module_id, status, ..
13662 } => {
13663 assert_eq!(module_id, "mcp");
13664 assert_eq!(status, HealthStatus::Ok);
13665 }
13666 other => panic!("unexpected health response: {other:?}"),
13667 }
13668
13669 forwarding
13674 .cleanup_connection(module_ctx.connection_id)
13675 .unwrap();
13676 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
13677 let cleanup_response = tokio::time::timeout(
13678 Duration::from_millis(200),
13679 handler.handle_control_frame(
13680 &cleanup_probe_ctx,
13681 supervisor_health_probe_frame(303, "mcp"),
13682 ),
13683 )
13684 .await
13685 .expect("probe should fail immediately when the control lane is gone")
13686 .unwrap();
13687 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
13688 assert_eq!(
13689 parse_error(&cleanup_response[0])["code"],
13690 "target_unavailable"
13691 );
13692 assert!(parse_error(&cleanup_response[0])["message"]
13693 .as_str()
13694 .unwrap()
13695 .contains("no module connection"));
13696
13697 handler
13698 .cleanup_connection(module_ctx.connection_id)
13699 .unwrap();
13700 }
13701
13702 #[tokio::test]
13703 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
13704 let registry = Arc::new(Registry::default());
13705 let supervisor_handle = SupervisorHandle::new();
13706 let supervisor =
13707 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13708 .with_handle(supervisor_handle.clone())
13709 .with_connection_file_path(
13710 std::env::temp_dir()
13711 .join(format!("subc-route-open-warming-{}", std::process::id())),
13712 );
13713 let module = supervisor
13714 .supervise_configured(
13715 ModuleSpec {
13716 carry_containment: false,
13717 module_id: "warming".to_string(),
13718 program: fake_aft_stub_path(),
13719 args: Vec::new(),
13720 env: Vec::new(),
13721 reserved: false,
13722 reserved_prefixes: Vec::new(),
13723 protocol: ModuleProtocol::Subc,
13724 overlap: Default::default(),
13725 },
13726 true,
13727 )
13728 .unwrap();
13729 assert_eq!(module.state().unwrap(), ModuleState::Running);
13730
13731 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13732 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13733 let response = handler
13734 .handle_control_frame(
13735 &ctx,
13736 route_open_frame(304, "warming", unique_project_root("warming")),
13737 )
13738 .await
13739 .unwrap();
13740 module.stop().await.unwrap();
13741
13742 assert_eq!(response[0].header.ty, FrameType::Error);
13743 let error = parse_error(&response[0]);
13744 assert_eq!(error["code"], "module_warming");
13745 assert!(error["message"]
13746 .as_str()
13747 .unwrap()
13748 .contains("state=running, enabled=true, live=false"));
13749 }
13750
13751 #[test]
13752 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13753 let handler = ControlHandler::new(Arc::new(Registry::default()));
13754 let capture = EventCapture::default();
13755 let _subscriber =
13756 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13757 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13758 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
13759 let pending = (0..limit).collect::<Vec<_>>();
13760 let response = handler
13761 .route_open_capacity_refusal(
13762 &ctx,
13763 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
13764 "busy",
13765 pending.len(),
13766 limit,
13767 )
13768 .unwrap();
13769 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13770 let event = capture
13771 .events()
13772 .into_iter()
13773 .find(|event| {
13774 event.target == "control"
13775 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
13776 })
13777 .expect("connection admission refusal event");
13778 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13779 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13780 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13783 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13784 }
13785
13786 #[test]
13787 fn route_open_target_cap_logs_admission_reason_and_capacity() {
13788 let handler = ControlHandler::new(Arc::new(Registry::default()));
13789 let capture = EventCapture::default();
13790 let _subscriber =
13791 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13792 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
13793 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
13794 let guards = (0..limit)
13795 .map(|_| {
13796 handler
13797 .route_bind_concurrency
13798 .try_admit("busy", limit)
13799 .unwrap()
13800 })
13801 .collect::<Vec<_>>();
13802 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
13803 Err(in_flight) => in_flight,
13804 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
13805 };
13806 let response = handler
13807 .route_open_target_capacity_refusal(
13808 &ctx,
13809 &RouteOpenCaller::unknown(),
13810 &route_open_frame(397, "busy", unique_project_root("target-cap")),
13811 "busy",
13812 in_flight,
13813 )
13814 .unwrap();
13815 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13816 let event = capture
13817 .events()
13818 .into_iter()
13819 .find(|event| {
13820 event.target == "control"
13821 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
13822 })
13823 .expect("target admission refusal event");
13824 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13825 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13826 drop(guards);
13827 }
13828
13829 #[cfg(unix)]
13830 #[tokio::test(start_paused = true)]
13831 async fn supervisor_list_shows_a_failed_modules_next_retry() {
13832 let registry = Arc::new(Registry::default());
13833 let handle = SupervisorHandle::new();
13834 let supervisor =
13835 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
13836 .with_handle(handle.clone());
13837 let module = supervisor
13838 .spawn(crate::ModuleSpec {
13839 carry_containment: false,
13840 module_id: "retry-list".into(),
13841 program: PathBuf::from("/bin/sh"),
13842 args: vec!["-c".into(), "exit 17".into()],
13843 env: vec![],
13844 reserved: false,
13845 reserved_prefixes: vec![],
13846 protocol: ModuleProtocol::None,
13847 overlap: Default::default(),
13848 })
13849 .unwrap();
13850 let deadline = std::time::Instant::now() + Duration::from_secs(10);
13851 while module.state().unwrap() != crate::ModuleState::Failed {
13852 assert!(
13853 std::time::Instant::now() < deadline,
13854 "waiting for budget failure"
13855 );
13856 tokio::task::yield_now().await;
13857 }
13858 let handler = ControlHandler::new(registry).with_supervisor(handle);
13859 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13860 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13861 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13862 assert_eq!(body["modules"][0]["state"], "failed");
13863 let retry = body["modules"][0]["next_retry_at_ms"]
13864 .as_u64()
13865 .expect("scheduled retry on the wire");
13866 let now = std::time::SystemTime::now()
13867 .duration_since(std::time::UNIX_EPOCH)
13868 .unwrap()
13869 .as_millis() as u64;
13870 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
13871 module.set_enabled(false).await.unwrap();
13872 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13873 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13874 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13875 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
13876 }
13877
13878 #[tokio::test]
13879 async fn slow_step_before_pid_read_does_not_spend_its_budget() {
13880 let _guard = crate::subtree_resources::fault_read_guard().await;
13881 let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
13882 let supervisor =
13883 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13884 let module = supervisor
13885 .spawn(crate::ModuleSpec {
13886 carry_containment: false,
13887 module_id: "pid-read-budget".into(),
13888 program: std::env::current_exe().unwrap(),
13889 args: vec![
13890 "--exact".into(),
13891 "control::tests::pid_resource_read_fixture".into(),
13892 ],
13893 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13894 .into_iter()
13895 .map(|key| (key.into(), dir.display().to_string()))
13896 .chain(std::iter::once((
13897 "SUBC_SUBTREE_PID_FIXTURE".into(),
13898 "1".into(),
13899 )))
13900 .collect(),
13901 reserved: false,
13902 reserved_prefixes: Vec::new(),
13903 protocol: ModuleProtocol::None,
13904 overlap: Default::default(),
13905 })
13906 .unwrap();
13907 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
13908 while !matches!(
13909 module.child_resource_usage(),
13910 subc_control::ChildResourceUsage::Measured(_)
13911 ) {
13912 assert!(
13913 tokio::time::Instant::now() < ready_deadline,
13914 "fixture process was not measurable"
13915 );
13916 tokio::time::sleep(Duration::from_millis(10)).await;
13917 }
13918 tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
13919 let reading = crate::subtree_resources::read_process(module.clone()).await;
13920 module.stop().await.unwrap();
13921 assert!(
13922 matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
13923 "{reading:?}"
13924 );
13925 }
13926
13927 #[test]
13928 fn pid_resource_read_fixture() {
13929 if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
13930 loop {
13931 std::thread::park_timeout(Duration::from_secs(600));
13932 }
13933 }
13934 }
13935
13936 #[tokio::test]
13937 async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
13938 let _guard = crate::subtree_resources::fault_read_guard().await;
13939 let registry = Arc::new(Registry::default());
13940 let handle = SupervisorHandle::new();
13941 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
13942 .with_handle(handle.clone());
13943 let module = supervisor
13944 .supervise_configured(
13945 crate::ModuleSpec {
13946 carry_containment: false,
13947 module_id: "memory-read".into(),
13948 program: PathBuf::from("disabled-module"),
13949 args: Vec::new(),
13950 env: Vec::new(),
13951 reserved: false,
13952 reserved_prefixes: Vec::new(),
13953 protocol: ModuleProtocol::Subc,
13954 overlap: Default::default(),
13955 },
13956 false,
13957 )
13958 .unwrap();
13959 let (target, release, began) = crate::subtree_resources::held();
13960 module.set_subtree_target_for_test(target);
13961 let handler = ControlHandler::new(registry).with_supervisor(handle);
13962 let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
13963 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13964 let replies = handler
13965 .handle_client_control_request(
13966 &ctx,
13967 frame,
13968 ClientControlRequest::SupervisorList {},
13969 &mut None,
13970 )
13971 .await
13972 .unwrap();
13973 assert!(
13974 replies.is_empty(),
13975 "list reply must be detached from the connection reader"
13976 );
13977 began.await.unwrap();
13978 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13979 let replies = handler
13980 .handle_client_control_request(
13981 &ctx,
13982 frame,
13983 ClientControlRequest::SupervisorSpawnSnapshot {},
13984 &mut None,
13985 )
13986 .await
13987 .unwrap();
13988 assert_eq!(
13989 replies.len(),
13990 1,
13991 "another request must complete while the subtree read stalls"
13992 );
13993 let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
13994 .await
13995 .unwrap()
13996 .unwrap();
13997 release.send(()).unwrap();
13998 drop(release);
13999 let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
14000 assert_eq!(body["modules"][0]["subtree"]["source"], "unavailable");
14001 assert_eq!(body["modules"][0]["subtree"]["reason"], "timeout");
14002 assert_eq!(
14003 body["modules"][0]["subtree"]["process_count"], 1,
14004 "a count obtained before the stalled read must survive timeout"
14005 );
14006 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14007 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14008 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14009 assert_eq!(
14010 body["modules"][0]["subtree"]["source"], "process_sum",
14011 "the next request must read afresh"
14012 );
14013 }
14014
14015 #[tokio::test]
14016 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
14017 let registry = Arc::new(Registry::default());
14018 let handle = SupervisorHandle::new();
14019 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14020 .with_handle(handle.clone());
14021 supervisor
14022 .supervise_configured(
14023 crate::ModuleSpec {
14024 carry_containment: false,
14025 module_id: "drift".into(),
14026 program: PathBuf::from("disabled-module"),
14027 args: Vec::new(),
14028 env: Vec::new(),
14029 reserved: false,
14030 reserved_prefixes: Vec::new(),
14031 protocol: ModuleProtocol::Subc,
14032 overlap: Default::default(),
14033 },
14034 false,
14035 )
14036 .unwrap();
14037 let handler = ControlHandler::new(registry).with_supervisor(handle);
14038 for configured in [None, Some(false), Some(true)] {
14039 handler.capability_evaluator.configure(
14040 configured.map(|enabled| ("drift".to_string(), enabled)),
14041 BTreeMap::new(),
14042 );
14043 let frame =
14044 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14045 let replies = handler
14046 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
14047 .await
14048 .unwrap();
14049 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14050 assert!(body.get("configured_enabled").is_none());
14051 assert_eq!(body["op"], "supervisor.ack");
14052 let frame =
14053 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14054 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14055 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14056 assert_eq!(
14057 body["modules"][0].get("configured_enabled"),
14058 configured.map(Value::Bool).as_ref()
14059 );
14060 assert_eq!(body["modules"][0]["subtree"]["source"], "unavailable");
14061 assert_eq!(body["modules"][0]["subtree"]["reason"], "not_running");
14062 }
14063 }
14064
14065 #[tokio::test(flavor = "current_thread")]
14066 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
14067 let handler = ControlHandler::new(Arc::new(Registry::default()));
14068 let capture = EventCapture::default();
14069 let _subscriber =
14070 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14071 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
14072 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14073
14074 let _ = handler
14075 .handle_client_control_request(
14076 &ctx,
14077 frame,
14078 ClientControlRequest::SupervisorSetEnabled {
14079 module_id: "broca".to_string(),
14080 enabled: false,
14081 },
14082 &mut None,
14083 )
14084 .await
14085 .unwrap();
14086
14087 let events = capture
14088 .events()
14089 .into_iter()
14090 .filter(|event| {
14091 event.target == "control"
14092 && event.fields.get("message").map(String::as_str)
14093 == Some("supervisor request received")
14094 })
14095 .collect::<Vec<_>>();
14096 assert_eq!(events.len(), 1, "one supervisor request log line");
14097 let event = &events[0];
14098 assert_eq!(event.level, tracing::Level::INFO);
14099 assert_eq!(
14100 event.fields.get("op"),
14101 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
14102 );
14103 assert_eq!(
14104 event.fields.get("module_id"),
14105 Some(&"\"broca\"".to_string())
14106 );
14107 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
14108 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
14109 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
14110 }
14111
14112 #[tokio::test(flavor = "current_thread")]
14113 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
14114 let registry = Arc::new(Registry::default());
14115 let module_connection = ConnectionId::new(303);
14116 registry
14117 .register_with_control_ops(
14118 manifest("aft", PROTOCOL_VERSION),
14119 PROTOCOL_VERSION,
14120 module_connection,
14121 module_baseline_control_ops(),
14122 )
14123 .unwrap();
14124 let handler = ControlHandler::new(registry);
14125 let capture = EventCapture::default();
14126 let _subscriber =
14127 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14128 let (ctx, _rx) = route_ctx(module_connection);
14129 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14130
14131 let _ = handler
14132 .handle_client_control_request(
14133 &ctx,
14134 frame,
14135 ClientControlRequest::SupervisorSetEnabled {
14136 module_id: "broca".to_string(),
14137 enabled: false,
14138 },
14139 &mut None,
14140 )
14141 .await
14142 .unwrap();
14143
14144 let callers = capture
14145 .events()
14146 .into_iter()
14147 .filter(|event| {
14148 event.fields.get("message").map(String::as_str)
14149 == Some("supervisor request received")
14150 })
14151 .map(|event| event.fields.get("caller").cloned())
14152 .collect::<Vec<_>>();
14153 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
14154 }
14155
14156 #[tokio::test(flavor = "current_thread")]
14157 async fn supervisor_list_does_not_log_request_received() {
14158 let handler = ControlHandler::new(Arc::new(Registry::default()));
14159 let capture = EventCapture::default();
14160 let _subscriber =
14161 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14162 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
14163 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14164
14165 handler
14166 .handle_client_control_request(
14167 &ctx,
14168 frame,
14169 ClientControlRequest::SupervisorList {},
14170 &mut None,
14171 )
14172 .await
14173 .unwrap();
14174
14175 assert!(capture.events().into_iter().all(|event| {
14176 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14177 }));
14178 }
14179
14180 #[tokio::test(flavor = "current_thread")]
14181 async fn supervisor_rescan_preview_does_not_log_request_received() {
14182 let handler = ControlHandler::new(Arc::new(Registry::default()));
14183 let capture = EventCapture::default();
14184 let _subscriber =
14185 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14186 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
14187 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14188
14189 handler
14190 .handle_client_control_request(
14191 &ctx,
14192 frame,
14193 ClientControlRequest::SupervisorRescan { preview: true },
14194 &mut None,
14195 )
14196 .await
14197 .unwrap();
14198
14199 assert!(capture.events().into_iter().all(|event| {
14200 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14201 }));
14202 }
14203
14204 struct ApproveOperator;
14205 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
14206 fn prompt(
14207 &self,
14208 _: &str,
14209 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
14210 ) -> crate::operator_confirm::ProviderResult {
14211 crate::operator_confirm::ProviderResult::Approved
14212 }
14213 }
14214
14215 #[tokio::test]
14216 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
14217 assert_eq!(
14218 ControlHandler::default().privileged_operator_prompt,
14219 PrivilegedOperatorPrompt::Log
14220 );
14221 for (initial, changed, running) in [
14222 ("log", "enforce", PrivilegedOperatorPrompt::Log),
14223 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
14224 ] {
14225 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
14226 let path = dir.join("subc.jsonc");
14227 std::fs::write(
14228 &path,
14229 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
14230 )
14231 .unwrap();
14232 let config = crate::daemon_config::load(&path).unwrap().unwrap();
14233 let registry = Arc::new(Registry::default());
14234 let handle = SupervisorHandle::new();
14235 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
14236 .with_handle(handle.clone());
14237 let handler = ControlHandler::new(registry)
14238 .with_supervisor(handle)
14239 .with_privileged_operator_prompt(config.privileged_operator_prompt)
14240 .with_supervisor_rescan(supervisor, &path, config.port);
14241 handler.forwarding.operator_confirms().configure(
14242 Arc::new(ApproveOperator),
14243 crate::operator_confirm::OperatorLimits::default(),
14244 );
14245 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
14246
14247 for (disk_mode, needs_restart) in [
14248 (Some(initial), false),
14249 (Some(changed), true),
14250 (Some(changed), true),
14251 (None, running == PrivilegedOperatorPrompt::Enforce),
14252 (Some(initial), false),
14253 ] {
14254 let mut doc = json!({"version":1});
14255 if let Some(mode) = disk_mode {
14256 doc["privileged_operator_prompt"] = json!(mode);
14257 }
14258 std::fs::write(&path, doc.to_string()).unwrap();
14259 for preview in [true, false] {
14260 let request = ClientControlRequest::SupervisorRescan { preview };
14261 let frame = Frame::build(
14262 FrameType::Request,
14263 control_flags(),
14264 0,
14265 0,
14266 1,
14267 serde_json::to_vec(&request).unwrap(),
14268 )
14269 .unwrap();
14270 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14271 assert_eq!(frames.len(), 1);
14272 let ClientControlResponse::SupervisorRescan { result } =
14273 serde_json::from_slice(&frames[0].body).unwrap()
14274 else {
14275 panic!("expected a rescan response");
14276 };
14277 let expected = if needs_restart {
14278 vec!["privileged_operator_prompt"]
14279 } else {
14280 Vec::new()
14281 };
14282 assert_eq!(result.restart_required, expected);
14283 assert_eq!(handler.privileged_operator_prompt, running);
14284 assert_eq!(
14285 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
14286 running
14287 );
14288 }
14289 }
14290 }
14291 }
14292
14293 #[tokio::test]
14294 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
14295 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
14296 let path = dir.join("subc.jsonc");
14297 let registry = Arc::new(Registry::default());
14298 let handle = SupervisorHandle::new();
14299 let supervisor =
14300 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
14301 let handler = ControlHandler::new(registry)
14302 .with_supervisor(handle)
14303 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
14304 .with_supervisor_rescan(supervisor, &path, None);
14305 handler.forwarding.operator_confirms().configure(
14306 Arc::new(ApproveOperator),
14307 crate::operator_confirm::OperatorLimits::default(),
14308 );
14309 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
14310 for preview in [true, false] {
14311 std::fs::write(
14312 &path,
14313 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
14314 )
14315 .unwrap();
14316 let request = ClientControlRequest::SupervisorRescan { preview };
14317 let frame = Frame::build(
14318 FrameType::Request,
14319 control_flags(),
14320 0,
14321 0,
14322 1,
14323 serde_json::to_vec(&request).unwrap(),
14324 )
14325 .unwrap();
14326 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14327 assert_eq!(frames.len(), 1);
14328 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
14329 assert_eq!(
14330 handler.privileged_operator_prompt,
14331 PrivilegedOperatorPrompt::Enforce
14332 );
14333 }
14334 }
14335
14336 #[tokio::test]
14341 async fn route_open_refusal_names_the_check_that_refused() {
14342 let handler = ControlHandler::new(Arc::new(Registry::default()));
14343 let capture = EventCapture::default();
14344 let _subscriber =
14345 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14346 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14347 let response = handler
14348 .handle_control_frame(
14349 &ctx,
14350 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
14351 )
14352 .await
14353 .unwrap();
14354
14355 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14356 let event = capture
14357 .events()
14358 .into_iter()
14359 .find(|event| {
14360 event.target == "control"
14361 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14362 })
14363 .expect("route.open refusal event");
14364 assert_eq!(
14365 event.fields.get("reason"),
14366 Some(&"\"not_registered\"".to_string())
14367 );
14368 }
14369
14370 #[tokio::test]
14371 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
14372 let registry = Arc::new(Registry::default());
14373 let supervisor_handle = SupervisorHandle::new();
14374 let supervisor =
14375 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
14376 .with_handle(supervisor_handle.clone())
14377 .with_connection_file_path(std::env::temp_dir().join(format!(
14378 "subc-route-open-refusal-info-{}",
14379 std::process::id()
14380 )));
14381 let module = supervisor
14382 .supervise_configured(
14383 ModuleSpec {
14384 carry_containment: false,
14385 module_id: "warming".to_string(),
14386 program: fake_aft_stub_path(),
14387 args: Vec::new(),
14388 env: Vec::new(),
14389 reserved: false,
14390 reserved_prefixes: Vec::new(),
14391 protocol: ModuleProtocol::Subc,
14392 overlap: Default::default(),
14393 },
14394 true,
14395 )
14396 .unwrap();
14397 assert_eq!(module.state().unwrap(), ModuleState::Running);
14398
14399 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14400 assert!(handler
14401 .counters()
14402 .snapshot()
14403 .get("route_open_refused_by_code")
14404 .is_none());
14405 let capture = EventCapture::default();
14406 let _subscriber =
14407 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14408 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
14409 let response = handler
14410 .handle_control_frame(
14411 &ctx,
14412 route_open_frame(394, "warming", unique_project_root("refusal-info")),
14413 )
14414 .await
14415 .unwrap();
14416 module.stop().await.unwrap();
14417
14418 assert_eq!(parse_error(&response[0])["code"], "module_warming");
14419 let event = capture
14420 .events()
14421 .into_iter()
14422 .find(|event| {
14423 event.target == "control"
14424 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
14425 })
14426 .expect("route.open refusal event");
14427 assert_eq!(
14428 event.fields.get("module_id"),
14429 Some(&"\"warming\"".to_string())
14430 );
14431 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
14432 assert_eq!(
14433 event.fields.get("reason"),
14434 Some(&"\"supervised_not_registered\"".to_string())
14435 );
14436 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
14437 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
14438 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
14439 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14440 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14441 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
14442 assert_eq!(
14443 handler.counters().snapshot()["route_open_refused_by_code"],
14444 json!({ "module_warming": 1 })
14445 );
14446 }
14447
14448 const OUTAGE_START: &str = "route.open refusing module: not serving";
14449 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
14450
14451 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
14452 capture
14453 .events()
14454 .into_iter()
14455 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
14456 .collect()
14457 }
14458
14459 fn supervise_stub(
14460 registry: &Arc<Registry>,
14461 module_id: &str,
14462 enabled: bool,
14463 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
14464 let supervisor_handle = SupervisorHandle::new();
14465 let supervisor =
14466 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
14467 .with_handle(supervisor_handle.clone())
14468 .with_connection_file_path(std::env::temp_dir().join(format!(
14469 "subc-route-outage-{module_id}-{}",
14470 std::process::id()
14471 )));
14472 let module = supervisor
14473 .supervise_configured(
14474 ModuleSpec {
14475 carry_containment: false,
14476 module_id: module_id.to_string(),
14477 program: fake_aft_stub_path(),
14478 args: Vec::new(),
14479 env: Vec::new(),
14480 reserved: false,
14481 reserved_prefixes: Vec::new(),
14482 protocol: ModuleProtocol::Subc,
14483 overlap: Default::default(),
14484 },
14485 enabled,
14486 )
14487 .unwrap();
14488 (supervisor_handle, module)
14489 }
14490
14491 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
14492 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
14493 module_id: module_id.to_string(),
14494 drain_timeout_ms: Some(50),
14495 })
14496 .unwrap();
14497 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
14498 }
14499
14500 #[test]
14504 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
14505 let registry = Arc::new(Registry::default());
14506 let forwarding = Arc::new(ForwardingTable::default());
14507 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14508 let second = ControlHandler::with_forwarding(registry, forwarding);
14509 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
14510 }
14511
14512 #[tokio::test(flavor = "current_thread")]
14516 async fn route_open_unknown_module_refusals_add_no_outage_state() {
14517 let handler = ControlHandler::new(Arc::new(Registry::default()));
14518 let capture = EventCapture::default();
14519 let _subscriber =
14520 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14521 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
14522 for corr in 0..8 {
14523 let response = handler
14524 .handle_control_frame(
14525 &ctx,
14526 route_open_frame(
14527 380 + corr,
14528 &format!("nobody-{corr}"),
14529 unique_project_root("outage-unknown"),
14530 ),
14531 )
14532 .await
14533 .unwrap();
14534 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14535 }
14536
14537 assert_eq!(handler.route_outages.tracked_module_count(), 0);
14538 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
14539 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
14540 }
14541
14542 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14547 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
14548 let registry = Arc::new(Registry::default());
14549 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
14550 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14551 let capture = EventCapture::default();
14552 let _subscriber =
14553 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14554 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
14555 handler.route_outages.record_accepted("outage-restart");
14558
14559 let response = handler
14560 .handle_control_frame(
14561 &ctx,
14562 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
14563 )
14564 .await
14565 .unwrap();
14566 assert_eq!(response[0].header.ty, FrameType::Error);
14567 let starts = outage_lines(&capture, OUTAGE_START);
14568 assert_eq!(starts.len(), 1, "{starts:?}");
14569 assert_eq!(starts[0].level, tracing::Level::WARN);
14570 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
14571 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
14572 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
14573 handler.route_outages.record_accepted("outage-restart");
14574 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
14575
14576 let restart = handler
14577 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
14578 .await
14579 .unwrap();
14580 assert_eq!(
14581 restart[0].header.ty,
14582 FrameType::Response,
14583 "{:?}",
14584 parse_error(&restart[0])
14585 );
14586 handler
14587 .handle_control_frame(
14588 &ctx,
14589 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
14590 )
14591 .await
14592 .unwrap();
14593 module.stop().await.unwrap();
14594
14595 let starts = outage_lines(&capture, OUTAGE_START);
14596 assert_eq!(starts.len(), 2, "{starts:?}");
14597 assert_eq!(starts[1].level, tracing::Level::INFO);
14598 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
14599 }
14600
14601 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14605 async fn failed_operator_restart_leaves_no_operator_mark() {
14606 let registry = Arc::new(Registry::default());
14607 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
14608 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14609 let capture = EventCapture::default();
14610 let _subscriber =
14611 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14612 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
14613 handler.route_outages.record_accepted("outage-disabled");
14614
14615 let restart = handler
14616 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
14617 .await
14618 .unwrap();
14619 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
14620 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
14621
14622 handler
14623 .handle_control_frame(
14624 &ctx,
14625 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
14626 )
14627 .await
14628 .unwrap();
14629 let starts = outage_lines(&capture, OUTAGE_START);
14630 assert_eq!(starts.len(), 1, "{starts:?}");
14631 assert_eq!(starts[0].level, tracing::Level::WARN);
14632 }
14633
14634 #[tokio::test(flavor = "current_thread")]
14635 async fn route_open_unknown_module_escapes_target_module_id() {
14636 let handler = ControlHandler::new(Arc::new(Registry::default()));
14637 let capture = EventCapture::default();
14638 let _subscriber =
14639 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14640 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
14641 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14642 let response = handler
14643 .handle_control_frame(
14644 &ctx,
14645 route_open_frame(
14646 395,
14647 hostile_module_id,
14648 unique_project_root("hostile-target-module-id"),
14649 ),
14650 )
14651 .await
14652 .unwrap();
14653
14654 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14655 let event = capture
14656 .events()
14657 .into_iter()
14658 .find(|event| {
14659 event.target == "control"
14660 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14661 })
14662 .expect("route.open unknown-module refusal event");
14663 let logged = event.fields.get("module_id").expect("module_id field");
14664 assert!(!logged.bytes().any(|byte| byte < 0x20));
14665 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14666 }
14667
14668 #[tokio::test(flavor = "current_thread")]
14671 async fn route_open_refusal_carries_daemon_origin() {
14672 let handler = ControlHandler::new(Arc::new(Registry::default()));
14673 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14674 let response = handler
14675 .handle_control_frame(
14676 &ctx,
14677 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
14678 )
14679 .await
14680 .unwrap();
14681
14682 assert_eq!(response.len(), 1);
14683 assert_eq!(response[0].header.ty, FrameType::Error);
14684 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14685 assert!(
14686 response[0].header.flags.is_daemon_origin(),
14687 "route.open refusal must carry daemon origin: {:?}",
14688 response[0].header.flags
14689 );
14690 }
14691
14692 #[tokio::test(flavor = "current_thread")]
14696 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
14697 let registry = Arc::new(Registry::default());
14698 let forwarding = Arc::new(ForwardingTable::default());
14699 let handler =
14700 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14701 let module_connection = ConnectionId::new(95);
14702 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14703 hello_via_sink(
14704 &handler,
14705 &module_ctx,
14706 &mut module_rx,
14707 hello_frame("aft", PROTOCOL_VERSION, 395),
14708 )
14709 .await;
14710 forwarding
14711 .begin_module_drain("aft", RouteCloseReason::Restart)
14712 .unwrap()
14713 .expect("the module endpoint drains");
14714
14715 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
14716 let response = handler
14717 .handle_control_frame(
14718 &client_ctx,
14719 route_open_frame(396, "aft", unique_project_root("drain-origin")),
14720 )
14721 .await
14722 .unwrap();
14723
14724 assert_eq!(response.len(), 1);
14725 assert_eq!(response[0].header.ty, FrameType::Error);
14726 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14727 assert!(
14728 response[0].header.flags.is_daemon_origin(),
14729 "drain module_reloading must carry daemon origin: {:?}",
14730 response[0].header.flags
14731 );
14732 }
14733
14734 #[tokio::test(flavor = "current_thread")]
14738 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
14739 let registry = Arc::new(Registry::default());
14740 let forwarding = Arc::new(ForwardingTable::default());
14741 let handler =
14742 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14743 let module_connection = ConnectionId::new(95);
14744 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14745 hello_via_sink(
14746 &handler,
14747 &module_ctx,
14748 &mut module_rx,
14749 hello_frame("aft", PROTOCOL_VERSION, 395),
14750 )
14751 .await;
14752
14753 let client_connection = ConnectionId::new(96);
14754 let (client_ctx, _client_rx) = route_ctx(client_connection);
14755 let (route_task, bind) = relay_route_open(
14756 &handler,
14757 client_connection,
14758 &client_ctx.egress,
14759 &mut module_rx,
14760 396,
14761 "aft",
14762 "spoofed-reloading",
14763 )
14764 .await;
14765 let rejection = Frame::build(
14768 FrameType::Error,
14769 crate::router::daemon_error_flags(),
14770 0,
14771 0,
14772 bind.header.corr,
14773 serde_json::to_vec(&ErrorBody::new(
14774 "module_reloading",
14775 "module refused route.bind",
14776 ))
14777 .unwrap(),
14778 )
14779 .unwrap();
14780 handler
14781 .handle_control_frame(&module_ctx, rejection)
14782 .await
14783 .unwrap();
14784
14785 let response = route_task.await.unwrap();
14786 assert_eq!(response.len(), 1);
14787 assert_eq!(response[0].header.ty, FrameType::Error);
14788 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14789 assert!(
14790 !response[0].header.flags.is_daemon_origin(),
14791 "a module's own rejection must not carry daemon origin: {:?}",
14792 response[0].header.flags
14793 );
14794 }
14795
14796 #[tokio::test(flavor = "current_thread")]
14797 async fn route_open_module_rejection_uses_daemon_counter_key() {
14798 let registry = Arc::new(Registry::default());
14799 let forwarding = Arc::new(ForwardingTable::default());
14800 let handler =
14801 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14802 let module_connection = ConnectionId::new(95);
14803 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14804 hello_via_sink(
14805 &handler,
14806 &module_ctx,
14807 &mut module_rx,
14808 hello_frame("aft", PROTOCOL_VERSION, 395),
14809 )
14810 .await;
14811
14812 let client_connection = ConnectionId::new(96);
14813 let (client_ctx, _client_rx) = route_ctx(client_connection);
14814 let capture = EventCapture::default();
14815 let _subscriber =
14816 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14817 let (route_task, bind) = relay_route_open(
14818 &handler,
14819 client_connection,
14820 &client_ctx.egress,
14821 &mut module_rx,
14822 396,
14823 "aft",
14824 "hostile-module-code",
14825 )
14826 .await;
14827 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
14828 let rejection = Frame::build(
14829 FrameType::Error,
14830 control_flags(),
14831 0,
14832 0,
14833 bind.header.corr,
14834 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
14835 )
14836 .unwrap();
14837 handler
14838 .handle_control_frame(&module_ctx, rejection)
14839 .await
14840 .unwrap();
14841
14842 let response = route_task.await.unwrap();
14843 assert_eq!(parse_error(&response[0])["code"], hostile_code);
14844 let counters = handler.counters().snapshot();
14845 assert_eq!(
14846 counters["route_open_refused_by_code"],
14847 json!({ "module_rejected": 1 })
14848 );
14849 assert!(counters["route_open_refused_by_code"]
14850 .get(hostile_code)
14851 .is_none());
14852
14853 let event = capture
14854 .events()
14855 .into_iter()
14856 .find(|event| {
14857 event.target == "control"
14858 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
14859 })
14860 .expect("route.open module-rejection refusal event");
14861 let logged = event.fields.get("module_code").expect("module_code field");
14862 assert!(!logged.bytes().any(|byte| byte < 0x20));
14863 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14864 }
14865
14866 #[tokio::test]
14867 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
14868 let registry = Arc::new(Registry::default());
14869 let supervisor_handle = SupervisorHandle::new();
14870 let missing_program = std::env::temp_dir().join(format!(
14871 "subc-route-open-missing-program-{}",
14872 std::process::id()
14873 ));
14874 let supervisor =
14875 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
14876 .with_handle(supervisor_handle.clone());
14877 let module = supervisor
14878 .supervise_configured(
14879 ModuleSpec {
14880 carry_containment: false,
14881 module_id: "failed".to_string(),
14882 program: missing_program,
14883 args: Vec::new(),
14884 env: Vec::new(),
14885 reserved: false,
14886 reserved_prefixes: Vec::new(),
14887 protocol: ModuleProtocol::Subc,
14888 overlap: Default::default(),
14889 },
14890 true,
14891 )
14892 .unwrap();
14893 assert_eq!(module.state().unwrap(), ModuleState::Failed);
14894
14895 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14896 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
14897 let response = handler
14898 .handle_control_frame(
14899 &ctx,
14900 route_open_frame(305, "failed", unique_project_root("failed")),
14901 )
14902 .await
14903 .unwrap();
14904
14905 assert_eq!(response[0].header.ty, FrameType::Error);
14906 let error = parse_error(&response[0]);
14907 assert_eq!(error["code"], "target_unavailable");
14908 assert!(error["message"]
14909 .as_str()
14910 .unwrap()
14911 .contains("state=failed, enabled=true, live=false"));
14912 }
14913
14914 #[tokio::test]
14915 async fn route_open_role_mismatch_remains_target_unavailable() {
14916 let registry = Arc::new(Registry::default());
14917 let handler = ControlHandler::new(Arc::clone(®istry));
14918 handler
14919 .handle_control(
14920 ConnectionId::new(41),
14921 non_routable_hello_frame_with_control_ops("health-only", 306, None),
14922 )
14923 .unwrap();
14924
14925 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
14926 let response = handler
14927 .handle_control_frame(
14928 &ctx,
14929 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
14930 )
14931 .await
14932 .unwrap();
14933
14934 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14935 assert!(parse_error(&response[0])["message"]
14936 .as_str()
14937 .unwrap()
14938 .contains("does not provide the requested target"));
14939 }
14940
14941 #[tokio::test]
14942 async fn route_open_inactive_registration_remains_target_unavailable() {
14943 let registry = Arc::new(Registry::default());
14944 let handler = ControlHandler::new(Arc::clone(®istry));
14945 handler
14946 .handle_control(
14947 ConnectionId::new(43),
14948 hello_frame("inactive", PROTOCOL_VERSION, 308),
14949 )
14950 .unwrap();
14951 assert!(registry
14952 .set_module_state_for_test("inactive", ChannelState::Closed)
14953 .unwrap());
14954
14955 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
14956 let response = handler
14957 .handle_control_frame(
14958 &ctx,
14959 route_open_frame(309, "inactive", unique_project_root("inactive")),
14960 )
14961 .await
14962 .unwrap();
14963
14964 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14965 assert!(parse_error(&response[0])["message"]
14966 .as_str()
14967 .unwrap()
14968 .contains("is not active"));
14969 }
14970
14971 #[tokio::test]
14972 async fn late_health_reply_is_recorded_through_the_module_response_path() {
14973 let registry = Arc::new(Registry::default());
14974 let forwarding = Arc::new(ForwardingTable::default());
14975 let supervisor_handle = SupervisorHandle::new();
14976 let supervisor =
14977 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
14978 .with_forwarding(Arc::clone(&forwarding))
14979 .with_handle(supervisor_handle.clone());
14980 let module = supervisor
14981 .supervise_configured(
14982 crate::ModuleSpec {
14983 carry_containment: false,
14984 module_id: "late-health-response".to_string(),
14985 program: PathBuf::from("disabled-module"),
14986 args: Vec::new(),
14987 env: Vec::new(),
14988 reserved: false,
14989 reserved_prefixes: Vec::new(),
14990 protocol: ModuleProtocol::Subc,
14991 overlap: Default::default(),
14992 },
14993 false,
14994 )
14995 .unwrap();
14996 let handler =
14997 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14998 .with_supervisor(supervisor_handle);
14999 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
15000 handler
15001 .handle_control_frame(
15002 &module_ctx,
15003 hello_frame_with_control_ops(
15004 "late-health-response",
15005 PROTOCOL_VERSION,
15006 7,
15007 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15008 ),
15009 )
15010 .await
15011 .unwrap();
15012 let probe_started_at = Instant::now() - Duration::from_millis(80);
15013 let pending = forwarding
15014 .begin_health_probe_rpc_for(
15015 "late-health-response",
15016 MODULE_CONTROL_OP_HEALTH_CHECK,
15017 probe_started_at,
15018 Instant::now() - Duration::from_millis(1),
15019 )
15020 .unwrap();
15021 assert!(forwarding
15022 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
15023 .unwrap());
15024
15025 let responses = handler
15026 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
15027 .await
15028 .unwrap();
15029
15030 assert!(responses.is_empty());
15031 let health = module.status().unwrap().health;
15032 assert_eq!(health.late_answer_count, 1);
15033 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
15034 }
15035
15036 #[tokio::test]
15037 async fn health_probe_timeout_and_module_death_are_typed() {
15038 let registry = Arc::new(Registry::default());
15039 let forwarding = Arc::new(ForwardingTable::default());
15040 let handler =
15041 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15042 .with_health_probe_timeout(Duration::from_millis(50));
15043 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
15044 hello_via_sink(
15045 &handler,
15046 &module_ctx,
15047 &mut module_rx,
15048 hello_frame_with_control_ops(
15049 "aft",
15050 PROTOCOL_VERSION,
15051 7,
15052 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15053 ),
15054 )
15055 .await;
15056
15057 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
15058 let responses = handler
15059 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
15060 .await
15061 .unwrap();
15062 assert_eq!(responses[0].header.ty, FrameType::Error);
15063 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
15064 let _ = module_rx.try_recv();
15065
15066 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
15067 let health_handler = handler.clone();
15068 let death_task = tokio::spawn(async move {
15069 health_handler
15070 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
15071 .await
15072 .unwrap()
15073 });
15074 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15075 .await
15076 .unwrap()
15077 .unwrap();
15078 handler
15079 .cleanup_connection(module_ctx.connection_id)
15080 .unwrap();
15081 let responses = death_task.await.unwrap();
15082 assert_eq!(responses[0].header.ty, FrameType::Error);
15083 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
15084 }
15085
15086 #[test]
15087 fn hello_requires_exact_protocol_version() {
15088 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
15089 let registry = Arc::new(Registry::default());
15090 let handler = ControlHandler::new(Arc::clone(®istry));
15091 let responses = handler
15092 .handle_control(
15093 ConnectionId::new(connection),
15094 hello_frame("aft", offered, 9),
15095 )
15096 .unwrap();
15097
15098 assert_eq!(responses.len(), 1);
15099 assert_eq!(responses[0].header.ty, FrameType::Error);
15100 let error = parse_error(&responses[0]);
15101 assert_eq!(error["code"], "version_unsupported");
15102 assert!(registry.get_module("aft").unwrap().is_none());
15103 assert_eq!(registry.active_registration_count().unwrap(), 0);
15104 }
15105 }
15106
15107 #[test]
15108 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
15109 let registry = Arc::new(Registry::default());
15110 let forwarding = Arc::new(ForwardingTable::default());
15111 let handler =
15112 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15113 let module_connection = ConnectionId::new(301);
15114 let registration = registry
15115 .register_with_control_ops(
15116 manifest("aft-push", PROTOCOL_VERSION),
15117 PROTOCOL_VERSION,
15118 module_connection,
15119 module_baseline_control_ops(),
15120 )
15121 .unwrap();
15122 let (module_tx, _module_rx) = mpsc::channel(8);
15123 let endpoint = forwarding
15124 .register_module_connection(
15125 module_connection,
15126 "aft-push".to_string(),
15127 PROTOCOL_VERSION,
15128 manifest_concurrency(®istration.manifest),
15129 FrameSink::new(module_tx),
15130 )
15131 .unwrap();
15132
15133 let unknown = Frame::build(
15135 FrameType::Push,
15136 control_flags(),
15137 0,
15138 0,
15139 5,
15140 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
15141 )
15142 .unwrap();
15143 let out = handler.handle_status_update(endpoint, unknown).unwrap();
15144 assert!(
15145 out.is_empty(),
15146 "unknown push op must be ignored, got {out:?}"
15147 );
15148
15149 let malformed = Frame::build(
15151 FrameType::Push,
15152 control_flags(),
15153 0,
15154 0,
15155 6,
15156 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
15157 )
15158 .unwrap();
15159 let out = handler.handle_status_update(endpoint, malformed).unwrap();
15160 assert_eq!(out.len(), 1);
15161 assert_eq!(out[0].header.ty, FrameType::Error);
15162 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
15163 }
15164
15165 #[test]
15166 fn hello_rejected_when_connection_already_owns_client_routes() {
15167 let registry = Arc::new(Registry::default());
15168 let forwarding = Arc::new(ForwardingTable::default());
15169 let handler =
15170 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15171 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
15173 let client_connection = ConnectionId::new(202);
15174
15175 let responses = handler
15178 .handle_control(
15179 client_connection,
15180 hello_frame("aft-second", PROTOCOL_VERSION, 9),
15181 )
15182 .unwrap();
15183 assert_eq!(responses[0].header.ty, FrameType::Error);
15184 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
15185 assert!(registry.get_module("aft-second").unwrap().is_none());
15186 }
15187
15188 #[tokio::test]
15189 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
15190 let registry = Arc::new(Registry::default());
15191 let forwarding = Arc::new(ForwardingTable::default());
15192 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
15193 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
15194 hello_via_sink(
15195 &handler,
15196 &module_ctx,
15197 &mut module_rx,
15198 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
15199 )
15200 .await;
15201 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
15202 let pending = forwarding
15203 .begin_route_bind_relay_for_test(
15204 client_ctx.connection_id,
15205 client_ctx.egress.clone(),
15206 2,
15207 "alpha",
15208 )
15209 .unwrap();
15210 forwarding
15211 .complete_pending_relay(
15212 module_ctx.connection_id,
15213 pending.corr,
15214 RouteBindRelayOutcome::Accepted,
15215 )
15216 .unwrap();
15217 client_rx.try_recv().unwrap();
15218 for module_id in ["beta", "alpha"] {
15219 let replies = handler
15220 .handle_control_frame(
15221 &module_ctx,
15222 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
15223 )
15224 .await
15225 .unwrap();
15226 assert_eq!(replies.len(), 1, "second HELLO must be refused");
15227 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
15228 }
15229 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
15230 assert!(registry.get_module("beta").unwrap().is_none());
15231 assert!(matches!(
15232 forwarding
15233 .lookup_data_route(
15234 client_ctx.connection_id,
15235 pending.client_channel,
15236 pending.client_epoch,
15237 )
15238 .unwrap(),
15239 DataRoute::Client(DataRouteState::Bound(_))
15240 ));
15241 assert!(handler
15242 .hello_launch_nonces
15243 .lock()
15244 .unwrap()
15245 .presented(module_ctx.connection_id, Some("alpha-nonce")));
15246 assert!(module_rx.try_recv().is_err());
15247 }
15248
15249 #[test]
15250 fn reserved_module_hello_requires_matching_launch_nonce() {
15251 let registry = Arc::new(Registry::default());
15252 let supervisor = SupervisorHandle::new();
15253 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
15256 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15257
15258 let no_nonce = handler
15260 .handle_control(
15261 ConnectionId::new(1),
15262 hello_frame("vault", PROTOCOL_VERSION, 1),
15263 )
15264 .unwrap();
15265 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
15266 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
15267 assert!(registry.get_module("vault").unwrap().is_none());
15268
15269 let wrong = handler
15271 .handle_control(
15272 ConnectionId::new(2),
15273 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
15274 )
15275 .unwrap();
15276 assert_eq!(wrong[0].header.ty, FrameType::Error);
15277 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
15278 assert!(registry.get_module("vault").unwrap().is_none());
15279
15280 let ok = handler
15282 .handle_control(
15283 ConnectionId::new(3),
15284 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
15285 )
15286 .unwrap();
15287 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
15288 assert!(registry.get_module("vault").unwrap().is_some());
15289 }
15290
15291 #[test]
15292 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
15293 let registry = Arc::new(Registry::default());
15294 let supervisor = SupervisorHandle::new();
15295 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15296 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15297 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15298
15299 let squat = handler
15300 .handle_control(
15301 ConnectionId::new(1),
15302 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
15303 )
15304 .unwrap();
15305 assert_eq!(squat[0].header.ty, FrameType::Error);
15306 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
15307 assert!(parse_error(&squat[0])["message"]
15308 .as_str()
15309 .unwrap()
15310 .contains("fed:"));
15311
15312 let accepted_peer = handler
15313 .handle_control(
15314 ConnectionId::new(2),
15315 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
15316 )
15317 .unwrap();
15318 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
15319
15320 let accepted_short = handler
15321 .handle_control(
15322 ConnectionId::new(3),
15323 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
15324 )
15325 .unwrap();
15326 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
15327
15328 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
15329 let response = handler
15330 .handle_control(
15331 ConnectionId::new(conn),
15332 hello_frame(module_id, PROTOCOL_VERSION, conn),
15333 )
15334 .unwrap();
15335 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
15336 }
15337 }
15338
15339 #[test]
15340 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
15341 let registry = Arc::new(Registry::default());
15342 let supervisor = SupervisorHandle::new();
15343 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15344 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15345 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
15346 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15347
15348 let owner_nonce = handler
15349 .handle_control(
15350 ConnectionId::new(1),
15351 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
15352 )
15353 .unwrap();
15354 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
15355 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
15356 assert!(registry.get_module("fed:special").unwrap().is_none());
15357
15358 let exact_nonce = handler
15359 .handle_control(
15360 ConnectionId::new(2),
15361 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
15362 )
15363 .unwrap();
15364 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
15365 assert!(registry.get_module("fed:special").unwrap().is_some());
15366 }
15367
15368 #[test]
15369 fn non_reserved_module_ignores_launch_nonce() {
15370 let registry = Arc::new(Registry::default());
15371 let handler = ControlHandler::new(Arc::clone(®istry));
15374 let no_nonce = handler
15375 .handle_control(
15376 ConnectionId::new(1),
15377 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
15378 )
15379 .unwrap();
15380 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
15381 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
15382
15383 let echoed_nonce = handler
15384 .handle_control(
15385 ConnectionId::new(2),
15386 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
15387 )
15388 .unwrap();
15389 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
15390 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
15391 }
15392
15393 #[test]
15394 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
15395 let handler = ControlHandler::default();
15396 let conn = ConnectionId::new(1);
15397 let malformed = Frame::build(
15398 FrameType::Hello,
15399 control_flags(),
15400 0,
15401 0,
15402 3,
15403 b"{not json".to_vec(),
15404 )
15405 .unwrap();
15406
15407 let error = handler.handle_control(conn, malformed).unwrap();
15408 assert_eq!(error[0].header.ty, FrameType::Error);
15409 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
15410
15411 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15412 let pong = handler.handle_control(conn, ping).unwrap();
15413 assert_eq!(pong[0].header.ty, FrameType::Pong);
15414 assert_eq!(pong[0].header.corr, 4);
15415 }
15416
15417 #[test]
15418 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
15419 let registry = Arc::new(Registry::default());
15420 let handler = ControlHandler::new(Arc::clone(®istry));
15421
15422 handler
15423 .handle_control(
15424 ConnectionId::new(1),
15425 hello_frame("aft", PROTOCOL_VERSION, 1),
15426 )
15427 .unwrap();
15428 let duplicate = handler
15429 .handle_control(
15430 ConnectionId::new(2),
15431 hello_frame("aft", PROTOCOL_VERSION, 2),
15432 )
15433 .unwrap();
15434
15435 assert_eq!(duplicate[0].header.ty, FrameType::Error);
15436 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15437 let registration = registry.get_module("aft").unwrap().unwrap();
15438 assert_eq!(registration.connection_id, ConnectionId::new(1));
15439 }
15440
15441 #[test]
15442 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
15443 let registry = Arc::new(Registry::default());
15444 let forwarding = Arc::new(ForwardingTable::default());
15445 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
15446 let handler =
15447 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15448 .with_process_liveness(process_liveness);
15449 let (ctx, route_channel, route_epoch) =
15450 bind_liveness_route(®istry, &forwarding, "aft-dead");
15451 let responses = handler
15452 .handle_route_poll(
15453 &ctx,
15454 route_poll_frame(41, PollKind::Liveness, route_channel),
15455 route_channel,
15456 route_epoch,
15457 PollKind::Liveness,
15458 )
15459 .unwrap();
15460
15461 assert_eq!(responses.len(), 1);
15462 assert_eq!(responses[0].header.ty, FrameType::Response);
15463 assert_route_poll_liveness(&responses[0], false);
15464 }
15465
15466 #[test]
15467 fn liveness_poll_without_process_source_uses_bound_route() {
15468 let registry = Arc::new(Registry::default());
15469 let forwarding = Arc::new(ForwardingTable::default());
15470 let handler =
15471 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15472 let (ctx, route_channel, route_epoch) =
15473 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
15474 let responses = handler
15475 .handle_route_poll(
15476 &ctx,
15477 route_poll_frame(42, PollKind::Liveness, route_channel),
15478 route_channel,
15479 route_epoch,
15480 PollKind::Liveness,
15481 )
15482 .unwrap();
15483
15484 assert_route_poll_liveness(&responses[0], true);
15485 }
15486
15487 #[test]
15488 fn liveness_poll_untracked_process_source_uses_bound_route() {
15489 let registry = Arc::new(Registry::default());
15490 let forwarding = Arc::new(ForwardingTable::default());
15491 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
15492 let handler =
15493 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15494 .with_process_liveness(process_liveness);
15495 let (ctx, route_channel, route_epoch) =
15496 bind_liveness_route(®istry, &forwarding, "aft-untracked");
15497 let responses = handler
15498 .handle_route_poll(
15499 &ctx,
15500 route_poll_frame(43, PollKind::Liveness, route_channel),
15501 route_channel,
15502 route_epoch,
15503 PollKind::Liveness,
15504 )
15505 .unwrap();
15506
15507 assert_route_poll_liveness(&responses[0], true);
15508 }
15509
15510 #[tokio::test]
15511 async fn unknown_op_returns_unknown_control_op() {
15512 let handler = ControlHandler::default();
15513 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
15514 let request = Frame::build(
15515 FrameType::Request,
15516 control_flags(),
15517 0,
15518 0,
15519 55,
15520 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
15521 )
15522 .unwrap();
15523
15524 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15525
15526 assert_eq!(response.len(), 1);
15527 assert_eq!(response[0].header.ty, FrameType::Error);
15528 assert_eq!(response[0].header.corr, 55);
15529 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
15530 }
15531
15532 #[tokio::test]
15533 async fn supervisor_provenance_rejects_unknown_exact_module() {
15534 let handler = ControlHandler::default();
15535 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
15536 let request = Frame::build(
15537 FrameType::Request,
15538 control_flags(),
15539 0,
15540 0,
15541 57,
15542 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
15543 )
15544 .unwrap();
15545
15546 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15547
15548 assert_eq!(response.len(), 1);
15549 assert_eq!(response[0].header.ty, FrameType::Error);
15550 assert_eq!(response[0].header.corr, 57);
15551 let error = parse_error(&response[0]);
15552 assert_eq!(error["code"], "unknown_module");
15553 assert_eq!(error["message"], "module_id 'missing' is not supervised");
15554 }
15555
15556 #[test]
15557 fn provenance_probe_override_keeps_handler_tests_deterministic() {
15558 let expected = subc_control::RunningImageAgreement::Unavailable {
15559 reason: subc_control::RunningImageUnavailableReason::HashFailed,
15560 };
15561 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
15562 assert_eq!(handler.provenance_probe_override, Some(expected));
15563 }
15564
15565 #[test]
15566 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
15567 let verdict = reload_verdict(
15568 std::path::Path::new("/bin/new"),
15569 Some(std::path::Path::new("/bin/old")),
15570 subc_control::RunningImageAgreement::Unavailable {
15571 reason: subc_control::RunningImageUnavailableReason::HashFailed,
15572 },
15573 );
15574 assert!(matches!(
15575 verdict.path,
15576 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
15577 if configured == std::path::Path::new("/bin/new")
15578 && spawned_from == std::path::Path::new("/bin/old")
15579 ));
15580 }
15581
15582 #[test]
15583 fn launch_spec_without_running_digest_is_unavailable_not_match() {
15584 use subc_control::{
15585 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
15586 };
15587 let configured = crate::supervise::ModuleSpec {
15588 carry_containment: false,
15589 module_id: "legacy".into(),
15590 program: "/module".into(),
15591 args: Vec::new(),
15592 env: Vec::new(),
15593 reserved: false,
15594 reserved_prefixes: Vec::new(),
15595 protocol: subc_control::ModuleProtocol::Subc,
15596 overlap: Default::default(),
15597 };
15598 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
15599 assert_eq!(
15600 launch_spec_agreement(&configured, false, Some(&digest)),
15601 Agreement::Unavailable {
15602 reason: Reason::NotRunning
15603 }
15604 );
15605 assert_eq!(
15606 launch_spec_agreement(&configured, true, None),
15607 Agreement::Unavailable {
15608 reason: Reason::SpawnedSpecUnavailable
15609 }
15610 );
15611 assert_eq!(
15612 launch_spec_agreement(&configured, true, Some(&digest)),
15613 Agreement::Match
15614 );
15615 }
15616
15617 #[cfg(windows)]
15618 #[test]
15619 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
15620 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
15621 let configured = dir.join("module.exe");
15622 std::fs::write(&configured, b"executable path fixture").unwrap();
15623 let spawned = std::fs::canonicalize(&configured).unwrap();
15624 assert_ne!(
15625 configured, spawned,
15626 "fixture must exercise different path spellings"
15627 );
15628 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
15629 let lower_case = configured.to_string_lossy().to_lowercase();
15630 for spelling in [
15631 &configured,
15632 Path::new(&forward_slashes),
15633 Path::new(&lower_case),
15634 ] {
15635 let verdict = reload_verdict(
15636 spelling,
15637 Some(&spawned),
15638 subc_control::RunningImageAgreement::Unavailable {
15639 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15640 },
15641 );
15642 assert_eq!(
15643 verdict.path,
15644 ReloadPathAgreement::Match,
15645 "configured {spelling:?}, spawned {spawned:?}"
15646 );
15647 }
15648 }
15649
15650 #[cfg(windows)]
15651 #[test]
15652 fn windows_reload_path_keeps_different_existing_program_pending() {
15653 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
15654 let old = dir.join("old.exe");
15655 let new = dir.join("new.exe");
15656 std::fs::write(&old, b"same executable content").unwrap();
15657 std::fs::copy(&old, &new).unwrap();
15658 let spawned = std::fs::canonicalize(&old).unwrap();
15659 let image = subc_control::RunningImageAgreement::Unavailable {
15660 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15661 };
15662 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
15663 assert!(
15664 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
15665 if configured == new && spawned_from == spawned)
15666 );
15667 let missing = dir.join("missing.exe");
15668 assert!(matches!(
15669 reload_verdict(&missing, Some(&spawned), image).path,
15670 ReloadPathAgreement::Mismatch { .. }
15671 ));
15672 }
15673
15674 #[test]
15675 fn reload_verdict_detects_replaced_image_at_same_path() {
15676 let image = subc_control::RunningImageAgreement::Mismatch {
15677 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
15678 digest: "old".into(),
15679 },
15680 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
15681 digest: "new".into(),
15682 },
15683 };
15684 let verdict = reload_verdict(
15685 std::path::Path::new("/bin/same"),
15686 Some(std::path::Path::new("/bin/same")),
15687 image.clone(),
15688 );
15689 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15690 assert_eq!(verdict.image, image);
15691 }
15692
15693 #[test]
15694 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
15695 let image = subc_control::RunningImageAgreement::Unavailable {
15696 reason: subc_control::RunningImageUnavailableReason::NotRunning,
15697 };
15698 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
15699 assert_eq!(
15700 verdict.path,
15701 subc_control::ReloadPathAgreement::Unavailable {
15702 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
15703 }
15704 );
15705 assert_eq!(verdict.image, image);
15706
15707 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
15708 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15709 };
15710 let verdict = reload_verdict(
15711 std::path::Path::new("/bin/same"),
15712 Some(std::path::Path::new("/bin/same")),
15713 unconfirmed.clone(),
15714 );
15715 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15716 assert_eq!(verdict.image, unconfirmed);
15717 }
15718
15719 #[test]
15720 fn reload_verdict_preserves_each_image_unavailability_reason() {
15721 use subc_control::RunningImageUnavailableReason as Reason;
15722
15723 for reason in [
15724 Reason::NotRunning,
15725 Reason::UnsupportedPlatform,
15726 Reason::RunningExecutableUnreadable,
15727 Reason::SpawnedPathUnreadable,
15728 Reason::HashFailed,
15729 Reason::ProcessIdentityUnconfirmed,
15730 Reason::Unknown("future_probe_reason".to_string()),
15731 ] {
15732 let image = subc_control::RunningImageAgreement::Unavailable {
15733 reason: reason.clone(),
15734 };
15735 let verdict = reload_verdict(
15736 std::path::Path::new("/bin/same"),
15737 Some(std::path::Path::new("/bin/same")),
15738 image.clone(),
15739 );
15740 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15741 assert_eq!(verdict.image, image, "{reason:?}");
15742 }
15743 }
15744
15745 #[tokio::test]
15746 async fn malformed_control_bodies_return_invalid_control_body() {
15747 let handler = ControlHandler::default();
15748 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
15749
15750 for (corr, body) in [
15751 (56, br#"{"route_channel":1}"#.as_slice()),
15752 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
15753 (
15754 58,
15755 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
15756 ),
15757 ] {
15758 let request = Frame::build(
15759 FrameType::Request,
15760 control_flags(),
15761 0,
15762 0,
15763 corr,
15764 body.to_vec(),
15765 )
15766 .unwrap();
15767 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15768
15769 assert_eq!(response.len(), 1);
15770 assert_eq!(response[0].header.ty, FrameType::Error);
15771 assert_eq!(response[0].header.corr, corr);
15772 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
15773 }
15774 }
15775
15776 #[tokio::test]
15777 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
15778 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15779 let registry = Arc::new(Registry::default());
15780 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
15781 let router = Router::with_control_handler(Arc::clone(&control));
15782 let connection = router.begin_connection();
15783 let (ctx, mut rx) = route_ctx(connection.id());
15784
15785 router
15786 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
15787 .await
15788 .unwrap();
15789 let response = rx.recv().await.unwrap();
15790 let ack = parse_ack(&response);
15791 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15792 let channel = 1;
15793
15794 let goodbye =
15795 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
15796 router.route_for_connection(&ctx, goodbye).await.unwrap();
15797 assert!(rx.try_recv().is_err());
15798 assert!(registry.get_module("aft").unwrap().is_none());
15799
15800 router
15801 .route_for_connection(&ctx, channel_request(channel, 13))
15802 .await
15803 .unwrap();
15804 let error_frame = rx.recv().await.unwrap();
15805 assert_eq!(error_frame.header.ty, FrameType::Error);
15806 assert_eq!(error_frame.header.channel, channel);
15807 let captured = crate::router::test_log::captured_logs(&logs);
15808 assert_eq!(
15809 captured
15810 .lines()
15811 .filter(|line| {
15812 line.contains("module_id=aft")
15813 && line.contains("reason=explicit_goodbye")
15814 && line.contains("module registration ended")
15815 })
15816 .count(),
15817 1,
15818 "unexpected GOODBYE registry log: {captured}"
15819 );
15820 }
15821
15822 #[tokio::test]
15823 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
15824 let registry = Arc::new(Registry::default());
15825 let handler = ControlHandler::new(registry).with_capability_config(
15826 [("prov".to_string(), true), ("cons".to_string(), true)],
15827 BTreeMap::new(),
15828 );
15829 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
15830 register_capability_manifest(
15831 &handler,
15832 &provider_ctx,
15833 &mut provider_rx,
15834 capability_manifest("prov", &["thing/v1"], &[]),
15835 1,
15836 )
15837 .await;
15838 let mut consumer = capability_manifest("cons", &[], &[]);
15839 consumer.capabilities.as_mut().unwrap().requires.push(
15840 subc_protocol::manifest::CapabilityRequirement {
15841 capability: "thing/v1".to_string(),
15842 need: subc_protocol::manifest::CapabilityNeed::Required,
15843 },
15844 );
15845 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
15846 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
15847 assert_eq!(
15848 handler.capability_evaluator.verdict("cons", "thing/v1"),
15849 Some(CapabilityVerdict::Provided)
15850 );
15851 let (mut client_rx, _) = open_route_for_capability_test(
15852 &handler,
15853 &provider_ctx,
15854 &mut provider_rx,
15855 703,
15856 3,
15857 "prov",
15858 None,
15859 )
15860 .await;
15861 let goodbye =
15862 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15863 handler
15864 .handle_control_frame(&provider_ctx, goodbye)
15865 .await
15866 .unwrap();
15867 assert_eq!(
15868 handler.capability_evaluator.verdict("cons", "thing/v1"),
15869 Some(CapabilityVerdict::NeverProvided)
15870 );
15871 let closed = client_rx
15872 .try_recv()
15873 .expect("GOODBYE pushes route.closed before route GOODBYE");
15874 assert!(
15875 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
15876 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
15877 );
15878 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
15879 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
15880 }
15881
15882 struct RouteClosedFixture {
15883 handler: ControlHandler,
15884 #[cfg(unix)]
15885 supervisor: Supervisor,
15886 module_ctx: RouteCtx,
15887 #[cfg(unix)]
15888 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15889 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15890 route: Arc<crate::forwarding::RouteBinding>,
15891 }
15892
15893 async fn route_closed_fixture() -> RouteClosedFixture {
15894 let registry = Arc::new(Registry::default());
15895 let handler = ControlHandler::new(Arc::clone(®istry));
15896 #[cfg(unix)]
15897 let supervisor = Supervisor::new(registry, RestartPolicy::default())
15898 .with_forwarding(Arc::clone(&handler.forwarding));
15899 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
15900 register_capability_manifest(
15901 &handler,
15902 &module_ctx,
15903 &mut module_rx,
15904 capability_manifest("prov", &[], &[]),
15905 1,
15906 )
15907 .await;
15908 let (client_rx, bind) = open_route_for_capability_test(
15909 &handler,
15910 &module_ctx,
15911 &mut module_rx,
15912 702,
15913 2,
15914 "prov",
15915 None,
15916 )
15917 .await;
15918 let ModuleControlRequest::RouteBind {
15919 route_channel: channel,
15920 epoch: route_epoch,
15921 ..
15922 } = bind
15923 else {
15924 panic!("expected route.bind");
15925 };
15926 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
15927 handler
15928 .forwarding
15929 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
15930 .unwrap()
15931 else {
15932 panic!("the acknowledged route must be bound");
15933 };
15934 RouteClosedFixture {
15935 handler,
15936 #[cfg(unix)]
15937 supervisor,
15938 module_ctx,
15939 #[cfg(unix)]
15940 module_rx,
15941 client_rx,
15942 route,
15943 }
15944 }
15945
15946 impl RouteClosedFixture {
15947 #[cfg(unix)]
15948 async fn cut(&mut self) {
15949 let notices = async {
15950 let mut draining = self.module_rx.recv().await.unwrap();
15951 assert!(matches!(
15952 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
15953 &draining.body
15954 )
15955 .unwrap(),
15956 subc_protocol::session::ModuleControlCommand::Draining {
15957 reason: RouteCloseReason::Disable,
15958 ..
15959 }
15960 ));
15961 draining.flushed.take().unwrap().send(()).unwrap();
15962 let mut closing = self.client_rx.recv().await.unwrap();
15963 assert!(matches!(
15964 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
15965 ClientControlPush::RouteClosing {
15966 reason: RouteCloseReason::Restart,
15967 ..
15968 }
15969 ));
15970 closing.flushed.take().unwrap().send(()).unwrap();
15971 };
15972 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
15973 result.unwrap();
15974 }
15975
15976 fn disconnect(&mut self) -> ClientControlPush {
15977 self.handler
15978 .cleanup_connection(self.module_ctx.connection_id)
15979 .unwrap();
15980 let closed = self.client_rx.try_recv().unwrap();
15981 assert_eq!(closed.header.ty, FrameType::Push);
15982 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
15983 assert!(matches!(
15984 &push,
15985 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
15986 if channels == &[self.route.client_channel]
15987 ));
15988 assert_eq!(
15989 self.client_rx.try_recv().unwrap().header.ty,
15990 FrameType::Goodbye
15991 );
15992 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
15993 push
15994 }
15995 }
15996
15997 #[cfg(unix)]
15998 #[tokio::test(start_paused = true)]
15999 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
16000 let mut fixture = route_closed_fixture().await;
16001 fixture.cut().await;
16002 assert!(matches!(
16003 fixture.disconnect(),
16004 ClientControlPush::RouteClosed {
16005 reason: RouteCloseReason::Restart,
16006 drained: true,
16007 ..
16008 }
16009 ));
16010 }
16011
16012 #[cfg(unix)]
16013 #[tokio::test(start_paused = true)]
16014 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
16015 let mut fixture = route_closed_fixture().await;
16016 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
16017 let started = tokio::time::Instant::now();
16018 fixture.cut().await;
16019 assert!(started.elapsed() >= Duration::from_secs(2));
16020 assert_eq!(fixture.route.flow.in_flight(), 1);
16021 assert!(matches!(
16022 fixture.disconnect(),
16023 ClientControlPush::RouteClosed {
16024 reason: RouteCloseReason::Restart,
16025 drained: false,
16026 ..
16027 }
16028 ));
16029 }
16030
16031 #[tokio::test]
16032 async fn module_connection_crash_reports_crash_and_not_drained() {
16033 let mut fixture = route_closed_fixture().await;
16034 assert!(matches!(
16038 fixture.disconnect(),
16039 ClientControlPush::RouteClosed {
16040 reason: RouteCloseReason::Crash,
16041 drained: false,
16042 ..
16043 }
16044 ));
16045 }
16046
16047 #[tokio::test]
16048 async fn dropping_router_connection_releases_registration() {
16049 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16050 let registry = Arc::new(Registry::default());
16051 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
16052 let router = Router::with_control_handler(Arc::clone(&control));
16053 let connection = router.begin_connection();
16054 let connection_id = connection.id();
16055 let (ctx, mut rx) = route_ctx(connection_id);
16056
16057 router
16058 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
16059 .await
16060 .unwrap();
16061 let response = rx.recv().await.unwrap();
16062 let ack = parse_ack(&response);
16063 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16064 assert!(registry.get_module("aft").unwrap().is_some());
16065
16066 drop(connection);
16067
16068 assert!(registry.get_module("aft").unwrap().is_none());
16069 assert_eq!(registry.active_registration_count().unwrap(), 0);
16070
16071 control
16072 .cleanup_connection(ConnectionId::new(u64::MAX))
16073 .unwrap();
16074 let captured = crate::router::test_log::captured_logs(&logs);
16075 println!("captured registry lifecycle logs:\n{captured}");
16076 let events: Vec<_> = captured
16077 .lines()
16078 .filter(|line| {
16079 line.contains("module registered module_id=aft ")
16080 || line.contains("module registration ended")
16081 })
16082 .collect();
16083 assert_eq!(
16084 events.len(),
16085 2,
16086 "unexpected registry lifecycle logs: {captured}"
16087 );
16088 assert!(events[0].contains("module registered module_id=aft "));
16089 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
16090 assert!(events[1].contains(&format!(
16091 "module_id=aft connection_id={}",
16092 connection_id.get()
16093 )));
16094 assert!(events[1].contains("reason=connection_closed"));
16095 assert!(events[1].contains("module registration ended"));
16096 assert_eq!(
16097 captured
16098 .lines()
16099 .filter(|line| line.contains("module registered module_id=aft "))
16100 .count(),
16101 1,
16102 "legacy registration admission line must appear once: {captured}"
16103 );
16104 }
16105
16106 #[tokio::test]
16107 async fn hello_registration_keeps_legacy_module_registered_line_once() {
16108 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16109 let registry = Arc::new(Registry::default());
16110 let control = ControlHandler::new(Arc::clone(®istry));
16111 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
16112 hello_via_sink(
16113 &control,
16114 &ctx,
16115 &mut rx,
16116 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
16117 )
16118 .await;
16119
16120 let captured = crate::router::test_log::captured_logs(&logs);
16121 let admissions: Vec<_> = captured
16122 .lines()
16123 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
16124 .collect();
16125 assert_eq!(
16126 admissions.len(),
16127 1,
16128 "legacy admission line must remain exactly once: {captured}"
16129 );
16130 assert!(admissions[0].contains("routable_provider=true"));
16131 assert!(admissions[0].contains("connection_id=777"));
16132 }
16133
16134 fn capability_manifest(
16135 module_id: &str,
16136 provides: &[&str],
16137 must_never_reach: &[&str],
16138 ) -> ModuleManifest {
16139 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
16140 manifest.capabilities = Some(CapabilityDeclarations {
16141 provides: provides
16142 .iter()
16143 .map(|capability| (*capability).to_string())
16144 .collect(),
16145 requires: Vec::new(),
16146 must_never_reach: must_never_reach
16147 .iter()
16148 .map(|capability| (*capability).to_string())
16149 .collect(),
16150 });
16151 manifest
16152 }
16153
16154 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
16155 Frame::build(
16156 FrameType::Hello,
16157 control_flags(),
16158 0,
16159 0,
16160 corr,
16161 serde_json::to_vec(&ModuleHelloBody {
16162 protocol_ver: manifest.protocol_ver,
16163 manifest,
16164 control_ops: None,
16165 launch_nonce: None,
16166 })
16167 .expect("capability test HELLO serializes"),
16168 )
16169 .expect("capability test HELLO frame builds")
16170 }
16171
16172 fn catalog_update_with_capabilities_frame(
16173 corr: u64,
16174 capabilities: CapabilityDeclarations,
16175 ) -> Frame {
16176 Frame::build(
16177 FrameType::Request,
16178 control_flags(),
16179 0,
16180 0,
16181 corr,
16182 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16183 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
16184 capabilities: Some(capabilities),
16185 ready: None,
16186 })
16187 .expect("capability catalog.update serializes"),
16188 )
16189 .expect("capability catalog.update frame builds")
16190 }
16191
16192 async fn register_capability_manifest(
16193 handler: &ControlHandler,
16194 ctx: &RouteCtx,
16195 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16196 manifest: ModuleManifest,
16197 corr: u64,
16198 ) {
16199 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
16200 }
16201
16202 async fn open_route_for_capability_test(
16203 handler: &ControlHandler,
16204 target_ctx: &RouteCtx,
16205 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16206 client_connection_id: u64,
16207 corr: u64,
16208 target_module_id: &str,
16209 consumer_identity: Option<ConsumerIdentity>,
16210 ) -> (
16211 mpsc::Receiver<crate::router::OutboundFrame>,
16212 ModuleControlRequest,
16213 ) {
16214 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
16215 let route_handler = handler.clone();
16216 let target_module_id = target_module_id.to_string();
16217 let route_task = tokio::spawn(async move {
16218 route_handler
16219 .handle_control_frame(
16220 &client_ctx,
16221 route_open_frame_with_admission_facts(
16222 corr,
16223 &target_module_id,
16224 unique_project_root("admission-facts"),
16225 consumer_identity,
16226 None,
16227 ),
16228 )
16229 .await
16230 .expect("capability test route.open succeeds")
16231 });
16232 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
16233 .await
16234 .expect("capability test route.open must reach route.bind")
16235 .expect("target control receiver stays open");
16236 let bind_request: ModuleControlRequest =
16237 serde_json::from_slice(&bind.body).expect("route.bind decodes");
16238 handler
16239 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
16240 .await
16241 .expect("capability test route.bind ACK succeeds");
16242 assert!(route_task.await.expect("route.open task joins").is_empty());
16243 let opened = client_rx
16244 .recv()
16245 .await
16246 .expect("successful route.open publishes a response");
16247 assert!(matches!(
16248 serde_json::from_slice::<ClientControlResponse>(&opened.body),
16249 Ok(ClientControlResponse::RouteOpen { .. })
16250 ));
16251 (client_rx, bind_request)
16252 }
16253
16254 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
16255 assert_eq!(frame.header.ty, FrameType::Push);
16256 assert_eq!(frame.header.channel, 0);
16257 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
16258 .expect("route.closed control push decodes");
16259 let ClientControlPush::RouteClosed { channels, .. } = &push else {
16260 panic!("expected route.closed");
16261 };
16262 assert_eq!(channels.len(), 1, "exactly one violating route closed");
16263 let closed_channel = channels[0];
16264 let channels = channels.clone();
16265 assert_eq!(
16266 push,
16267 ClientControlPush::RouteClosed {
16268 module_id: target_module_id.to_string(),
16269 channels,
16270 reason: RouteCloseReason::CapabilityDenied,
16271 drained: false,
16272 abandoned: 0,
16273 excluded_subscriptions: 0,
16274 terminal: Some(false),
16275 }
16276 );
16277 closed_channel
16278 }
16279
16280 #[tokio::test]
16281 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
16282 let registry = Arc::new(Registry::default());
16283 let forwarding = Arc::new(ForwardingTable::default());
16284 let supervisor = SupervisorHandle::new();
16285 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16286 let handler =
16287 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16288 .with_supervisor(supervisor);
16289 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
16290 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
16291 register_capability_manifest(
16292 &handler,
16293 &target_ctx,
16294 &mut target_rx,
16295 capability_manifest("target", &["credentials-provider/v1"], &[]),
16296 1,
16297 )
16298 .await;
16299 register_capability_manifest(
16300 &handler,
16301 &opener_ctx,
16302 &mut opener_rx,
16303 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16304 2,
16305 )
16306 .await;
16307
16308 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
16309 let replies = handler
16310 .handle_control_frame(
16311 &client_ctx,
16312 route_open_frame_with_admission_facts(
16313 3,
16314 "target",
16315 unique_project_root("admission-facts"),
16316 Some(ConsumerIdentity {
16317 module_id: "opener".to_string(),
16318 launch_nonce: "opener-nonce".to_string(),
16319 }),
16320 None,
16321 ),
16322 )
16323 .await
16324 .expect("denied route.open returns a typed frame");
16325 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16326 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16327 assert!(
16328 target_rx.try_recv().is_err(),
16329 "forbidden route.open must not relay route.bind"
16330 );
16331 }
16332
16333 #[tokio::test]
16334 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
16335 let registry = Arc::new(Registry::default());
16336 let forwarding = Arc::new(ForwardingTable::default());
16337 let supervisor = SupervisorHandle::new();
16338 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16339 let handler =
16340 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16341 .with_supervisor(supervisor);
16342 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
16343 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
16344 register_capability_manifest(
16345 &handler,
16346 &target_ctx,
16347 &mut target_rx,
16348 capability_manifest("target", &["credentials-provider/v1"], &[]),
16349 1,
16350 )
16351 .await;
16352 register_capability_manifest(
16353 &handler,
16354 &old_opener_ctx,
16355 &mut old_opener_rx,
16356 capability_manifest("opener", &[], &[]),
16357 2,
16358 )
16359 .await;
16360 let (mut client_rx, _) = open_route_for_capability_test(
16361 &handler,
16362 &target_ctx,
16363 &mut target_rx,
16364 712,
16365 3,
16366 "target",
16367 Some(ConsumerIdentity {
16368 module_id: "opener".to_string(),
16369 launch_nonce: "opener-nonce".to_string(),
16370 }),
16371 )
16372 .await;
16373 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16374
16375 handler
16376 .cleanup_connection(old_opener_ctx.connection_id)
16377 .expect("old opener registration cleans up");
16378 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
16379 register_capability_manifest(
16380 &handler,
16381 &new_opener_ctx,
16382 &mut new_opener_rx,
16383 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16384 4,
16385 )
16386 .await;
16387
16388 let closed_channel = assert_capability_denied_push(
16389 client_rx
16390 .try_recv()
16391 .expect("HELLO deny addition must emit route.closed")
16392 .frame,
16393 "target",
16394 );
16395 let goodbye = client_rx
16396 .try_recv()
16397 .expect("HELLO deny addition must send the client a route GOODBYE")
16398 .frame;
16399 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16400 assert_eq!(goodbye.header.channel, closed_channel);
16401 assert_ne!(goodbye.header.channel, 0);
16402 assert_ne!(goodbye.header.epoch, 0);
16403 assert!(client_rx.try_recv().is_err());
16404 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16405 assert!(matches!(
16406 target_rx.try_recv(),
16407 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16408 ));
16409 }
16410
16411 #[tokio::test]
16412 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
16413 let registry = Arc::new(Registry::default());
16414 let forwarding = Arc::new(ForwardingTable::default());
16415 let supervisor = SupervisorHandle::new();
16416 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16417 let handler =
16418 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16419 .with_supervisor(supervisor);
16420 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
16421 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
16422 register_capability_manifest(
16423 &handler,
16424 &target_ctx,
16425 &mut target_rx,
16426 capability_manifest("target", &[], &[]),
16427 1,
16428 )
16429 .await;
16430 register_capability_manifest(
16431 &handler,
16432 &opener_ctx,
16433 &mut opener_rx,
16434 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16435 2,
16436 )
16437 .await;
16438 let (mut client_rx, _) = open_route_for_capability_test(
16439 &handler,
16440 &target_ctx,
16441 &mut target_rx,
16442 722,
16443 3,
16444 "target",
16445 Some(ConsumerIdentity {
16446 module_id: "opener".to_string(),
16447 launch_nonce: "opener-nonce".to_string(),
16448 }),
16449 )
16450 .await;
16451 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16452
16453 let replies = handler
16454 .handle_control_frame(
16455 &target_ctx,
16456 catalog_update_with_capabilities_frame(
16457 4,
16458 CapabilityDeclarations {
16459 provides: vec!["credentials-provider/v1".to_string()],
16460 requires: Vec::new(),
16461 must_never_reach: Vec::new(),
16462 },
16463 ),
16464 )
16465 .await
16466 .expect("claim catalog.update succeeds");
16467 assert!(matches!(
16468 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
16469 Ok(ModuleControlResponseToModule::CatalogUpdate {})
16470 ));
16471 let closed_channel = assert_capability_denied_push(
16472 client_rx
16473 .try_recv()
16474 .expect("claim addition must emit route.closed")
16475 .frame,
16476 "target",
16477 );
16478 let goodbye = client_rx
16479 .try_recv()
16480 .expect("claim addition must send the client a route GOODBYE")
16481 .frame;
16482 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16483 assert_eq!(goodbye.header.channel, closed_channel);
16484 assert_ne!(goodbye.header.channel, 0);
16485 assert_ne!(goodbye.header.epoch, 0);
16486 assert!(client_rx.try_recv().is_err());
16487 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16488 assert!(matches!(
16489 target_rx.try_recv(),
16490 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16491 ));
16492 }
16493
16494 #[tokio::test]
16495 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
16496 let registry = Arc::new(Registry::default());
16497 let forwarding = Arc::new(ForwardingTable::default());
16498 let supervisor = SupervisorHandle::new();
16499 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16500 let handler =
16501 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16502 .with_supervisor(supervisor);
16503 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
16504 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
16505 register_capability_manifest(
16506 &handler,
16507 &target_ctx,
16508 &mut target_rx,
16509 capability_manifest("target", &["credentials-provider/v1"], &[]),
16510 1,
16511 )
16512 .await;
16513 register_capability_manifest(
16514 &handler,
16515 &opener_ctx,
16516 &mut opener_rx,
16517 capability_manifest("opener", &[], &[]),
16518 2,
16519 )
16520 .await;
16521 let (mut client_rx, _) = open_route_for_capability_test(
16522 &handler,
16523 &target_ctx,
16524 &mut target_rx,
16525 732,
16526 3,
16527 "target",
16528 Some(ConsumerIdentity {
16529 module_id: "opener".to_string(),
16530 launch_nonce: "opener-nonce".to_string(),
16531 }),
16532 )
16533 .await;
16534 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16535
16536 handler
16537 .handle_control_frame(
16538 &target_ctx,
16539 catalog_update_with_capabilities_frame(
16540 4,
16541 CapabilityDeclarations {
16542 provides: Vec::new(),
16543 requires: Vec::new(),
16544 must_never_reach: Vec::new(),
16545 },
16546 ),
16547 )
16548 .await
16549 .expect("claim removal catalog.update succeeds");
16550 assert_eq!(
16551 forwarding.active_binding_count().unwrap(),
16552 1,
16553 "removing an attested target claim must leave the route census unchanged"
16554 );
16555 assert!(
16556 client_rx.try_recv().is_err(),
16557 "claim removal must not emit route.closed or a client route GOODBYE"
16558 );
16559 assert!(
16560 target_rx.try_recv().is_err(),
16561 "claim removal must not send the target a route GOODBYE"
16562 );
16563 }
16564
16565 #[tokio::test]
16568 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
16569 let registry = Arc::new(Registry::default());
16570 let forwarding = Arc::new(ForwardingTable::default());
16571 let supervisor = SupervisorHandle::new();
16572 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16573 let handler =
16574 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16575 .with_supervisor(supervisor);
16576 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
16577 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
16578 register_capability_manifest(
16579 &handler,
16580 &target_ctx,
16581 &mut target_rx,
16582 capability_manifest("target", &["credentials-provider/v1"], &[]),
16583 1,
16584 )
16585 .await;
16586 register_capability_manifest(
16587 &handler,
16588 &opener_ctx,
16589 &mut opener_rx,
16590 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16591 2,
16592 )
16593 .await;
16594
16595 let (_client_rx, bind) = open_route_for_capability_test(
16596 &handler,
16597 &target_ctx,
16598 &mut target_rx,
16599 742,
16600 3,
16601 "target",
16602 None,
16603 )
16604 .await;
16605 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
16606 panic!("direct scope-honesty route must bind");
16607 };
16608 assert_eq!(principal, Some(Principal::Direct));
16609 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16610 }
16611
16612 #[tokio::test]
16615 async fn must_never_reach_self_route_is_capability_forbidden() {
16616 let registry = Arc::new(Registry::default());
16617 let forwarding = Arc::new(ForwardingTable::default());
16618 let supervisor = SupervisorHandle::new();
16619 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
16620 let handler =
16621 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16622 .with_supervisor(supervisor);
16623 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
16624 register_capability_manifest(
16625 &handler,
16626 &self_ctx,
16627 &mut self_rx,
16628 capability_manifest(
16629 "self-provider",
16630 &["credentials-provider/v1"],
16631 &["credentials-provider/v1"],
16632 ),
16633 1,
16634 )
16635 .await;
16636
16637 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
16638 let replies = handler
16639 .handle_control_frame(
16640 &client_ctx,
16641 route_open_frame_with_admission_facts(
16642 2,
16643 "self-provider",
16644 unique_project_root("admission-facts"),
16645 Some(ConsumerIdentity {
16646 module_id: "self-provider".to_string(),
16647 launch_nonce: "self-nonce".to_string(),
16648 }),
16649 None,
16650 ),
16651 )
16652 .await
16653 .expect("self-route refusal returns a typed frame");
16654 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16655 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16656 assert!(
16657 self_rx.try_recv().is_err(),
16658 "self denial must not relay route.bind"
16659 );
16660 }
16661
16662 #[test]
16663 fn unsupported_channel_zero_frame_returns_error() {
16664 let handler = ControlHandler::default();
16665 let request = Frame::build(
16666 FrameType::Request,
16667 control_flags(),
16668 0,
16669 0,
16670 21,
16671 b"opaque".to_vec(),
16672 )
16673 .unwrap();
16674
16675 let response = handler
16676 .handle_control(ConnectionId::new(1), request)
16677 .unwrap();
16678
16679 assert_eq!(response[0].header.ty, FrameType::Error);
16680 assert_eq!(
16681 parse_error(&response[0])["code"],
16682 "unsupported_control_frame"
16683 );
16684 }
16685
16686 mod swap {
16691 use super::*;
16692
16693 const INCUMBENT: ConnectionId = ConnectionId::new(30);
16694 const CANDIDATE: ConnectionId = ConnectionId::new(40);
16695
16696 struct Swap {
16697 registry: Arc<Registry>,
16698 forwarding: Arc<ForwardingTable>,
16699 handler: ControlHandler,
16700 incumbent_ctx: RouteCtx,
16701 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16702 candidate_ctx: RouteCtx,
16703 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16704 }
16705
16706 async fn swap_with_incumbent() -> Swap {
16707 let registry = Arc::new(Registry::default());
16708 let forwarding = Arc::new(ForwardingTable::default());
16709 let handler =
16710 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
16711 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
16712 hello_via_sink(
16713 &handler,
16714 &incumbent_ctx,
16715 &mut incumbent_rx,
16716 hello_frame("aft", PROTOCOL_VERSION, 7),
16717 )
16718 .await;
16719 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
16720 Swap {
16721 registry,
16722 forwarding,
16723 handler,
16724 incumbent_ctx,
16725 incumbent_rx,
16726 candidate_ctx,
16727 candidate_rx,
16728 }
16729 }
16730
16731 fn register_candidate(swap: &Swap, ready: Option<bool>) {
16732 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
16733 candidate_manifest.ready = ready;
16734 let registration = swap
16735 .registry
16736 .register_candidate_with_control_ops(
16737 candidate_manifest,
16738 PROTOCOL_VERSION,
16739 CANDIDATE,
16740 module_baseline_control_ops(),
16741 )
16742 .unwrap();
16743 swap.forwarding
16744 .register_candidate_module_connection(
16745 CANDIDATE,
16746 "aft".to_string(),
16747 PROTOCOL_VERSION,
16748 manifest_concurrency(®istration.manifest),
16749 swap.candidate_ctx.egress.clone(),
16750 )
16751 .unwrap();
16752 }
16753
16754 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
16755 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
16756 swap.registry.promote_candidate("aft").unwrap().unwrap();
16757 cutover.incumbent.unwrap()
16758 }
16759
16760 #[tokio::test]
16769 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
16770 let mut swap = swap_with_incumbent().await;
16771 let hold = Arc::new(std::sync::Barrier::new(2));
16772 let handler = swap
16773 .handler
16774 .clone()
16775 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
16776 handler
16777 .supervisor
16778 .open_swap("aft", "candidate-nonce".to_string());
16779 let hello = {
16780 let handler = handler.clone();
16781 let candidate_ctx = swap.candidate_ctx.clone();
16782 std::thread::spawn(move || {
16783 handler
16784 .handle_hello(
16785 candidate_ctx.connection_id,
16786 Some(candidate_ctx.egress.clone()),
16787 hello_frame_with_nonce(
16788 "aft",
16789 PROTOCOL_VERSION,
16790 8,
16791 Some("candidate-nonce"),
16792 ),
16793 )
16794 .unwrap()
16795 })
16796 };
16797
16798 hold.wait();
16799 assert!(swap
16802 .registry
16803 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
16804 .unwrap()
16805 .is_some());
16806 assert!(swap
16807 .registry
16808 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16809 .unwrap()
16810 .is_none());
16811 assert!(swap
16812 .forwarding
16813 .module_endpoint_for_connection(CANDIDATE)
16814 .unwrap()
16815 .is_none());
16816
16817 hold.wait();
16818 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
16819 assert_eq!(
16820 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
16821 FrameType::HelloAck
16822 );
16823 let published = swap
16824 .registry
16825 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16826 .unwrap()
16827 .expect("candidate is published once its HELLO completes");
16828 assert_eq!(published.connection_id, CANDIDATE);
16829 assert!(swap
16830 .forwarding
16831 .module_endpoint_for_connection(CANDIDATE)
16832 .unwrap()
16833 .is_some());
16834 }
16835
16836 fn keyed_total(counters: &Value, key: &str) -> u64 {
16837 counters[key]
16838 .as_object()
16839 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
16840 .unwrap_or(0)
16841 }
16842
16843 #[tokio::test]
16844 async fn replacement_logs_old_end_and_new_admission_once() {
16845 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16846 let mut swap = swap_with_incumbent().await;
16847 swap.handler
16848 .supervisor
16849 .open_swap("aft", "candidate-nonce".to_string());
16850 hello_via_sink(
16851 &swap.handler,
16852 &swap.candidate_ctx,
16853 &mut swap.candidate_rx,
16854 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
16855 )
16856 .await;
16857 cutover(&swap);
16858 swap.handler.cleanup_connection(INCUMBENT).unwrap();
16859
16860 let captured = crate::router::test_log::captured_logs(&logs);
16861 println!("captured replacement registry logs:\n{captured}");
16862 let new_admissions: Vec<_> = captured
16863 .lines()
16864 .filter(|line| {
16865 line.contains("connection_id=40")
16866 && line.contains("swap candidate registered; not routable until cutover")
16867 })
16868 .collect();
16869 assert_eq!(
16870 new_admissions.len(),
16871 1,
16872 "unexpected admission logs: {captured}"
16873 );
16874 assert!(new_admissions[0].contains("module_id=aft"));
16875 assert!(new_admissions[0].contains("connection_id=40"));
16876 assert!(
16877 new_admissions[0].contains("swap candidate registered; not routable until cutover")
16878 );
16879 assert!(new_admissions[0].contains("ready=true"));
16880
16881 let old_admissions: Vec<_> = captured
16882 .lines()
16883 .filter(|line| {
16884 line.contains("module registered module_id=aft ")
16885 && line.contains("connection_id=30")
16886 })
16887 .collect();
16888 assert_eq!(
16889 old_admissions.len(),
16890 1,
16891 "unexpected incumbent admission: {captured}"
16892 );
16893
16894 let old_ends: Vec<_> = captured
16895 .lines()
16896 .filter(|line| {
16897 line.contains("connection_id=30") && line.contains("module registration ended")
16898 })
16899 .collect();
16900 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
16901 assert!(old_ends[0].contains("module_id=aft"));
16902 assert!(old_ends[0].contains("connection_id=30"));
16903 assert!(old_ends[0].contains("reason=replaced"));
16904 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
16905 assert_eq!(
16906 captured
16907 .lines()
16908 .filter(|line| line.contains("module registration promoted"))
16909 .count(),
16910 1,
16911 "unexpected promotion logs: {captured}"
16912 );
16913 }
16914
16915 #[tokio::test]
16921 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
16922 let mut swap = swap_with_incumbent().await;
16923 let handler = swap.handler.clone();
16924
16925 let cotenant = ConnectionId::new(31);
16927 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
16928 let (cotenant_task, cotenant_bind) = relay_route_open(
16929 &handler,
16930 cotenant,
16931 &cotenant_ctx.egress,
16932 &mut swap.incumbent_rx,
16933 100,
16934 "aft",
16935 "swap-cotenant",
16936 )
16937 .await;
16938 handler
16939 .handle_control_frame(
16940 &swap.incumbent_ctx,
16941 route_bind_ack(cotenant_bind.header.corr),
16942 )
16943 .await
16944 .unwrap();
16945 assert!(cotenant_task.await.unwrap().is_empty());
16946 let (cotenant_channel, cotenant_epoch) =
16947 published_route(&cotenant_rx.recv().await.unwrap());
16948
16949 let caller = ConnectionId::new(32);
16951 let (caller_ctx, mut caller_rx) = route_ctx(caller);
16952 let (caller_task, caller_bind) = relay_route_open(
16953 &handler,
16954 caller,
16955 &caller_ctx.egress,
16956 &mut swap.incumbent_rx,
16957 101,
16958 "aft",
16959 "swap-caller",
16960 )
16961 .await;
16962 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
16963
16964 register_candidate(&swap, None);
16965 cutover(&swap);
16966
16967 let ack = handler
16969 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
16970 .await;
16971 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
16972 if module_loop_error.is_some() {
16973 handler.cleanup_connection(INCUMBENT).unwrap();
16976 }
16977
16978 assert!(
16980 cotenant_rx.try_recv().is_err(),
16981 "the co-tenant route on the incumbent was torn down by one late ack: \
16982 {module_loop_error:?}"
16983 );
16984 assert!(matches!(
16985 swap.forwarding
16986 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
16987 .unwrap(),
16988 DataRoute::Client(DataRouteState::Bound(_))
16989 ));
16990 assert_eq!(module_loop_error, None);
16991 assert!(swap
16992 .registry
16993 .get_module_by_connection(INCUMBENT)
16994 .unwrap()
16995 .is_some());
16996
16997 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
16999 .await
17000 .expect("the incumbent is told to drop the abandoned binding")
17001 .unwrap()
17002 .frame;
17003 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17004 assert_eq!(goodbye.header.channel, abandoned_channel);
17005 assert_eq!(goodbye.header.epoch, abandoned_epoch);
17006 assert!(swap.incumbent_rx.try_recv().is_err());
17007
17008 let response = caller_task.await.unwrap();
17010 assert_eq!(response.len(), 1);
17011 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17012 assert!(caller_rx.try_recv().is_err());
17013
17014 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
17018 let counters = handler.counters().snapshot();
17019 assert_eq!(
17020 keyed_total(&counters, "route_open_accepted_by_principal"),
17021 1
17022 );
17023 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
17024 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
17025 }
17026
17027 #[tokio::test]
17031 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
17032 let mut swap = swap_with_incumbent().await;
17033 register_candidate(&swap, None);
17034 let incumbent = cutover(&swap);
17035 swap.forwarding
17036 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
17037 .unwrap()
17038 .expect("the incumbent is still registered");
17039
17040 let client = ConnectionId::new(33);
17041 let (client_ctx, mut client_rx) = route_ctx(client);
17042 let route_handler = swap.handler.clone();
17043 let open_ctx = RouteCtx {
17044 connection_id: client,
17045 egress: client_ctx.egress.clone(),
17046 };
17047 let mut route_task = tokio::spawn(async move {
17048 route_handler
17049 .handle_control_frame(
17050 &open_ctx,
17051 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
17052 )
17053 .await
17054 .unwrap()
17055 });
17056 let bind = tokio::select! {
17057 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
17058 response = &mut route_task => {
17059 let response = response.unwrap();
17060 panic!(
17061 "post-cutover route.open was refused instead of relayed to the candidate: {}",
17062 parse_error(&response[0])["code"]
17063 );
17064 }
17065 };
17066 swap.handler
17067 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
17068 .await
17069 .unwrap();
17070 assert!(route_task.await.unwrap().is_empty());
17071 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
17072 match swap
17073 .forwarding
17074 .lookup_data_route(client, channel, epoch)
17075 .unwrap()
17076 {
17077 DataRoute::Client(DataRouteState::Bound(route)) => {
17078 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
17079 }
17080 other => panic!("expected a bound route on the candidate, got {other:?}"),
17081 }
17082 assert!(swap.incumbent_rx.try_recv().is_err());
17083 }
17084
17085 #[tokio::test]
17090 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
17091 let swap = swap_with_incumbent().await;
17092 register_candidate(&swap, Some(false));
17093 let update = Frame::build(
17094 FrameType::Request,
17095 control_flags(),
17096 0,
17097 0,
17098 55,
17099 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
17100 provides: manifest("aft", PROTOCOL_VERSION).provides,
17101 capabilities: None,
17102 ready: Some(true),
17103 })
17104 .unwrap(),
17105 )
17106 .unwrap();
17107
17108 let replies = swap
17109 .handler
17110 .handle_control_frame(&swap.candidate_ctx, update)
17111 .await
17112 .unwrap();
17113
17114 assert_eq!(replies.len(), 1);
17115 assert_eq!(
17116 replies[0].header.ty,
17117 FrameType::Response,
17118 "candidate catalog.update was refused: {:?}",
17119 serde_json::from_slice::<Value>(&replies[0].body).ok()
17120 );
17121 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
17122 assert_eq!(
17123 swap.registry
17124 .get_module("aft")
17125 .unwrap()
17126 .unwrap()
17127 .connection_id,
17128 INCUMBENT
17129 );
17130 }
17131 }
17132
17133 mod swap_admission {
17137 use super::*;
17138
17139 const INCUMBENT_NONCE: &str = "incumbent-nonce";
17140 const CANDIDATE_NONCE: &str = "candidate-nonce";
17141
17142 fn handler_with_incumbent(
17143 module_id: &str,
17144 reserved: bool,
17145 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
17146 let registry = Arc::new(Registry::default());
17147 let supervisor = SupervisorHandle::new();
17148 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
17149 if reserved {
17150 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
17151 }
17152 let handler =
17153 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
17154 let incumbent = handler
17155 .handle_control(
17156 ConnectionId::new(1),
17157 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
17158 )
17159 .unwrap();
17160 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
17161 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
17162 (registry, supervisor, handler)
17163 }
17164
17165 #[test]
17171 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
17172 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
17173
17174 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
17175 let replies = handler
17176 .handle_control(
17177 ConnectionId::new(connection),
17178 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
17179 )
17180 .unwrap();
17181 assert_eq!(replies[0].header.ty, FrameType::Error);
17182 assert_eq!(
17183 parse_error(&replies[0])["code"],
17184 "swap_token_invalid",
17185 "nonce {nonce:?}"
17186 );
17187 }
17188 assert!(registry.get_candidate("aft").unwrap().is_none());
17189 assert_eq!(
17190 registry.get_module("aft").unwrap().unwrap().connection_id,
17191 ConnectionId::new(1)
17192 );
17193
17194 let admitted = handler
17197 .handle_control(
17198 ConnectionId::new(4),
17199 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
17200 )
17201 .unwrap();
17202 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
17203 assert_eq!(
17204 registry
17205 .get_candidate("aft")
17206 .unwrap()
17207 .unwrap()
17208 .connection_id,
17209 ConnectionId::new(4)
17210 );
17211 assert_eq!(
17212 registry.get_module("aft").unwrap().unwrap().connection_id,
17213 ConnectionId::new(1),
17214 "the candidate must not take the active slot"
17215 );
17216 let replayed = handler
17217 .handle_control(
17218 ConnectionId::new(5),
17219 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
17220 )
17221 .unwrap();
17222 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
17223
17224 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
17228 let squatter = handler
17229 .handle_control(
17230 ConnectionId::new(6),
17231 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
17232 )
17233 .unwrap();
17234 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
17235 assert!(
17236 registry.get_module("aft").unwrap().is_none(),
17237 "a squatter took the active slot of an id being swapped"
17238 );
17239 }
17240
17241 #[test]
17246 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
17247 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
17248
17249 let replies = handler
17250 .handle_control(
17251 ConnectionId::new(2),
17252 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17253 )
17254 .unwrap();
17255
17256 assert_eq!(
17257 replies[0].header.ty,
17258 FrameType::HelloAck,
17259 "reserved candidate refused: {:?}",
17260 serde_json::from_slice::<Value>(&replies[0].body).ok()
17261 );
17262 assert_eq!(
17263 registry
17264 .get_candidate("vault")
17265 .unwrap()
17266 .unwrap()
17267 .connection_id,
17268 ConnectionId::new(2)
17269 );
17270 }
17271
17272 #[test]
17275 fn without_an_open_swap_the_ordinary_gates_decide() {
17276 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
17277 supervisor.close_swap("vault");
17278
17279 let candidate = handler
17280 .handle_control(
17281 ConnectionId::new(2),
17282 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17283 )
17284 .unwrap();
17285 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
17286 let duplicate = handler
17287 .handle_control(
17288 ConnectionId::new(3),
17289 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
17290 )
17291 .unwrap();
17292 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
17293 assert!(registry.get_candidate("vault").unwrap().is_none());
17294 }
17295 }
17296
17297 mod scopes {
17301 mod idle_tests {
17302 use super::*;
17303 include!("scope_idle_tests.rs");
17304 }
17305 mod route_open_timings {
17306 use super::*;
17307 include!("route_open_timing_tests.rs");
17308 }
17309 mod route_open_refusal_log {
17310 use super::*;
17311 include!("route_open_refusal_log_tests.rs");
17312 }
17313 mod activity {
17314 include!("activity_scope_tests.rs");
17315 }
17316 mod delta {
17317 include!("scope_handler_delta_tests.rs");
17318 }
17319 mod call_modes {
17320 include!("scope_call_mode_tests.rs");
17321 }
17322 #[tokio::test]
17323 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
17324 let rig = rig().await;
17325 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
17326 "s".into(),
17327 1,
17328 "tool".into(),
17329 ))
17330 .unwrap();
17331 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
17332 let (direct, _rx) = wide_ctx(999);
17333 for (ctx, expected) in [
17334 (&rig.owner, FrameType::Response),
17335 (&direct, FrameType::Error),
17336 ] {
17337 let frame =
17338 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
17339 .unwrap();
17340 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
17341 assert_eq!(replies[0].header.ty, expected);
17342 if expected == FrameType::Response {
17343 let reply: subc_control::ScopeServedByReply =
17344 serde_json::from_slice(&replies[0].body).unwrap();
17345 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
17346 } else {
17347 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
17348 }
17349 }
17350 let reply = call(
17351 &rig.handler,
17352 &rig.owner,
17353 &ModuleControlRequestFromModule::ScopeDescribe {
17354 owner: Principal::Reserved {
17355 module_id: OWNER.into(),
17356 },
17357 scope_ref: "s".into(),
17358 },
17359 )
17360 .await;
17361 assert!(matches!(
17362 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
17363 ModuleControlResponseToModule::ScopeDescribe { .. }
17364 ));
17365 }
17366
17367 use subc_protocol::scope::{
17368 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
17369 ScopeStatus,
17370 };
17371
17372 use super::*;
17373
17374 const OWNER: &str = "prefrontal-core";
17375
17376 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
17377 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
17378 }
17379
17380 async fn call(
17381 handler: &ControlHandler,
17382 ctx: &RouteCtx,
17383 request: &ModuleControlRequestFromModule,
17384 ) -> Frame {
17385 let body = serde_json::to_vec(request).unwrap();
17386 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
17387 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
17388 assert_eq!(replies.len(), 1, "{replies:?}");
17389 replies.pop().unwrap()
17390 }
17391
17392 async fn sync(
17393 handler: &ControlHandler,
17394 ctx: &RouteCtx,
17395 generation: u64,
17396 scopes: Vec<ScopeRecord>,
17397 ) -> Result<ModuleControlResponseToModule, String> {
17398 let reply = call(
17399 handler,
17400 ctx,
17401 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
17402 )
17403 .await;
17404 match reply.header.ty {
17405 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
17406 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
17407 }
17408 }
17409
17410 async fn describe(
17411 handler: &ControlHandler,
17412 ctx: &RouteCtx,
17413 owner: &str,
17414 scope_ref: &str,
17415 ) -> ModuleControlResponseToModule {
17416 let reply = call(
17417 handler,
17418 ctx,
17419 &ModuleControlRequestFromModule::ScopeDescribe {
17420 owner: Principal::Reserved {
17421 module_id: owner.to_string(),
17422 },
17423 scope_ref: scope_ref.to_string(),
17424 },
17425 )
17426 .await;
17427 assert_eq!(
17428 reply.header.ty,
17429 FrameType::Response,
17430 "{:?}",
17431 parse_error(&reply)
17432 );
17433 serde_json::from_slice(&reply.body).unwrap()
17434 }
17435
17436 #[tokio::test]
17437 async fn scope_describe_never_echoes_bind_proof_or_hash() {
17438 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17439 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17440 let mut rig = rig().await;
17441 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17442 .await;
17443 let _bound = rig
17444 .bound(
17445 Some(AFT),
17446 PLEXUS,
17447 Some(
17448 rig.selector("s", Some(1))
17449 .with_bind_proof(Some(PROOF.into())),
17450 ),
17451 )
17452 .await;
17453 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
17454 let output = serde_json::to_string(&reply).unwrap();
17455 assert!(output.contains("\"status\":\"live\""));
17456 assert!(!output.contains(PROOF));
17457 assert!(!output.contains(HASH));
17458 }
17459
17460 async fn module(
17462 handler: &ControlHandler,
17463 connection: u64,
17464 module_id: &str,
17465 nonce: Option<&str>,
17466 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17467 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
17468 hello_via_sink(
17469 handler,
17470 &ctx,
17471 &mut rx,
17472 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
17473 )
17474 .await;
17475 (ctx, rx)
17476 }
17477
17478 #[tokio::test]
17481 async fn a_client_connection_cannot_sync_or_describe() {
17482 let handler = ControlHandler::new(Arc::new(Registry::default()));
17483 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
17484 for request in [
17485 ModuleControlRequestFromModule::ScopeSync {
17486 generation: 1,
17487 scopes: vec![head("s", 1)],
17488 },
17489 ModuleControlRequestFromModule::ScopeDescribe {
17490 owner: Principal::Direct,
17491 scope_ref: "s".to_string(),
17492 },
17493 ] {
17494 let reply = call(&handler, &ctx, &request).await;
17495 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
17496 }
17497 assert!(
17498 !handler
17499 .scopes
17500 .read()
17501 .unwrap()
17502 .describe(
17503 &Principal::Reserved {
17504 module_id: OWNER.to_string()
17505 },
17506 "s"
17507 )
17508 .owner_synced
17509 );
17510 }
17511
17512 #[tokio::test]
17515 async fn a_module_without_a_supervised_launch_cannot_sync() {
17516 let handler = ControlHandler::new(Arc::new(Registry::default()));
17517 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
17518 assert_eq!(
17519 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
17520 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17521 );
17522 }
17523
17524 #[tokio::test]
17525 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
17526 let supervisor = SupervisorHandle::new();
17527 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17528 let handler = ControlHandler::new(Arc::new(Registry::default()))
17529 .with_supervisor(supervisor.clone());
17530 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17531 sync(&handler, &incumbent, 1, vec![head("s", 1)])
17532 .await
17533 .expect("the current launch syncs");
17534
17535 supervisor.open_swap(OWNER, "n2".to_string());
17538 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
17539 assert_eq!(
17540 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17541 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17542 );
17543 sync(&handler, &incumbent, 2, vec![head("s", 1)])
17544 .await
17545 .expect("the serving owner syncs during the swap");
17546
17547 supervisor.close_swap(OWNER);
17550 assert_eq!(
17551 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17552 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17553 );
17554 sync(&handler, &incumbent, 3, vec![head("s", 1)])
17555 .await
17556 .expect("the serving owner syncs after the rollback");
17557 handler.cleanup_connection(candidate.connection_id).unwrap();
17558
17559 supervisor.open_swap(OWNER, "n3".to_string());
17564 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
17565 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
17566 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
17567 .await
17568 .expect("the promoted launch takes authority");
17569 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17570 panic!("unexpected reply {reply:?}");
17571 };
17572 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
17573 assert_eq!(
17574 sync(&handler, &incumbent, 4, Vec::new()).await,
17575 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17576 );
17577 }
17578
17579 #[tokio::test]
17582 async fn closing_the_authority_connection_frees_sync_authority() {
17583 let supervisor = SupervisorHandle::new();
17584 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17585 let handler = ControlHandler::new(Arc::new(Registry::default()))
17586 .with_supervisor(supervisor.clone());
17587 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17588 sync(&handler, &first, 10, vec![head("s", 1)])
17589 .await
17590 .unwrap();
17591 handler.cleanup_connection(first.connection_id).unwrap();
17592
17593 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
17594 sync(&handler, &second, 1, vec![head("s", 1)])
17595 .await
17596 .expect("the next connection takes the released authority");
17597 }
17598
17599 #[tokio::test]
17600 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
17601 let supervisor = SupervisorHandle::new();
17602 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17603 let handler =
17604 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
17605 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
17606 sync(&handler, &first, 10, vec![head("s", 1)])
17607 .await
17608 .unwrap();
17609 handler
17610 .handle_control_frame(
17611 &first,
17612 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
17613 )
17614 .await
17615 .unwrap();
17616 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
17617 sync(&handler, &second, 1, vec![head("s", 1)])
17618 .await
17619 .expect("GOODBYE releases authority even if the old socket remains open");
17620 }
17621
17622 #[tokio::test]
17623 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
17624 let registry = Arc::new(Registry::default());
17625 let supervisor_handle = SupervisorHandle::new();
17626 let supervisor =
17627 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
17628 .with_handle(supervisor_handle.clone())
17629 .with_daemon_incarnation("incarnation-7".to_string());
17630 supervisor
17633 .supervise_configured(
17634 ModuleSpec {
17635 carry_containment: false,
17636 module_id: OWNER.to_string(),
17637 program: PathBuf::from("/nonexistent/prefrontal-core"),
17638 args: Vec::new(),
17639 env: Vec::new(),
17640 reserved: false,
17641 reserved_prefixes: Vec::new(),
17642 protocol: ModuleProtocol::Subc,
17643 overlap: Default::default(),
17644 },
17645 false,
17646 )
17647 .unwrap();
17648 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
17649 let handler =
17650 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17651 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
17652
17653 let ModuleControlResponseToModule::ScopeDescribe {
17655 status,
17656 daemon_incarnation,
17657 owner_synced,
17658 owner_configured,
17659 scope,
17660 ..
17661 } = describe(&handler, &reader, OWNER, "s").await
17662 else {
17663 panic!("not a describe reply");
17664 };
17665 assert_eq!(status, ScopeStatus::NotLive);
17666 assert_eq!(daemon_incarnation, "incarnation-7");
17667 assert!(!owner_synced);
17668 assert!(owner_configured);
17669 assert!(scope.is_none());
17670
17671 let ModuleControlResponseToModule::ScopeDescribe {
17674 status,
17675 owner_configured,
17676 ..
17677 } = describe(&handler, &reader, "ghost", "s").await
17678 else {
17679 panic!("not a describe reply");
17680 };
17681 assert_eq!(status, ScopeStatus::NotLive);
17682 assert!(!owner_configured);
17683
17684 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
17686 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
17687 let ModuleControlResponseToModule::ScopeDescribe {
17688 status,
17689 scope_epoch,
17690 owner_synced,
17691 scope,
17692 ..
17693 } = describe(&handler, &reader, OWNER, "s").await
17694 else {
17695 panic!("not a describe reply");
17696 };
17697 assert_eq!(status, ScopeStatus::Live);
17698 assert_eq!(scope_epoch, Some(4));
17699 assert!(owner_synced);
17700 let stamp = scope.expect("a live scope carries its stamp");
17701 assert!(
17702 stamp.owner_authorized,
17703 "prefrontal-core is the default authority"
17704 );
17705 assert_eq!(stamp.kind, ScopeKind::Head);
17706 }
17707
17708 #[tokio::test]
17709 async fn scope_authority_owners_decides_owner_authorized() {
17710 let supervisor = SupervisorHandle::new();
17711 supervisor.set_spawn_nonce("broca", "b1".to_string());
17712 let handler = ControlHandler::new(Arc::new(Registry::default()))
17713 .with_supervisor(supervisor)
17714 .with_scope_authority_owners(vec!["broca".to_string()]);
17715 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
17716 let mut gated = head("s", 1);
17717 gated.attributes.agent_id = Some("agent".to_string());
17718 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
17719 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
17720 describe(&handler, &broca, "broca", "s").await
17721 else {
17722 panic!("not a describe reply");
17723 };
17724 assert!(scope.unwrap().owner_authorized);
17725 }
17726
17727 #[tokio::test]
17731 async fn scope_ops_and_the_scopes_capability_are_advertised() {
17732 let handler = ControlHandler::new(Arc::new(Registry::default()));
17733 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
17734 let ack = hello_via_sink(
17735 &handler,
17736 &ctx,
17737 &mut rx,
17738 hello_frame("m", PROTOCOL_VERSION, 1),
17739 )
17740 .await;
17741 let ack = parse_ack(&ack);
17742 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
17743 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
17744 }
17745 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
17746
17747 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
17748 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
17749 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
17750 let reply = handler
17751 .handle_control_frame(&client, frame)
17752 .await
17753 .unwrap()
17754 .pop()
17755 .unwrap();
17756 let ClientControlResponse::ServerDescribe { capabilities, .. } =
17757 serde_json::from_slice(&reply.body).unwrap()
17758 else {
17759 panic!("not a server.describe reply");
17760 };
17761 assert!(
17762 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
17763 "{capabilities:?}"
17764 );
17765 }
17766
17767 const PLEXUS: &str = "plexus";
17770 const OTHER: &str = "other";
17771 const AFT: &str = "aft";
17772 const BROCA: &str = "broca";
17773 const MAGIC: &str = "magic-context";
17774
17775 fn nonce(module_id: &str) -> String {
17776 format!("nonce-{module_id}")
17777 }
17778
17779 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17780 let (tx, rx) = mpsc::channel(64);
17781 (
17782 RouteCtx {
17783 connection_id: ConnectionId::new(connection),
17784 egress: FrameSink::new(tx),
17785 },
17786 rx,
17787 )
17788 }
17789
17790 struct Rig {
17794 handler: ControlHandler,
17795 forwarding: Arc<ForwardingTable>,
17796 owner: RouteCtx,
17797 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17798 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
17799 generation: u64,
17800 next_connection: u64,
17801 _supervisor: Supervisor,
17802 }
17803
17804 async fn rig() -> Rig {
17805 rig_with_flow_support(true).await
17806 }
17807
17808 async fn rig_with_flow_support(flow_support: bool) -> Rig {
17809 rig_with_scope_capabilities(flow_support, false).await
17810 }
17811
17812 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
17813 let mut provides = Vec::new();
17814 if flow_support {
17815 provides.push("flow-scopes/v1");
17816 }
17817 if run_support {
17818 provides.push("agent-run-scopes/v1");
17819 }
17820 rig_with_target_capabilities(&provides).await
17821 }
17822
17823 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
17824 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
17825 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17826 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17829 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
17830 Frame::build(
17831 FrameType::Hello,
17832 control_flags(),
17833 0,
17834 0,
17835 connection,
17836 serde_json::to_vec(&body).unwrap(),
17837 )
17838 .unwrap()
17839 }
17840
17841 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
17842 let registry = Arc::new(Registry::default());
17843 let forwarding = Arc::new(ForwardingTable::default());
17844 let supervisor_handle = SupervisorHandle::new();
17845 let supervisor =
17846 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
17847 .with_handle(supervisor_handle.clone());
17848 supervisor
17849 .supervise_configured(
17850 ModuleSpec {
17851 carry_containment: false,
17852 module_id: OWNER.to_string(),
17853 program: PathBuf::from("/nonexistent/prefrontal-core"),
17854 args: Vec::new(),
17855 env: Vec::new(),
17856 reserved: false,
17857 reserved_prefixes: Vec::new(),
17858 protocol: ModuleProtocol::Subc,
17859 overlap: Default::default(),
17860 },
17861 false,
17862 )
17863 .unwrap();
17864 for module_id in [OWNER, AFT, BROCA, MAGIC] {
17865 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
17866 }
17867 let handler =
17868 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17869 .with_supervisor(supervisor_handle);
17870 let (owner, mut owner_rx) = wide_ctx(1);
17871 hello_via_sink(
17872 &handler,
17873 &owner,
17874 &mut owner_rx,
17875 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
17876 )
17877 .await;
17878 let mut modules = BTreeMap::new();
17879 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
17880 let (ctx, mut rx) = wide_ctx(connection);
17881 let hello = hello_with_capabilities(module_id, connection, provides);
17882 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
17883 modules.insert(module_id.to_string(), (ctx, rx));
17884 }
17885 Rig {
17886 handler,
17887 forwarding,
17888 owner,
17889 _owner_rx: owner_rx,
17890 modules,
17891 generation: 0,
17892 next_connection: 100,
17893 _supervisor: supervisor,
17894 }
17895 }
17896
17897 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
17898 ScopeCarrier::new(Principal::Reserved {
17899 module_id: module_id.to_string(),
17900 })
17901 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
17902 }
17903
17904 fn session(scope_epoch: u64) -> ScopeRecord {
17907 let mut record = head("s", scope_epoch);
17908 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
17909 record.attributes.agent_id = Some("agent-1".to_string());
17910 record.attributes.delegates = true;
17911 record
17912 }
17913
17914 impl Rig {
17915 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
17916 self.generation += 1;
17917 sync(&self.handler, &self.owner, self.generation, scopes)
17918 .await
17919 .expect("the owner's sync is accepted");
17920 }
17921
17922 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17923 ScopeSelector::new(
17924 Principal::Reserved {
17925 module_id: OWNER.to_string(),
17926 },
17927 scope_ref.to_string(),
17928 scope_epoch,
17929 )
17930 }
17931
17932 fn open_frame(
17933 &mut self,
17934 opener: Option<&str>,
17935 target: &str,
17936 scope: Option<ScopeSelector>,
17937 ) -> (
17938 RouteCtx,
17939 mpsc::Receiver<crate::router::OutboundFrame>,
17940 Frame,
17941 ) {
17942 self.next_connection += 1;
17943 let (ctx, rx) = wide_ctx(self.next_connection);
17944 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
17945 busy_refusals: None,
17946 target: RouteTarget::ToolProvider {
17947 module_id: target.to_string(),
17948 },
17949 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
17950 consumer_identity: opener.map(|module_id| ConsumerIdentity {
17951 module_id: module_id.to_string(),
17952 launch_nonce: nonce(module_id),
17953 }),
17954 consumer_capabilities: None,
17955 role_versions: None,
17956 admission_facts: None,
17957 scope,
17958 })
17959 .unwrap();
17960 let frame = Frame::build(
17961 FrameType::Request,
17962 control_flags(),
17963 0,
17964 0,
17965 self.next_connection,
17966 body,
17967 )
17968 .unwrap();
17969 (ctx, rx, frame)
17970 }
17971
17972 async fn refused(
17974 &mut self,
17975 opener: Option<&str>,
17976 target: &str,
17977 scope: Option<ScopeSelector>,
17978 ) -> String {
17979 self.refusal_body(opener, target, scope).await["code"]
17980 .as_str()
17981 .unwrap()
17982 .to_string()
17983 }
17984
17985 async fn refusal_body(
17986 &mut self,
17987 opener: Option<&str>,
17988 target: &str,
17989 scope: Option<ScopeSelector>,
17990 ) -> Value {
17991 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
17992 let replies = tokio::time::timeout(
17993 Duration::from_secs(2),
17994 self.handler.handle_control_frame(&ctx, frame),
17995 )
17996 .await
17997 .expect("the open must be refused before waiting for a bind ack")
17998 .unwrap();
17999 assert_eq!(replies.len(), 1, "{replies:?}");
18000 assert_eq!(replies[0].header.ty, FrameType::Error);
18001 let (_, module_rx) = self.modules.get_mut(target).unwrap();
18002 assert!(
18003 module_rx.try_recv().is_err(),
18004 "a refused open relays nothing"
18005 );
18006 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
18007 parse_error(&replies[0])
18008 }
18009
18010 async fn relayed(
18012 &mut self,
18013 opener: Option<&str>,
18014 target: &str,
18015 scope: Option<ScopeSelector>,
18016 ) -> Relayed {
18017 self.relayed_with_legacy_root(opener, target, scope, None)
18018 .await
18019 }
18020
18021 async fn relayed_with_legacy_root(
18022 &mut self,
18023 opener: Option<&str>,
18024 target: &str,
18025 scope: Option<ScopeSelector>,
18026 legacy_root: Option<&std::path::Path>,
18027 ) -> Relayed {
18028 let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
18029 if let Some(root) = legacy_root {
18030 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
18031 body["identity"]["project_root"] = serde_json::json!(root);
18032 frame.body = serde_json::to_vec(&body).unwrap();
18033 }
18034 let handler = self.handler.clone();
18035 let task_ctx = ctx.clone();
18036 let task = tokio::spawn(async move {
18037 handler
18038 .handle_control_frame(&task_ctx, frame)
18039 .await
18040 .unwrap()
18041 });
18042 let (_, module_rx) = self.modules.get_mut(target).unwrap();
18043 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
18044 .await
18045 .expect("the target receives the relayed route.bind")
18046 .unwrap()
18047 .frame;
18048 Relayed {
18049 target: target.to_string(),
18050 client: ctx,
18051 client_rx: rx,
18052 task,
18053 bind,
18054 }
18055 }
18056
18057 async fn ack(&self, relayed: &Relayed) {
18058 let (module, _) = &self.modules[&relayed.target];
18059 self.handler
18060 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
18061 .await
18062 .unwrap();
18063 }
18064
18065 async fn bound(
18067 &mut self,
18068 opener: Option<&str>,
18069 target: &str,
18070 scope: Option<ScopeSelector>,
18071 ) -> Bound {
18072 let relayed = self.relayed(opener, target, scope).await;
18073 self.commit_relayed(relayed).await
18074 }
18075
18076 async fn bound_with_legacy_root(
18077 &mut self,
18078 opener: Option<&str>,
18079 target: &str,
18080 scope: Option<ScopeSelector>,
18081 root: &std::path::Path,
18082 ) -> Bound {
18083 let relayed = self
18084 .relayed_with_legacy_root(opener, target, scope, Some(root))
18085 .await;
18086 self.commit_relayed(relayed).await
18087 }
18088
18089 async fn commit_relayed(&self, relayed: Relayed) -> Bound {
18090 self.ack(&relayed).await;
18091 let Relayed {
18092 target,
18093 client,
18094 mut client_rx,
18095 task,
18096 bind,
18097 } = relayed;
18098 assert!(
18099 task.await.unwrap().is_empty(),
18100 "the open is answered by commit"
18101 );
18102 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
18103 Bound {
18104 target,
18105 client,
18106 client_rx,
18107 channel,
18108 epoch,
18109 bind,
18110 }
18111 }
18112
18113 fn live(&self, route: &Bound) -> bool {
18114 matches!(
18115 self.forwarding
18116 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18117 .unwrap(),
18118 DataRoute::Client(DataRouteState::Bound(_))
18119 )
18120 }
18121 }
18122
18123 struct Relayed {
18124 target: String,
18125 client: RouteCtx,
18126 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18127 task: tokio::task::JoinHandle<Vec<Frame>>,
18128 bind: Frame,
18129 }
18130
18131 struct Bound {
18132 target: String,
18133 client: RouteCtx,
18134 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18135 channel: u16,
18136 epoch: u32,
18137 bind: Frame,
18138 }
18139
18140 impl Bound {
18141 fn closed_reason(&mut self) -> RouteCloseReason {
18144 let mut reason = None;
18145 let mut goodbye = false;
18146 while let Ok(outbound) = self.client_rx.try_recv() {
18147 let frame = outbound.frame;
18148 match frame.header.ty {
18149 FrameType::Goodbye => {
18150 assert_eq!(
18151 (frame.header.channel, frame.header.epoch),
18152 (self.channel, self.epoch)
18153 );
18154 goodbye = true;
18155 }
18156 FrameType::Push => {
18157 let ClientControlPush::RouteClosed {
18158 reason: r,
18159 module_id,
18160 ..
18161 } = serde_json::from_slice(&frame.body).unwrap()
18162 else {
18163 panic!("unexpected push");
18164 };
18165 assert_eq!(module_id, self.target);
18166 reason = Some(r);
18167 }
18168 other => panic!("unexpected frame {other:?}"),
18169 }
18170 }
18171 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
18172 reason.expect("the client is told why the route closed")
18173 }
18174
18175 fn untouched(&mut self) -> bool {
18176 self.client_rx.try_recv().is_err()
18177 }
18178
18179 fn stamp(&self) -> Option<ScopeStamp> {
18180 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
18181 ModuleControlRequest::RouteBind { scope, .. } => scope,
18182 other => panic!("expected a route.bind, got {other:?}"),
18183 }
18184 }
18185 }
18186
18187 #[tokio::test]
18188 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
18189 ) {
18190 let mut rig = rig().await;
18191 let mut record = session(1);
18192 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
18193 record.child_owners = vec![Principal::Reserved {
18194 module_id: MAGIC.to_string(),
18195 }];
18196 rig.sync(vec![record]).await;
18197 let scope = || Some(rig_selector("s", Some(1)));
18198
18199 rig.bound(Some(OWNER), PLEXUS, scope()).await;
18202 rig.bound(Some(AFT), OTHER, scope()).await;
18203 rig.bound(Some(BROCA), PLEXUS, scope()).await;
18204
18205 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
18209 assert_eq!(
18210 rig.refused(opener, target, scope()).await,
18211 error_codes::SCOPE_NOT_CARRIER,
18212 "{opener:?} -> {target}"
18213 );
18214 }
18215 }
18216
18217 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18218 ScopeSelector::new(
18219 Principal::Reserved {
18220 module_id: OWNER.to_string(),
18221 },
18222 scope_ref.to_string(),
18223 scope_epoch,
18224 )
18225 }
18226
18227 #[tokio::test]
18228 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
18229 let mut rig = rig().await;
18230 rig.sync(vec![session(1)]).await;
18231 for opener in [OWNER, AFT] {
18232 assert_eq!(
18233 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
18234 .await,
18235 error_codes::SCOPE_EPOCH_REQUIRED,
18236 "{opener}"
18237 );
18238 }
18239 }
18240
18241 #[tokio::test]
18242 async fn admission_separates_not_synced_not_live_and_ended() {
18243 let mut rig = rig().await;
18244 let code = rig
18246 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18247 .await;
18248 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
18249 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
18250
18251 let ghost = ScopeSelector::new(
18253 Principal::Reserved {
18254 module_id: "ghost".to_string(),
18255 },
18256 "s".to_string(),
18257 Some(1),
18258 );
18259 assert_eq!(
18260 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
18261 error_codes::SCOPE_NOT_LIVE
18262 );
18263
18264 rig.sync(vec![session(2)]).await;
18265 assert_eq!(
18266 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
18267 .await,
18268 error_codes::SCOPE_NOT_LIVE
18269 );
18270 for epoch in [1, 3] {
18271 assert_eq!(
18272 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
18273 .await,
18274 error_codes::SCOPE_ENDED,
18275 "epoch {epoch}"
18276 );
18277 }
18278 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
18280 .await;
18281 }
18282
18283 #[tokio::test]
18284 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
18285 let mut rig = rig().await;
18286 rig.sync(vec![session(1)]).await;
18287 let route = rig
18288 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18289 .await;
18290 let stamp = route.stamp().expect("a scoped bind carries the stamp");
18291 assert_eq!(stamp.scope_ref, "s");
18292 assert_eq!(stamp.scope_epoch, 1);
18293 assert_eq!(stamp.kind, ScopeKind::Head);
18294 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
18295 assert!(stamp.attributes.delegates);
18296 assert!(stamp.owner_authorized);
18297 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18298 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
18299
18300 let (broca, mut broca_rx) = wide_ctx(50);
18303 hello_via_sink(
18304 &rig.handler,
18305 &broca,
18306 &mut broca_rx,
18307 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
18308 )
18309 .await;
18310 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
18311 .await
18312 .unwrap();
18313 let own = ScopeSelector::new(
18314 Principal::Reserved {
18315 module_id: BROCA.to_string(),
18316 },
18317 "b".to_string(),
18318 Some(1),
18319 );
18320 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
18321 assert!(!route.stamp().unwrap().owner_authorized);
18322 }
18323
18324 #[tokio::test]
18325 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
18326 let mut rig = rig().await;
18327 let mut record = head("s", 1);
18328 record.carriers = vec![carrier(AFT, None)];
18329 let flow_id = "Flow:run-7/step_2!~";
18330 record.attributes.flow_id = Some(flow_id.to_string());
18331 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
18332 .await
18333 .unwrap();
18334 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
18335 panic!("not a sync reply");
18336 };
18337 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
18338 let route = rig
18339 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18340 .await;
18341 assert!(rig.live(&route), "the stamped bind committed");
18342 let stamp = route.stamp().expect("a flow scope carries a stamp");
18343 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
18344 assert_eq!(stamp.attributes.agent_id, None);
18345 assert!(!stamp.attributes.delegates);
18346 assert!(stamp.owner_authorized);
18347 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18348 assert_eq!(unscoped.stamp(), None);
18349 }
18350
18351 #[tokio::test]
18352 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
18353 let mut rig = rig_with_flow_support(false).await;
18354 let mut record = session(1);
18355 record.attributes.flow_id = Some("flow:7".to_string());
18356 rig.sync(vec![record]).await;
18357 for opener in [OWNER, AFT] {
18358 let body = rig
18359 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18360 .await;
18361 assert_eq!(body["code"], "target_flow_unsupported");
18362 let message = body["message"].as_str().unwrap();
18363 for required in [PLEXUS, "flow-scopes/v1"] {
18364 assert!(message.contains(required), "{message}");
18365 }
18366 }
18367 }
18368
18369 #[tokio::test]
18370 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
18371 let mut rig = rig_with_flow_support(true).await;
18372 let mut record = session(1);
18373 record.attributes.flow_id = Some("flow:7".to_string());
18374 rig.sync(vec![record]).await;
18375 for opener in [OWNER, AFT] {
18376 let route = rig
18377 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18378 .await;
18379 assert!(rig.live(&route));
18380 assert_eq!(
18381 route.stamp().unwrap().attributes.flow_id.as_deref(),
18382 Some("flow:7")
18383 );
18384 }
18385 }
18386
18387 #[tokio::test]
18388 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
18389 use std::future::Future;
18390
18391 let mut rig = rig_with_flow_support(true).await;
18392 let mut record = session(1);
18393 record.attributes.flow_id = Some("flow:7".to_string());
18394 rig.sync(vec![record]).await;
18395 let (client, mut client_rx, frame) =
18396 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
18397 for _ in 0..64 {
18400 client.egress.try_send(route_bind_ack(1)).unwrap();
18401 }
18402 let handler = rig.handler.clone();
18403 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18404 std::future::poll_fn(|cx| {
18405 assert!(open.as_mut().poll(cx).is_pending());
18406 std::task::Poll::Ready(())
18407 })
18408 .await;
18409 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18410
18411 let old_connection = rig.modules[PLEXUS].0.connection_id;
18412 rig.handler.cleanup_connection(old_connection).unwrap();
18413 let (replacement, mut replacement_rx) = wide_ctx(200);
18414 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
18415 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18416 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18417 let hello = Frame::build(
18418 FrameType::Hello,
18419 control_flags(),
18420 0,
18421 0,
18422 200,
18423 serde_json::to_vec(&body).unwrap(),
18424 )
18425 .unwrap();
18426 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
18427
18428 client_rx.try_recv().unwrap();
18429 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18430 .await
18431 .expect("the replacement is refused without waiting for a bind ack")
18432 .unwrap();
18433 assert_eq!(replies.len(), 1);
18434 let body = parse_error(&replies[0]);
18435 assert_eq!(body["code"], "target_flow_unsupported");
18436 for required in [PLEXUS, "flow-scopes/v1"] {
18437 assert!(body["message"].as_str().unwrap().contains(required));
18438 }
18439 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
18440 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18441 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18442 assert!(rig
18443 .handler
18444 .registry
18445 .get_module_by_connection(replacement.connection_id)
18446 .unwrap()
18447 .is_some());
18448 }
18449
18450 #[tokio::test]
18451 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
18452 use std::future::Future;
18453 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
18454 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18455 const NEXT_HASH: &str =
18456 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18457 let mut rig = rig().await;
18458 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
18459 record.carriers.push(ScopeCarrier::new(Principal::Direct));
18460 rig.sync(vec![record.clone()]).await;
18461 let (client, mut rx, frame) = rig.open_frame(
18462 None,
18463 PLEXUS,
18464 Some(
18465 rig.selector("s", Some(1))
18466 .with_bind_proof(Some(PROOF.into())),
18467 ),
18468 );
18469 for _ in 0..64 {
18472 client.egress.try_send(route_bind_ack(1)).unwrap();
18473 }
18474 let handler = rig.handler.clone();
18475 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18476 std::future::poll_fn(|cx| {
18477 assert!(open.as_mut().poll(cx).is_pending());
18478 std::task::Poll::Ready(())
18479 })
18480 .await;
18481 record.bind_proof_sha256 = Some(NEXT_HASH.into());
18482 rig.sync(vec![record]).await;
18483 rx.try_recv().unwrap();
18484 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18485 .await
18486 .unwrap()
18487 .unwrap();
18488 assert_eq!(
18489 parse_error(&replies[0])["code"],
18490 error_codes::SCOPE_BIND_PROOF_MISMATCH
18491 );
18492 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
18493 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18494 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18495 }
18496
18497 #[tokio::test]
18498 async fn scope_bind_proof_reserved_openers_ignore_proof() {
18499 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18500 let mut rig = rig().await;
18501 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18502 .await;
18503 for opener in [OWNER, AFT] {
18504 for proof in [None, Some("malformed+proof=")] {
18505 let route = rig
18506 .bound(
18507 Some(opener),
18508 PLEXUS,
18509 Some(
18510 rig.selector("s", Some(1))
18511 .with_bind_proof(proof.map(str::to_string)),
18512 ),
18513 )
18514 .await;
18515 assert!(rig.live(&route));
18516 }
18517 }
18518 }
18519
18520 #[tokio::test]
18521 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
18522 use std::future::Future;
18523 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18524 const NEXT_HASH: &str =
18525 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18526 let mut rig = rig().await;
18527 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18528 .await;
18529 let (client, mut rx, frame) =
18530 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
18531 for _ in 0..64 {
18532 client.egress.try_send(route_bind_ack(1)).unwrap();
18533 }
18534 let handler = rig.handler.clone();
18535 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18536 std::future::poll_fn(|cx| {
18537 assert!(open.as_mut().poll(cx).is_pending());
18538 std::task::Poll::Ready(())
18539 })
18540 .await;
18541 rig.sync(vec![
18542 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
18543 ])
18544 .await;
18545 rx.try_recv().unwrap();
18546 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18547 let bind = std::future::poll_fn(|cx| {
18548 assert!(open.as_mut().poll(cx).is_pending());
18549 module_rx.poll_recv(cx)
18550 })
18551 .await
18552 .unwrap()
18553 .frame;
18554 handler
18555 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
18556 .await
18557 .unwrap();
18558 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18559 .await
18560 .unwrap()
18561 .unwrap();
18562 assert_eq!(replies.len(), 1);
18563 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
18566 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18567 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
18568 let route = rig
18569 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
18570 .await;
18571 assert!(rig.live(&route));
18572 }
18573
18574 #[tokio::test]
18575 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
18576 {
18577 let mut rig = rig_with_flow_support(false).await;
18578 rig.sync(vec![session(1)]).await;
18579 let route = rig
18580 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18581 .await;
18582 assert!(rig.live(&route));
18583 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
18584 let mut record = session(1);
18585 record.attributes.flow_id = Some("flow:7".to_string());
18586 rig.sync(vec![record]).await;
18587 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
18588 assert!(rig.live(&unscoped));
18589 assert_eq!(unscoped.stamp(), None);
18590 }
18591
18592 #[tokio::test]
18593 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
18594 let mut rig = rig().await;
18595 let mut record = session(1);
18596 record.attributes.flow_id = Some("flow:7".to_string());
18597 rig.sync(vec![record.clone()]).await;
18598 let before = rig.forwarding.published_scope_tag(OWNER, "s");
18599 let mut owner_route = rig
18600 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18601 .await;
18602 let mut carrier_route = rig
18603 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
18604 .await;
18605 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18606 rig.sync(vec![record.clone()]).await;
18607 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18608 assert!(rig.live(&owner_route) && owner_route.untouched());
18609 assert!(rig.live(&carrier_route) && carrier_route.untouched());
18610
18611 record.attributes.flow_id = Some("flow:8".to_string());
18612 rig.sync(vec![record]).await;
18613 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
18614 let before = before.unwrap();
18615 assert_eq!(after.scope_epoch, before.scope_epoch);
18616 assert!(after.version > before.version);
18617 for route in [&mut owner_route, &mut carrier_route] {
18618 assert!(!rig.live(route));
18619 assert_eq!(
18620 route.closed_reason(),
18621 RouteCloseReason::ScopeDelegationChanged
18622 );
18623 }
18624 assert!(rig.live(&unscoped) && unscoped.untouched());
18625 for target in [PLEXUS, OTHER] {
18628 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
18629 let goodbye = module_rx
18630 .try_recv()
18631 .expect("the provider sees the drain")
18632 .frame;
18633 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18634 assert!(module_rx.try_recv().is_err());
18635 }
18636 let rebound = rig
18637 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18638 .await;
18639 assert_eq!(
18640 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
18641 Some("flow:8")
18642 );
18643 }
18644
18645 #[tokio::test]
18650 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
18651 let mut rig = rig().await;
18652 rig.sync(vec![session(1)]).await;
18653 let mut cotenant = rig
18654 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18655 .await;
18656
18657 let mut changed = session(1);
18658 changed.child_owners.push(Principal::Reserved {
18659 module_id: MAGIC.to_string(),
18660 });
18661 let mut ended = None;
18662 for (code, next) in [
18663 (error_codes::SCOPE_CHANGED, vec![changed]),
18664 (error_codes::SCOPE_ENDED, Vec::new()),
18665 ] {
18666 let relayed = rig
18667 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18668 .await;
18669 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
18670 ended = Some(next.is_empty());
18671 rig.sync(next).await;
18672 rig.ack(&relayed).await;
18673 let replies = relayed.task.await.unwrap();
18674 assert_eq!(replies.len(), 1, "{replies:?}");
18675 assert_eq!(parse_error(&replies[0])["code"], code);
18676 assert_eq!(
18677 subc_protocol::error_codes::is_retryable_route_open(code),
18678 code == error_codes::SCOPE_CHANGED
18679 );
18680 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18681 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18683 let mut goodbyes = Vec::new();
18686 while let Ok(outbound) = plexus_rx.try_recv() {
18687 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
18688 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
18689 }
18690 assert!(
18691 goodbyes.contains(&(bind_channel, bind_epoch)),
18692 "{goodbyes:?}"
18693 );
18694 assert!(rig
18695 .handler
18696 .registry
18697 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
18698 .unwrap()
18699 .is_some());
18700 }
18701 assert_eq!(ended, Some(true));
18702 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
18705 }
18706
18707 #[tokio::test]
18710 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
18711 struct Case {
18712 name: &'static str,
18713 change: fn(&mut ScopeRecord),
18714 closed: [Option<RouteCloseReason>; 4],
18717 }
18718 use RouteCloseReason::*;
18719 let cases = [
18720 Case {
18721 name: "a carrier entry removed",
18722 change: |r| {
18723 r.carriers.retain(|c| {
18724 c.principal
18725 != Principal::Reserved {
18726 module_id: AFT.to_string(),
18727 }
18728 })
18729 },
18730 closed: [None, Some(ScopeCarrierRemoved), None, None],
18731 },
18732 Case {
18733 name: "a target removed from a carrier",
18734 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
18735 closed: [None, None, None, Some(ScopeCarrierRemoved)],
18736 },
18737 Case {
18738 name: "a bare carrier narrowed to targets",
18739 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
18740 closed: [None, Some(ScopeCarrierRemoved), None, None],
18741 },
18742 Case {
18743 name: "delegates turned off",
18744 change: |r| r.attributes.delegates = false,
18745 closed: [Some(ScopeDelegationChanged); 4],
18746 },
18747 Case {
18748 name: "agent_id changed",
18749 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
18750 closed: [Some(ScopeDelegationChanged); 4],
18751 },
18752 Case {
18753 name: "a carrier added, child owners changed, the record re-sent",
18754 change: |r| {
18755 r.carriers.push(carrier(MAGIC, None));
18756 r.child_owners.push(Principal::Reserved {
18757 module_id: MAGIC.to_string(),
18758 });
18759 },
18760 closed: [None; 4],
18761 },
18762 Case {
18763 name: "a target added",
18764 change: |r| {
18765 r.carriers[1]
18766 .targets
18767 .as_mut()
18768 .unwrap()
18769 .push("third".to_string())
18770 },
18771 closed: [None; 4],
18772 },
18773 Case {
18774 name: "delegates turned on",
18775 change: |r| r.attributes.delegates = true,
18776 closed: [None; 4],
18777 },
18778 ];
18779 for case in cases {
18780 let mut rig = rig().await;
18781 rig.sync(vec![session(1)]).await;
18782 let scope = || Some(rig_selector("s", Some(1)));
18783 let mut routes = [
18784 rig.bound(Some(OWNER), PLEXUS, scope()).await,
18785 rig.bound(Some(AFT), PLEXUS, scope()).await,
18786 rig.bound(Some(BROCA), PLEXUS, scope()).await,
18787 rig.bound(Some(BROCA), OTHER, scope()).await,
18788 ];
18789 let mut record = session(1);
18790 (case.change)(&mut record);
18791 rig.sync(vec![record]).await;
18792 for (index, expected) in case.closed.iter().enumerate() {
18793 let route = &mut routes[index];
18794 match expected {
18795 Some(reason) => {
18796 assert!(!rig.live(route), "{}: route {index} still live", case.name);
18797 assert_eq!(
18798 route.closed_reason(),
18799 *reason,
18800 "{}: route {index}",
18801 case.name
18802 );
18803 }
18804 None => {
18805 assert!(rig.live(route), "{}: route {index} closed", case.name);
18806 assert!(
18807 route.untouched(),
18808 "{}: route {index} was told something",
18809 case.name
18810 );
18811 }
18812 }
18813 }
18814 }
18815 }
18816
18817 #[tokio::test]
18818 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
18819 for next in [Vec::new(), vec![session(2)]] {
18821 let mut rig = rig().await;
18822 rig.sync(vec![session(1)]).await;
18823 let mut route = rig
18824 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18825 .await;
18826 rig.sync(next).await;
18827 assert!(!rig.live(&route));
18828 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
18829 }
18830
18831 let mut rig = rig().await;
18834 let mut child = session(1);
18835 child.scope_ref = "child".to_string();
18836 child.kind = ScopeKind::Worker;
18837 child.parent = Some(ScopeParent::new(
18838 Principal::Reserved {
18839 module_id: OWNER.to_string(),
18840 },
18841 "s".to_string(),
18842 1,
18843 ));
18844 rig.sync(vec![session(1), child.clone()]).await;
18845 let mut child_route = rig
18846 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
18847 .await;
18848 assert_eq!(
18849 child_route.stamp().unwrap().parent_state,
18850 Some(ParentState::Linked)
18851 );
18852 rig.sync(vec![child]).await;
18853 assert!(!rig.live(&child_route));
18854 assert_eq!(
18855 child_route.closed_reason(),
18856 RouteCloseReason::ScopeParentEnded
18857 );
18858 }
18859
18860 #[tokio::test]
18861 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
18862 ) {
18863 let mut rig = rig().await;
18864 rig.sync(vec![session(1)]).await;
18865 let mut route = rig
18866 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18867 .await;
18868 let before = rig.forwarding.published_scope_tag(OWNER, "s");
18869 rig.sync(vec![session(1)]).await;
18870 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18871 assert!(rig.live(&route) && route.untouched());
18872
18873 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
18877 .forwarding
18878 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18879 .unwrap()
18880 else {
18881 panic!("the route is bound");
18882 };
18883 binding.flow.acquire_tagged(9, false).await.unwrap();
18884 let mut widened = session(1);
18885 widened.carriers.push(carrier(MAGIC, None));
18886 rig.sync(vec![widened]).await;
18887 assert!(rig.live(&route) && route.untouched());
18888 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18889 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
18890 assert_eq!(binding.flow.in_flight(), 1);
18893 binding
18894 .flow
18895 .acquire_tagged(10, false)
18896 .await
18897 .expect("the flow is still open");
18898 }
18899
18900 #[tokio::test]
18903 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
18904 let mut rig = rig().await;
18905 rig.sync(vec![session(1)]).await;
18906 let mut on_incumbent = rig
18907 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18908 .await;
18909
18910 let (candidate, _candidate_rx) = wide_ctx(9);
18913 let registration = rig
18914 .handler
18915 .registry
18916 .register_candidate_with_control_ops(
18917 manifest(PLEXUS, PROTOCOL_VERSION),
18918 PROTOCOL_VERSION,
18919 candidate.connection_id,
18920 module_baseline_control_ops(),
18921 )
18922 .unwrap();
18923 rig.forwarding
18924 .register_candidate_module_connection(
18925 candidate.connection_id,
18926 PLEXUS.to_string(),
18927 PROTOCOL_VERSION,
18928 manifest_concurrency(®istration.manifest),
18929 candidate.egress.clone(),
18930 )
18931 .unwrap();
18932 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
18933 rig.handler
18934 .registry
18935 .promote_candidate(PLEXUS)
18936 .unwrap()
18937 .unwrap();
18938 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
18939
18940 rig.sync(Vec::new()).await;
18941 assert!(!rig.live(&on_incumbent));
18942 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
18943 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18944 let goodbye = incumbent_rx
18945 .try_recv()
18946 .expect("the superseded endpoint is told")
18947 .frame;
18948 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18949 }
18950 }
18951}
18952
18953#[cfg(test)]
18954mod concurrency_default_exposure_tests {
18955 use super::*;
18956
18957 fn hello_body(role_json: &str) -> Vec<u8> {
18958 format!(
18959 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
18960 )
18961 .into_bytes()
18962 }
18963
18964 fn manifest_from(body: &[u8]) -> ModuleManifest {
18965 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
18966 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
18967 .expect("manifest parses")
18968 }
18969
18970 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
18971
18972 #[test]
18973 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
18974 let body = hello_body(&format!(
18975 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
18976 ));
18977 let manifest = manifest_from(&body);
18978 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18981 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
18982 }
18983
18984 #[test]
18985 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
18986 let body = hello_body(&format!(
18987 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
18988 ));
18989 let manifest = manifest_from(&body);
18990 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18991 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
18992 }
18993
18994 #[test]
18995 fn non_management_roles_are_never_reported() {
18996 let body = hello_body(
18997 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
18998 );
18999 let manifest = manifest_from(&body);
19000 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19001 }
19002}