1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_confirm_operations,
25 validate_hello_event_declarations, validate_hello_flow_operations,
26 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
27 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
28 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
29 },
30 peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
31 scope::{
32 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
33 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
34 SCOPE_SYNC_OP,
35 },
36 session::{
37 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
38 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
39 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
40 ROLE_VERSIONS_FIELD,
41 },
42 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
43 Priority, RouteTarget, PROTOCOL_VERSION,
44};
45use tokio::time::{timeout_at, Instant};
46use tracing::{debug, info, instrument::WithSubscriber, warn};
47
48use crate::{
49 caller_identity::CallerIdentity,
50 capability_requirements::{
51 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
52 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
53 RequirementStatus, RuntimeModule,
54 },
55 child_roster::DaemonShutdownFlag,
56 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
57 forwarding::{
58 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
59 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
60 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
61 },
62 observability::{
63 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
64 },
65 privileged_prompt::{prompt_text, PrivilegedOperation},
66 provenance::{
67 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
68 },
69 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
70 router::{daemon_error_flags, RouteCtx, RouterError},
71 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
72 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
73 stderr_tail::{CaptureState, TailEntry},
74 supervise::{
75 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
76 SupervisorHandle, SwapHelloAdmission,
77 },
78 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
79};
80
81pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
87
88const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
89const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
90const CAP_PING_PONG: &str = "ping_pong_v1";
91const CAP_SESSION_ATTACH: &str = "session_attach_v1";
92const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
93
94const SUBC_CONTROL_OPS: &[&str] = &[
95 ops::DAEMON_SHUTDOWN_INTENT,
96 ops::ACTIVITY_RECORD,
97 ops::ACTIVITY_QUERY,
98 ops::ACTIVITY_SUBSCRIBE,
99 ops::SERVER_DESCRIBE,
100 ops::CATALOG_LIST,
101 ops::ROUTE_OPEN,
102 ops::ROUTE_POLL,
103 ops::ROUTE_CLOSING,
104 ops::ROUTE_CLOSED,
105 ops::SUPERVISOR_LIST,
106 ops::SUPERVISOR_RESTART,
107 ops::SUPERVISOR_SWAP,
108 ops::SUPERVISOR_RELOAD,
109 ops::SUPERVISOR_RESCAN,
110 ops::SUPERVISOR_RELEASE_RESERVED,
111 ops::SUPERVISOR_SET_ENABLED,
112 ops::SUPERVISOR_HEALTH_PROBE,
113 ops::SUPERVISOR_HEALTH,
114 ops::SUPERVISOR_STDERR_TAIL,
115 ops::SUPERVISOR_TERMINALS,
116 ops::SUPERVISOR_ROUTES,
117 ops::SUPERVISOR_PROVENANCE,
118 ops::SUPERVISOR_SPAWN_SNAPSHOT,
119 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
120];
121
122const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
129
130mod rescan_checked;
131use rescan_checked::{RescanMode, RescanPlan};
132
133const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
134 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
135 "supervisor.live_roots",
136 SCOPE_SYNC_OP,
137 SCOPE_APPLY_OP,
138 SCOPE_DESCRIBE_OP,
139 "operator.confirm",
140];
141
142const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
146
147const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
148
149pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
157
158pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
173
174pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
192
193const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
194const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
195
196fn reload_verdict(
197 configured: &Path,
198 spawned_from: Option<&Path>,
199 image: subc_control::RunningImageAgreement,
200) -> PendingReloadVerdict {
201 let path = match spawned_from {
202 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
203 ReloadPathAgreement::Match
204 }
205 Some(spawned_from) => ReloadPathAgreement::Mismatch {
206 configured: configured.to_path_buf(),
207 spawned_from: spawned_from.to_path_buf(),
208 },
209 None => ReloadPathAgreement::Unavailable {
210 reason: if matches!(
211 image,
212 subc_control::RunningImageAgreement::Unavailable {
213 reason: subc_control::RunningImageUnavailableReason::NotRunning
214 }
215 ) {
216 ReloadPathUnavailableReason::NotRunning
217 } else {
218 ReloadPathUnavailableReason::SpawnedPathUnavailable
219 },
220 },
221 };
222 PendingReloadVerdict {
223 path,
224 image,
225 spec: None,
226 }
227}
228
229fn launch_spec_agreement(
230 configured: &crate::supervise::ModuleSpec,
231 running: bool,
232 spawned: Option<&crate::supervise::LaunchSpecDigest>,
233) -> subc_control::LaunchSpecAgreement {
234 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
235 if !running {
236 return Agreement::Unavailable {
237 reason: Reason::NotRunning,
238 };
239 }
240 match spawned {
241 None => Agreement::Unavailable {
242 reason: Reason::SpawnedSpecUnavailable,
243 },
244 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
245 Agreement::Match
246 }
247 Some(_) => Agreement::Mismatch,
248 }
249}
250
251fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
252 if configured == spawned_from {
253 return true;
254 }
255 #[cfg(windows)]
256 {
257 match (
263 std::fs::canonicalize(configured),
264 std::fs::canonicalize(spawned_from),
265 ) {
266 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
267 _ => false,
268 }
269 }
270 #[cfg(not(windows))]
271 {
272 false
273 }
274}
275
276#[derive(Clone)]
277struct DaemonProvenanceFacts {
278 build: DaemonBuildProvenance,
279 pid: Option<u32>,
280 started_at_ms: Option<u64>,
281 start_clock: Option<crate::clock::StartClock>,
282 executable_path: Option<PathBuf>,
283 executable_identity: Option<SpawnedFileIdentity>,
284 process_start_time: Option<u64>,
285 probe: ExecutableIdentityProbe,
286}
287
288impl Default for DaemonProvenanceFacts {
289 fn default() -> Self {
290 Self {
291 build: DaemonBuildProvenance {
292 build_git_sha: None,
293 build_lock_digest: None,
294 },
295 pid: None,
296 started_at_ms: None,
297 start_clock: None,
298 executable_path: None,
299 executable_identity: None,
300 process_start_time: None,
301 probe: ExecutableIdentityProbe::default(),
302 }
303 }
304}
305
306#[derive(Debug, Clone)]
307struct SupervisorRescanContext {
308 supervisor: Supervisor,
309 config_path: PathBuf,
310 configured_port: Option<u16>,
311 storage_config: Option<crate::daemon_config::StorageConfig>,
312 admission_facts_carrier_module_id: Option<String>,
313 admission_facts_targets: Option<Vec<String>>,
314 scope_authority_owners: Vec<String>,
315 privileged_operator_prompt: PrivilegedOperatorPrompt,
316 generation: Arc<std::sync::atomic::AtomicU64>,
322}
323
324const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
345 "reloading",
346 "supervisor_not_live",
347 "registration_not_active",
348 "no_forwarding_connection",
349 "relay_send_failed",
350];
351
352#[derive(Clone)]
354pub struct ControlHandler {
355 registry: Arc<Registry>,
356 activity: Option<crate::activity::ActivityLedger>,
357 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
358 forwarding: Arc<ForwardingTable>,
359 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
360 supervisor: SupervisorHandle,
361 subc_capabilities: Arc<[String]>,
362 route_bind_relay_timeout: Duration,
366 route_bind_relay_timeouts: BTreeMap<String, Duration>,
370 route_bind_breakers: RouteBindBreakers,
373 route_bind_concurrency: RouteBindConcurrency,
376 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
381 route_bind_breaker_threshold: u32,
383 route_bind_breaker_cooldown: Duration,
385 health_probe_timeout: Duration,
386 storage_config: Option<crate::daemon_config::StorageConfig>,
389 machine_id: Option<crate::machine_id::MachineId>,
393 admission_facts_carrier_module_id: Option<String>,
394 admission_facts_targets: Option<Vec<String>>,
395 scopes: Arc<RwLock<ScopeTable>>,
399 scope_deadline_added: Arc<tokio::sync::Notify>,
403 scope_authority_owners: Vec<String>,
406 privileged_operator_prompt: PrivilegedOperatorPrompt,
408 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
411 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
412 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
413 local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
414 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
415 pub(crate) operator_apps:
416 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
417 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
422 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
428 #[cfg(test)]
430 pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
431 #[cfg(test)]
432 pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
433 #[allow(dead_code)]
437 #[cfg(windows)]
438 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
439 #[allow(dead_code)]
440 #[cfg(windows)]
441 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
442 #[allow(dead_code)]
446 #[cfg(windows)]
447 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
448 #[cfg(all(test, windows))]
451 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
452 #[cfg(test)]
453 caller_lookup: Option<TestCallerLookup>,
454 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
457 rescan: Option<SupervisorRescanContext>,
458 connected_clients: ConnectedClients,
459 counters: DaemonCounters,
460 local_endpoint_unavailable: Option<String>,
463 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
464 daemon_shutdown: DaemonShutdownFlag,
469 daemon_provenance: DaemonProvenanceFacts,
470 #[cfg(test)]
471 control_dispatch_delay: Option<Duration>,
472 #[cfg(test)]
473 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
474 #[cfg(test)]
478 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
479 #[cfg(test)]
485 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
486}
487
488impl fmt::Debug for ControlHandler {
489 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
490 f.debug_struct("ControlHandler")
491 .field("registry", &self.registry)
492 .field("forwarding", &self.forwarding)
493 .field("process_liveness", &self.process_liveness.is_some())
494 .field("supervisor", &self.supervisor)
495 .field("subc_capabilities", &self.subc_capabilities)
496 .finish()
497 }
498}
499
500struct RouteOpenRequest {
501 target: RouteTarget,
502 identity: BindIdentity,
503 consumer_identity: Option<ConsumerIdentity>,
504 consumer_capabilities: Option<Vec<String>>,
505 role_versions: Option<BTreeMap<String, String>>,
506 busy_refusals: Option<bool>,
507 admission_facts: Option<serde_json::Value>,
508 scope: Option<ScopeSelector>,
509}
510
511#[derive(Clone, Copy)]
512enum RouteOpenPhase {
513 Admission,
514 ProjectRoot,
515 EgressReservation,
516 BindSend,
517 BindAckWait,
518}
519
520struct RouteOpenTimings {
524 module_id: String,
525 principal: String,
526 admission: Duration,
527 project_root: Duration,
528 egress_reservation: Duration,
529 bind_send: Duration,
530 bind_ack_wait: Duration,
531 phase: Option<RouteOpenPhase>,
532 phase_started_at: Instant,
533}
534
535impl RouteOpenTimings {
536 fn new() -> Self {
537 Self {
538 module_id: String::new(),
539 principal: "unresolved".to_string(),
541 admission: Duration::ZERO,
542 project_root: Duration::ZERO,
543 egress_reservation: Duration::ZERO,
544 bind_send: Duration::ZERO,
545 bind_ack_wait: Duration::ZERO,
546 phase: Some(RouteOpenPhase::Admission),
547 phase_started_at: Instant::now(),
548 }
549 }
550
551 fn finish(&mut self) {
552 let elapsed = self.phase_started_at.elapsed();
553 match self.phase.take() {
554 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
555 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
556 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
557 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
558 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
559 None => {}
560 }
561 }
562
563 fn enter(&mut self, phase: RouteOpenPhase) {
564 self.finish();
565 self.phase = Some(phase);
566 self.phase_started_at = Instant::now();
567 }
568}
569
570struct RouteBindReservationGuard {
571 forwarding: Arc<ForwardingTable>,
572 endpoint: ModuleEndpointId,
573 relay_corr: u64,
574 armed: bool,
575}
576
577struct ModuleControlRpcGuard {
578 forwarding: Arc<ForwardingTable>,
579 endpoint: ModuleEndpointId,
580 corr: u64,
581 armed: bool,
582}
583
584impl ModuleControlRpcGuard {
585 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
586 Self {
587 forwarding,
588 endpoint,
589 corr,
590 armed: true,
591 }
592 }
593
594 fn disarm(&mut self) {
595 self.armed = false;
596 }
597}
598
599impl Drop for ModuleControlRpcGuard {
600 fn drop(&mut self) {
601 if self.armed {
602 let _ = self
603 .forwarding
604 .cancel_module_control_rpc(self.endpoint, self.corr);
605 }
606 }
607}
608
609impl RouteBindReservationGuard {
610 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
611 Self {
612 forwarding,
613 endpoint,
614 relay_corr,
615 armed: true,
616 }
617 }
618
619 fn release_and_disarm(&mut self) {
620 if !self.armed {
621 return;
622 }
623 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
624 self.endpoint,
625 self.relay_corr,
626 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
627 ) {
628 send_goodbye_target_best_effort(
629 &self.forwarding.counters(),
630 &target,
631 "canceled route.bind",
632 );
633 }
634 self.armed = false;
635 }
636
637 fn disarm(&mut self) {
638 self.armed = false;
639 }
640}
641
642impl Drop for RouteBindReservationGuard {
643 fn drop(&mut self) {
644 self.release_and_disarm();
645 }
646}
647
648#[derive(Debug, Clone, Default)]
678pub(crate) struct RouteBindBreakers {
679 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
680}
681
682#[derive(Debug, Clone, Default)]
683pub(crate) struct RouteBindConcurrency {
684 modules: Arc<Mutex<HashMap<String, usize>>>,
685}
686
687struct RouteBindConcurrencyGuard {
688 concurrency: RouteBindConcurrency,
689 module_id: String,
690}
691
692impl RouteBindConcurrency {
693 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
696 let mut modules = self
697 .modules
698 .lock()
699 .expect("route.bind concurrency mutex poisoned");
700 let in_flight = modules.entry(module_id.to_string()).or_default();
701 if *in_flight >= limit {
702 return Err(*in_flight);
703 }
704 *in_flight += 1;
705 Ok(RouteBindConcurrencyGuard {
706 concurrency: self.clone(),
707 module_id: module_id.to_string(),
708 })
709 }
710}
711
712impl Drop for RouteBindConcurrencyGuard {
713 fn drop(&mut self) {
714 let mut modules = self
715 .concurrency
716 .modules
717 .lock()
718 .expect("route.bind concurrency mutex poisoned");
719 let remove = {
720 let in_flight = modules
721 .get_mut(&self.module_id)
722 .expect("admitted route.bind has a concurrency entry");
723 *in_flight -= 1;
724 *in_flight == 0
725 };
726 if remove {
727 modules.remove(&self.module_id);
728 }
729 }
730}
731
732#[derive(Debug, Default)]
733struct ModuleBreakerState {
734 consecutive_timeouts: u32,
736 cooldown_until: Option<Instant>,
739 probe_in_flight: Option<Arc<()>>,
744}
745
746enum RouteBindAdmission<'a> {
748 Admitted {
749 guard: RouteBindBreakerGuard<'a>,
750 probe: bool,
753 },
754 Refused {
755 consecutive_timeouts: u32,
756 retry_in: Duration,
760 probe_in_flight: bool,
761 },
762}
763
764struct RouteBindBreakerGuard<'a> {
771 breakers: RouteBindBreakers,
772 module_id: &'a str,
773 probe_token: Option<Arc<()>>,
774 settled: bool,
775}
776
777impl RouteBindBreakerGuard<'_> {
778 fn record_accepted(&mut self) -> bool {
782 self.settled = true;
783 self.breakers.record_accepted(self.module_id)
784 }
785
786 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
789 self.settled = true;
790 self.breakers.record_timeout(
791 self.module_id,
792 self.probe_token.as_ref(),
793 threshold,
794 cooldown,
795 )
796 }
797
798 fn record_inconclusive(&mut self) {
807 self.settled = true;
808 self.breakers
809 .record_inconclusive(self.module_id, self.probe_token.as_ref());
810 }
811}
812
813impl Drop for RouteBindBreakerGuard<'_> {
814 fn drop(&mut self) {
815 if !self.settled {
816 self.breakers
817 .record_inconclusive(self.module_id, self.probe_token.as_ref());
818 }
819 }
820}
821
822struct BreakerOpened {
826 consecutive_timeouts: u32,
827 reopened_after_probe: bool,
830}
831
832impl RouteBindBreakers {
833 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
834 self.modules
835 .lock()
836 .expect("route.bind breaker mutex poisoned")
837 }
838
839 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
842 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
843 probe: probe_token.is_some(),
844 guard: RouteBindBreakerGuard {
845 breakers: self.clone(),
846 module_id,
847 probe_token,
848 settled: false,
849 },
850 };
851
852 let mut modules = self.lock();
853 let Some(state) = modules.get_mut(module_id) else {
854 return admitted(None);
855 };
856 let Some(cooldown_until) = state.cooldown_until else {
857 return admitted(None);
858 };
859 if state.probe_in_flight.is_some() {
860 return RouteBindAdmission::Refused {
861 consecutive_timeouts: state.consecutive_timeouts,
862 retry_in: Duration::ZERO,
863 probe_in_flight: true,
864 };
865 }
866 let now = Instant::now();
867 if now < cooldown_until {
868 return RouteBindAdmission::Refused {
869 consecutive_timeouts: state.consecutive_timeouts,
870 retry_in: cooldown_until - now,
871 probe_in_flight: false,
872 };
873 }
874 let token = Arc::new(());
875 state.probe_in_flight = Some(Arc::clone(&token));
876 admitted(Some(token))
877 }
878
879 fn record_accepted(&self, module_id: &str) -> bool {
880 self.lock()
881 .remove(module_id)
882 .is_some_and(|state| state.cooldown_until.is_some())
883 }
884
885 fn record_timeout(
886 &self,
887 module_id: &str,
888 probe_token: Option<&Arc<()>>,
889 threshold: u32,
890 cooldown: Duration,
891 ) -> Option<BreakerOpened> {
892 let mut modules = self.lock();
893 let state = modules.entry(module_id.to_string()).or_default();
894 let was_open = state.cooldown_until.is_some();
895 let was_probe = Self::owns_probe(state, probe_token);
896 if was_probe {
897 state.probe_in_flight = None;
898 }
899 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
900 if state.consecutive_timeouts < threshold {
901 return None;
902 }
903 state.cooldown_until = Some(Instant::now() + cooldown);
904 Some(BreakerOpened {
905 consecutive_timeouts: state.consecutive_timeouts,
906 reopened_after_probe: was_open && was_probe,
907 })
908 }
909
910 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
911 state
915 .probe_in_flight
916 .as_ref()
917 .zip(token)
918 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
919 }
920
921 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
922 if let Some(state) = self.lock().get_mut(module_id) {
923 if Self::owns_probe(state, probe_token) {
924 state.probe_in_flight = None;
925 }
926 }
927 }
928
929 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
944 self.lock()
945 .remove(module_id)
946 .map(|state| state.consecutive_timeouts)
947 .filter(|discarded| *discarded > 0)
948 }
949
950 fn open_snapshot(&self) -> Option<serde_json::Value> {
957 let now = Instant::now();
958 let modules = self.lock();
959 let open = modules
960 .iter()
961 .filter_map(|(module_id, state)| {
962 let cooldown_until = state.cooldown_until?;
963 Some((
964 module_id.clone(),
965 serde_json::json!({
966 "consecutive_timeouts": state.consecutive_timeouts,
967 "cooldown_remaining_ms":
968 cooldown_until.saturating_duration_since(now).as_millis() as u64,
969 "probe_in_flight": state.probe_in_flight.is_some(),
970 }),
971 ))
972 })
973 .collect::<serde_json::Map<String, serde_json::Value>>();
974 (!open.is_empty()).then_some(serde_json::Value::Object(open))
975 }
976}
977
978impl ControlHandler {
979 pub fn new(registry: Arc<Registry>) -> Self {
980 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
981 }
982
983 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
984 let counters = forwarding.counters();
985 let route_bind_breakers = forwarding.route_bind_breakers();
989 let route_bind_concurrency = forwarding.route_bind_concurrency();
990 let route_outages = forwarding.route_outages();
991 Self {
992 registry,
993 activity: None,
994 activity_streams: Arc::new(Mutex::new(HashMap::new())),
995 forwarding,
996 process_liveness: None,
997 supervisor: SupervisorHandle::new(),
998 subc_capabilities: Arc::from([
999 CAP_MANIFEST_REGISTRATION.to_string(),
1000 CAP_CHANNEL_LIFECYCLE.to_string(),
1001 CAP_PING_PONG.to_string(),
1002 CAP_SESSION_ATTACH.to_string(),
1003 CAP_ADMISSION_FACTS_RELAY.to_string(),
1004 CAP_SCOPES_V1.to_string(),
1005 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1006 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1007 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1008 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1009 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1010 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1011 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1012 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1013 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1014 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1015 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1016 ]),
1017 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1018 route_bind_relay_timeouts: BTreeMap::new(),
1019 route_bind_breakers,
1020 route_bind_concurrency,
1021 route_outages,
1022 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1023 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1024 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1025 storage_config: None,
1026 machine_id: None,
1027 admission_facts_carrier_module_id: None,
1028 admission_facts_targets: None,
1029 scopes: Arc::new(RwLock::new(ScopeTable::new(
1030 crate::daemon_config::default_scope_authority_owners(),
1031 ))),
1032 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1033 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1034 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1035 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1036 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1037 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1038 local_connections: Arc::new(Mutex::new(HashSet::new())),
1039 operator_app_policy: Default::default(),
1040 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1041 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1042 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1043 #[cfg(test)]
1044 app_open_test_platform: None,
1045 #[cfg(test)]
1046 app_open_test_launch: None,
1047 #[cfg(windows)]
1048 windows_verify: None,
1049 #[cfg(windows)]
1050 windows_flash: None,
1051 #[cfg(windows)]
1052 windows_track: None,
1053 #[cfg(all(test, windows))]
1054 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1055 #[cfg(test)]
1056 caller_lookup: None,
1057 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1058 rescan: None,
1059 connected_clients: ConnectedClients::new(),
1060 local_endpoint_unavailable: None,
1061 counters,
1062 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1063 daemon_shutdown: DaemonShutdownFlag::default(),
1064 daemon_provenance: DaemonProvenanceFacts::default(),
1065 #[cfg(test)]
1066 control_dispatch_delay: None,
1067 #[cfg(test)]
1068 provenance_probe_override: None,
1069 #[cfg(test)]
1070 route_bind_relay_sent_hold: None,
1071 #[cfg(test)]
1072 hello_registry_to_forwarding_hold: None,
1073 }
1074 }
1075
1076 pub fn with_storage_config(
1079 mut self,
1080 storage_config: Option<crate::daemon_config::StorageConfig>,
1081 ) -> Self {
1082 self.storage_config = storage_config;
1083 self
1084 }
1085
1086 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1089 self.activity = Some(crate::activity::ActivityLedger::open(
1090 run_dir.join("activity.jsonl"),
1091 incarnation,
1092 ));
1093 self.supervisor.set_activity(self.activity.clone());
1094 self
1095 }
1096
1097 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1100 self.machine_id = machine_id;
1101 self
1102 }
1103
1104 pub fn with_admission_facts_config(
1108 mut self,
1109 carrier_module_id: Option<String>,
1110 targets: Option<Vec<String>>,
1111 ) -> Self {
1112 self.admission_facts_carrier_module_id = carrier_module_id;
1113 self.admission_facts_targets = targets;
1114 self
1115 }
1116
1117 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1121 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1122 self.scope_authority_owners = owners;
1123 self
1124 }
1125
1126 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1128 self.privileged_operator_prompt = mode;
1129 self
1130 }
1131
1132 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1135 self.route_bind_relay_timeout = timeout;
1136 self
1137 }
1138
1139 pub fn with_route_bind_relay_timeouts(
1145 mut self,
1146 timeouts: impl IntoIterator<Item = (String, Duration)>,
1147 ) -> Self {
1148 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1149 self
1150 }
1151
1152 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1158 self.route_bind_relay_timeouts
1159 .get(module_id)
1160 .copied()
1161 .unwrap_or(self.route_bind_relay_timeout)
1162 }
1163
1164 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1172 self.route_bind_breaker_threshold = threshold.max(1);
1173 self.route_bind_breaker_cooldown = cooldown;
1174 self
1175 }
1176
1177 #[cfg(test)]
1178 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1179 self.health_probe_timeout = timeout;
1180 self
1181 }
1182
1183 #[cfg(test)]
1184 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1185 self.control_dispatch_delay = Some(delay);
1186 self
1187 }
1188
1189 #[cfg(test)]
1193 pub(crate) fn with_route_bind_relay_sent_hold(
1194 mut self,
1195 hold: Arc<tokio::sync::Barrier>,
1196 ) -> Self {
1197 self.route_bind_relay_sent_hold = Some(hold);
1198 self
1199 }
1200
1201 #[cfg(test)]
1206 pub(crate) fn with_hello_registry_to_forwarding_hold(
1207 mut self,
1208 hold: Arc<std::sync::Barrier>,
1209 ) -> Self {
1210 self.hello_registry_to_forwarding_hold = Some(hold);
1211 self
1212 }
1213
1214 #[cfg(test)]
1215 fn hold_between_registry_and_forwarding(&self) {
1216 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1217 hold.wait();
1218 hold.wait();
1219 }
1220 }
1221
1222 pub fn with_process_liveness(
1223 mut self,
1224 process_liveness: Arc<dyn ModuleProcessLiveness>,
1225 ) -> Self {
1226 self.process_liveness = Some(process_liveness);
1227 self
1228 }
1229
1230 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1231 self.supervisor = supervisor;
1232 self.supervisor.set_activity(self.activity.clone());
1233 self
1234 }
1235
1236 pub fn with_daemon_provenance(
1237 mut self,
1238 pid: u32,
1239 started_at_ms: u64,
1240 executable_path: Option<PathBuf>,
1241 build_git_sha: Option<String>,
1242 build_lock_digest: Option<String>,
1243 ) -> Self {
1244 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1245 let process_start_time = process_start_time(pid);
1246 self.daemon_provenance = DaemonProvenanceFacts {
1247 build: DaemonBuildProvenance {
1248 build_git_sha,
1249 build_lock_digest,
1250 },
1251 pid: Some(pid),
1252 started_at_ms: Some(started_at_ms),
1253 start_clock: None,
1254 executable_path,
1255 executable_identity,
1256 process_start_time,
1257 probe: ExecutableIdentityProbe::default(),
1258 };
1259 self
1260 }
1261
1262 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1263 self.daemon_provenance.start_clock = Some(clock);
1264 self
1265 }
1266
1267 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1269 self.scopes
1270 .write()
1271 .unwrap_or_else(|poisoned| poisoned.into_inner())
1272 .set_wall_clock(Arc::new(clock));
1273 self
1274 }
1275
1276 #[cfg(test)]
1277 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1278 self.provenance_probe_override = Some(result);
1279 self
1280 }
1281
1282 pub fn with_capability_config(
1286 self,
1287 modules: impl IntoIterator<Item = (String, bool)>,
1288 reserved_capabilities: BTreeMap<String, String>,
1289 ) -> Self {
1290 self.capability_evaluator
1291 .configure(modules, reserved_capabilities);
1292 self
1293 }
1294
1295 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1299 self.daemon_shutdown = flag;
1300 self
1301 }
1302
1303 pub fn with_supervisor_rescan(
1304 mut self,
1305 supervisor: Supervisor,
1306 config_path: impl Into<PathBuf>,
1307 configured_port: Option<u16>,
1308 ) -> Self {
1309 self.rescan = Some(SupervisorRescanContext {
1310 supervisor,
1311 config_path: config_path.into(),
1312 configured_port,
1313 storage_config: self.storage_config.clone(),
1314 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1315 admission_facts_targets: self.admission_facts_targets.clone(),
1316 scope_authority_owners: self.scope_authority_owners.clone(),
1317 privileged_operator_prompt: self.privileged_operator_prompt,
1318 generation: Arc::default(),
1319 });
1320 self
1321 }
1322
1323 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1324 self.connected_clients = connected_clients;
1325 self
1326 }
1327
1328 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1331 self.local_endpoint_unavailable = reason;
1332 self
1333 }
1334
1335 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1336 Arc::clone(&self.forwarding)
1337 }
1338
1339 pub(crate) fn counters(&self) -> DaemonCounters {
1340 self.counters.clone()
1341 }
1342
1343 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1346 tokio::spawn(async move {
1347 loop {
1348 self.capability_evaluator
1349 .wait_for_change_or_deadline()
1350 .await;
1351 self.refresh_capability_requirements();
1352 }
1353 });
1354 }
1355
1356 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1362 tokio::spawn(async move {
1363 loop {
1364 let has_deadlines = self
1365 .scopes
1366 .read()
1367 .unwrap_or_else(|poisoned| poisoned.into_inner())
1368 .has_deadlines();
1369 if !has_deadlines {
1370 self.scope_deadline_added.notified().await;
1371 continue;
1372 }
1373 tokio::time::sleep(Duration::from_secs(1)).await;
1374 if let Err(error) = self.sweep_expired_scopes() {
1375 warn!(%error, "scope expiry sweep failed");
1376 }
1377 }
1378 });
1379 }
1380
1381 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1382 let mut table = self
1383 .scopes
1384 .write()
1385 .unwrap_or_else(|poisoned| poisoned.into_inner());
1386 let swept = table.sweep_expired();
1387 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1388 drop(table);
1389 self.close_scope_drained_routes(drained);
1390 Ok(())
1391 }
1392
1393 fn publish_scope_changes(
1394 &self,
1395 changes: &[crate::scopes::ScopeTagChange],
1396 expired: &[crate::scopes::ScopeExpired],
1397 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1398 if changes.is_empty() {
1399 return Ok(Vec::new());
1400 }
1401 let drained = self
1402 .forwarding
1403 .publish_scope_changes(changes)
1404 .map_err(RouterError::Forwarding)?;
1405 let mut counts = HashMap::new();
1406 for route in &drained {
1407 *counts
1408 .entry((
1409 route.scope.owner.as_str(),
1410 route.scope.scope_ref.as_str(),
1411 route.scope.tag.scope_epoch,
1412 ))
1413 .or_insert(0usize) += 1;
1414 }
1415 for scope in expired {
1416 let routes_closed = counts
1417 .get(&(
1418 scope.owner.as_str(),
1419 scope.scope_ref.as_str(),
1420 scope.scope_epoch,
1421 ))
1422 .copied()
1423 .unwrap_or(0);
1424 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1425 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1426 routes_closed, "scope expired");
1427 }
1428 Ok(drained)
1429 }
1430
1431 fn runtime_capability_snapshot(
1432 &self,
1433 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1434 let runtime = self
1435 .supervisor
1436 .list()
1437 .into_iter()
1438 .map(|module| {
1439 let status = module.status().map_err(|err| {
1440 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1441 })?;
1442 Ok(RuntimeModule {
1443 module_id: status.module_id,
1444 state: status.state,
1445 enabled: status.enabled,
1446 })
1447 })
1448 .collect::<Result<Vec<_>, RouterError>>()?;
1449 let registrations = self.registry.list_registered_modules().map_err(|err| {
1455 RouterError::backend(
1456 0,
1457 0,
1458 format!("failed to list capability registrations: {err}"),
1459 )
1460 })?;
1461 let registrations = registrations
1462 .into_iter()
1463 .map(|registration| RegisteredModule {
1464 module_id: registration.manifest.module_id,
1465 module_version: registration.manifest.module_version,
1466 capabilities: registration.manifest.capabilities,
1467 })
1468 .collect();
1469 Ok((runtime, registrations))
1470 }
1471
1472 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1479 let cached_registration = RegisteredModule {
1480 module_id: registration.manifest.module_id.clone(),
1481 module_version: registration.manifest.module_version.clone(),
1482 capabilities: registration.manifest.capabilities.clone(),
1483 };
1484 let change = self.capability_evaluator.record_hello(&cached_registration);
1485 log_manifest_change(&cached_registration.module_id, &change);
1486 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1487 self.enforce_capability_denies();
1488 }
1489 self.refresh_capability_requirements();
1490 }
1491
1492 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1496 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1497 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1498 self.supervisor.set_swap_promotion_observer(observer);
1499 }
1500
1501 pub fn refresh_capability_requirements(&self) {
1502 match self.runtime_capability_snapshot() {
1503 Ok((runtime, registrations)) => {
1504 log_requirement_events(
1505 self.capability_evaluator
1506 .evaluate_now(&runtime, ®istrations),
1507 &self.daemon_shutdown,
1508 );
1509 }
1510 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1511 }
1512 }
1513
1514 fn enforce_capability_denies(&self) {
1518 let registrations = match self.registry.list_registered_modules() {
1519 Ok(snapshot) => snapshot,
1520 Err(err) => {
1521 warn!(error = %err, "failed to read registrations for capability deny census");
1522 return;
1523 }
1524 };
1525 let manifests = registrations
1526 .into_iter()
1527 .map(|registration| {
1528 (
1529 registration.manifest.module_id.clone(),
1530 registration.manifest,
1531 )
1532 })
1533 .collect::<BTreeMap<_, _>>();
1534 let census = match self.forwarding.route_census(None) {
1535 Ok(census) => census,
1536 Err(err) => {
1537 warn!(error = %err, "failed to read route census for capability deny enforcement");
1538 return;
1539 }
1540 };
1541
1542 for (target_module_id, routes) in census {
1543 let Some(target_manifest) = manifests.get(&target_module_id) else {
1544 continue;
1545 };
1546 let mut closed_routes = Vec::new();
1547 let mut goodbyes = Vec::new();
1548 for route in routes {
1549 let Principal::Reserved {
1550 module_id: opening_module_id,
1551 } = &route.principal
1552 else {
1553 continue;
1554 };
1555 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1556 continue;
1557 };
1558 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1559 continue;
1560 };
1561
1562 match self.forwarding.release_client_route(
1563 route.goodbye_target.connection_id,
1564 route.goodbye_target.channel,
1565 route.goodbye_target.epoch,
1566 ) {
1567 Ok(RouteRelease::Removed(module_goodbye)) => {
1568 warn!(
1569 opening_module_id,
1570 target_module_id,
1571 capability,
1572 "force-closing route because an attested capability deny edge now matches"
1573 );
1574 goodbyes.push(module_goodbye);
1575 goodbyes.push(route.goodbye_target.clone());
1576 closed_routes.push(route);
1577 }
1578 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1579 Err(err) => warn!(
1580 opening_module_id,
1581 target_module_id,
1582 capability,
1583 error = %err,
1584 "failed to force-close capability-denied route"
1585 ),
1586 }
1587 }
1588
1589 if closed_routes.is_empty() {
1590 continue;
1591 }
1592 send_route_control_pushes(
1593 &self.forwarding,
1594 closed_routes,
1595 ClientControlPush::RouteClosed {
1596 module_id: target_module_id,
1597 channels: Vec::new(),
1598 reason: RouteCloseReason::CapabilityDenied,
1599 drained: false,
1600 abandoned: 0,
1601 excluded_subscriptions: 0,
1602 terminal: Some(false),
1603 },
1604 );
1605 self.emit_route_goodbyes(goodbyes);
1608 }
1609 }
1610
1611 fn not_ready_reason(
1617 &self,
1618 registration: &crate::registry::ModuleRegistration,
1619 ) -> Option<NotReadyReason> {
1620 if !registration.ready {
1621 return Some(NotReadyReason {
1622 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1623 capability: None,
1624 });
1625 }
1626 self.first_unprovided_required_capability(registration)
1627 .map(|capability| NotReadyReason {
1628 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1629 capability: Some(capability),
1630 })
1631 }
1632
1633 fn first_unprovided_required_capability(
1651 &self,
1652 registration: &crate::registry::ModuleRegistration,
1653 ) -> Option<String> {
1654 let required = registration
1655 .manifest
1656 .capabilities
1657 .iter()
1658 .flat_map(|declarations| declarations.requires.iter())
1659 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1660 .map(|requirement| requirement.capability.as_str())
1661 .collect::<BTreeSet<_>>();
1662 if required.is_empty() {
1663 return None;
1664 }
1665 let module_id = registration.manifest.module_id.as_str();
1666 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1667 if required
1668 .iter()
1669 .any(|capability| verdict(capability).is_none())
1670 {
1671 self.refresh_capability_requirements();
1672 }
1673 required
1674 .into_iter()
1675 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1676 .map(str::to_string)
1677 }
1678
1679 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1680 self.capability_evaluator
1681 .statuses()
1682 .into_iter()
1683 .map(capability_requirement_status)
1684 .collect()
1685 }
1686
1687 fn deregister_connection(
1694 &self,
1695 connection_id: ConnectionId,
1696 reason: RegistrationEndReason,
1697 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1698 self.registry
1699 .deregister_connection_with_reason(connection_id, reason)
1700 }
1701
1702 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1703 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1704 return None;
1705 }
1706 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1707 parse_client_control_request(&frame.body)
1708 else {
1709 return None;
1710 };
1711 Some(target_module_id(&target).to_string())
1712 }
1713
1714 pub(crate) fn route_open_capacity_refusal(
1715 &self,
1716 ctx: &RouteCtx,
1717 frame: &Frame,
1718 target_module_id: &str,
1719 in_flight: usize,
1720 limit: usize,
1721 ) -> Result<Frame, RouterError> {
1722 let caller = self.route_open_caller_from_frame(frame);
1725 self.route_open_admission_refusal_frame(
1726 ctx,
1727 &caller,
1728 frame,
1729 target_module_id,
1730 "open_admission_full",
1731 (in_flight, limit),
1732 format!(
1733 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1734 ),
1735 )
1736 }
1737
1738 fn route_open_target_capacity_refusal(
1739 &self,
1740 ctx: &RouteCtx,
1741 caller: &RouteOpenCaller,
1742 frame: &Frame,
1743 target_module_id: &str,
1744 in_flight: usize,
1745 ) -> Result<Frame, RouterError> {
1746 self.route_open_admission_refusal_frame(
1747 ctx,
1748 caller,
1749 frame,
1750 target_module_id,
1751 "target_binds_full",
1752 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1753 format!(
1754 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1755 ),
1756 )
1757 }
1758
1759 #[allow(clippy::too_many_arguments)]
1767 fn route_open_admission_refusal_frame(
1768 &self,
1769 ctx: &RouteCtx,
1770 caller: &RouteOpenCaller,
1771 frame: &Frame,
1772 target_module_id: &str,
1773 reason: &'static str,
1774 (in_flight, limit): (usize, usize),
1775 message: impl Into<String>,
1776 ) -> Result<Frame, RouterError> {
1777 let code = error_codes::TARGET_UNAVAILABLE;
1778 self.counters.increment_route_open_refused(code);
1779 info!(
1780 target: "control",
1781 code,
1782 reason,
1783 module_id = ?target_module_id,
1784 connection_id = ctx.connection_id.get(),
1785 principal = %caller.principal,
1786 scope = %caller.scope,
1787 harness = %caller.harness,
1788 in_flight,
1789 limit,
1790 "route.open refused"
1791 );
1792 control_error_frame(frame, code, message.into())
1793 }
1794
1795 #[cfg(test)]
1801 pub fn handle_control(
1802 &self,
1803 connection_id: ConnectionId,
1804 frame: Frame,
1805 ) -> Result<Vec<Frame>, RouterError> {
1806 match frame.header.ty {
1807 FrameType::Ping => Ok(vec![pong(&frame)?]),
1808 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1809 FrameType::Goodbye => self.handle_goodbye(connection_id),
1810 ty => Ok(vec![control_error_frame(
1811 &frame,
1812 "unsupported_control_frame",
1813 format!("unsupported channel-0 frame {ty:?}"),
1814 )?]),
1815 }
1816 }
1817
1818 pub async fn handle_control_frame(
1819 &self,
1820 ctx: &RouteCtx,
1821 frame: Frame,
1822 ) -> Result<Vec<Frame>, RouterError> {
1823 self.handle_control_frame_timed(ctx, frame, None).await
1824 }
1825
1826 #[cfg(any(test, feature = "test-support"))]
1827 pub fn with_test_operator_app_policy(
1828 mut self,
1829 policy: crate::operator_apps::TestOperatorAppPolicy,
1830 ) -> Self {
1831 self.operator_app_policy = policy.into();
1832 self
1833 }
1834
1835 #[cfg(all(windows, any(test, feature = "test-support")))]
1839 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
1840 self.windows_verify = Some(verify);
1841 self
1842 }
1843
1844 #[cfg(all(windows, any(test, feature = "test-support")))]
1846 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
1847 self.windows_flash = Some(flash);
1848 self
1849 }
1850
1851 #[cfg(all(windows, any(test, feature = "test-support")))]
1855 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
1856 self.windows_track = Some(track);
1857 self
1858 }
1859
1860 pub(crate) fn record_captured_peer(
1861 &self,
1862 connection: ConnectionId,
1863 peer: subc_os::local_peer::CapturedPeer,
1864 ) {
1865 self.record_observed_peer(connection, peer.observation().clone());
1866 #[cfg(windows)]
1867 if let Some(stamp) = self.stamp_windows_launch(&peer) {
1868 self.operator_apps
1869 .lock()
1870 .unwrap_or_else(|p| p.into_inner())
1871 .insert(connection, stamp);
1872 }
1873 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1874 self.operator_apps
1875 .lock()
1876 .unwrap_or_else(|p| p.into_inner())
1877 .insert(connection, stamp);
1878 }
1879 }
1880
1881 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
1882 self.local_connections
1883 .lock()
1884 .unwrap_or_else(|p| p.into_inner())
1885 .insert(connection);
1886 }
1887
1888 fn connection_operator_app(
1891 &self,
1892 stamp: Option<&crate::operator_apps::OperatorAppStamp>,
1893 ) -> Option<subc_protocol::peer::OperatorApp> {
1894 stamp
1895 .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
1896 .map(|stamp| stamp.app.clone())
1897 }
1898
1899 fn handle_connection_describe(
1900 &self,
1901 ctx: &RouteCtx,
1902 frame: Frame,
1903 ) -> Result<Vec<Frame>, RouterError> {
1904 if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
1905 return Ok(vec![control_error_frame(
1906 &frame,
1907 "invalid_control_body",
1908 "malformed connection.describe body",
1909 )?]);
1910 }
1911 let local = self
1912 .local_connections
1913 .lock()
1914 .unwrap_or_else(|p| p.into_inner())
1915 .contains(&ctx.connection_id);
1916 let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
1917 let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
1918 Ok(vec![control_response_body_frame(
1919 &frame,
1920 &subc_control::ConnectionDescribeReply::new(local, operator_app),
1921 ops::CONNECTION_DESCRIBE,
1922 )?])
1923 }
1924
1925 pub(crate) fn record_observed_peer(
1926 &self,
1927 connection: ConnectionId,
1928 peer: subc_os::local_peer::ObservedPeer,
1929 ) {
1930 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1931 let peer = match peer {
1932 OsPeer::Unobserved => ObservedPeer::unobserved(),
1933 OsPeer::Observed(peer) => {
1934 let observed = match peer.user {
1935 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1936 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1937 };
1938 observed
1939 .with_exe(
1940 peer.exe
1941 .and_then(|path| path.into_os_string().into_string().ok()),
1942 )
1943 .with_signing_id(peer.signing_id)
1944 }
1945 };
1946 self.observed_peers
1947 .lock()
1948 .unwrap_or_else(|p| p.into_inner())
1949 .insert(connection, peer);
1950 }
1951
1952 pub(crate) fn record_caller_endpoints(
1953 &self,
1954 connection: ConnectionId,
1955 endpoints: Option<(SocketAddr, SocketAddr)>,
1956 ) {
1957 let mut callers = self
1958 .caller_endpoints
1959 .lock()
1960 .unwrap_or_else(|p| p.into_inner());
1961 callers.remove(&connection);
1962 if let Some(endpoints) = endpoints {
1963 callers.insert(connection, endpoints);
1964 }
1965 }
1966
1967 #[cfg(test)]
1968 pub(crate) fn with_caller_lookup_for_test(
1969 mut self,
1970 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1971 ) -> Self {
1972 self.caller_lookup = Some(Arc::new(lookup));
1973 self
1974 }
1975
1976 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1977 let endpoints = self
1978 .caller_endpoints
1979 .lock()
1980 .unwrap_or_else(|p| p.into_inner())
1981 .get(&connection)
1982 .copied();
1983 #[cfg(test)]
1984 let lookup = self.caller_lookup.clone();
1985 tokio::task::spawn_blocking(move || {
1986 #[cfg(test)]
1987 if let Some(lookup) = lookup {
1988 return lookup(endpoints);
1989 }
1990 endpoints
1991 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1992 .unwrap_or_default()
1993 })
1994 .await
1995 .unwrap_or_default()
1996 }
1997
1998 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1999 frame.header.channel == 0
2000 && frame.header.ty == FrameType::Request
2001 && self
2002 .forwarding
2003 .module_endpoint_for_connection(ctx.connection_id)
2004 .is_ok_and(|endpoint| endpoint.is_none())
2005 && parse_client_control_request(&frame.body)
2006 .ok()
2007 .is_some_and(|request| privileged_operation(&request).is_some())
2008 }
2009
2010 pub(crate) async fn handle_control_frame_timed(
2011 &self,
2012 ctx: &RouteCtx,
2013 frame: Frame,
2014 dispatch_started_at: Option<StdInstant>,
2015 ) -> Result<Vec<Frame>, RouterError> {
2016 match frame.header.ty {
2017 FrameType::Ping => Ok(vec![pong(&frame)?]),
2018 FrameType::Hello => {
2019 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
2020 }
2021 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2022 FrameType::Cancel => {
2023 if self
2026 .forwarding
2027 .operator_confirms()
2028 .cancel(ctx.connection_id, frame.header.corr)
2029 || self
2030 .supervisor
2031 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2032 || self.cancel_activity_subscription(ctx, &frame)?
2033 {
2034 Ok(Vec::new())
2035 } else {
2036 Ok(vec![control_error_frame(
2037 &frame,
2038 "unknown_subscription",
2039 "no subscription has this correlation id",
2040 )?])
2041 }
2042 }
2043 FrameType::Request => {
2044 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2047 if op
2048 .as_ref()
2049 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2050 {
2051 return self.handle_activity_record(ctx, frame).await;
2052 }
2053 if op.as_ref().is_some_and(|probe| {
2054 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2055 }) {
2056 return self.handle_activity_read(ctx, frame).await;
2057 }
2058 if op
2059 .as_ref()
2060 .is_some_and(|probe| probe.op == "operator.confirm")
2061 {
2062 return self.handle_operator_confirm(ctx, frame);
2063 }
2064 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2065 return self
2066 .observe_control_wait(
2067 ctx,
2068 None,
2069 ops::APP_OPEN,
2070 self.handle_app_open(ctx, frame),
2071 )
2072 .await;
2073 }
2074 if op
2075 .as_ref()
2076 .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2077 {
2078 return self.handle_connection_describe(ctx, frame);
2079 }
2080 if op
2081 .as_ref()
2082 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2083 {
2084 return self.handle_scope_served_by(ctx.connection_id, frame);
2085 }
2086 if self
2087 .forwarding
2088 .module_endpoint_for_connection(ctx.connection_id)
2089 .map_err(RouterError::Forwarding)?
2090 .is_some()
2091 {
2092 if !is_known_module_request_op(&frame.body) {
2093 return Ok(vec![control_error_frame(
2094 &frame,
2095 "unsupported_control_frame",
2096 "module-originated channel-0 REQUEST is not supported",
2097 )?]);
2098 }
2099 let request = match parse_module_control_request_from_module(&frame.body) {
2100 Ok(request) => request,
2101 Err((err, ControlRequestBodyError::UnknownOp)) => {
2102 return Ok(vec![control_error_frame(
2103 &frame,
2104 "unsupported_control_frame",
2105 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2106 )?])
2107 }
2108 Err((err, ControlRequestBodyError::InvalidBody)) => {
2109 return Ok(vec![control_error_frame(
2110 &frame,
2111 "invalid_control_body",
2112 format!("malformed module control body: {err}"),
2113 )?])
2114 }
2115 };
2116 let op = module_control_request_op(&request);
2117 let corr = frame.header.corr;
2118 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2119 let result =
2120 self.handle_module_control_request(ctx.connection_id, frame, request);
2121 log_slow_control_dispatch(
2122 dispatch_started_at,
2123 op,
2124 ctx.connection_id,
2125 corr,
2126 None,
2127 );
2128 return result;
2129 }
2130
2131 if is_known_module_request_op(&frame.body) {
2132 return Ok(vec![control_error_frame(
2133 &frame,
2134 "not_registered",
2135 "catalog.update requires an active module registration owned by this connection",
2136 )?]);
2137 }
2138
2139 let request = match parse_client_control_request(&frame.body) {
2140 Ok(request) => request,
2141 Err((err, ControlRequestBodyError::UnknownOp)) => {
2142 return Ok(vec![control_error_frame(
2143 &frame,
2144 "unknown_control_op",
2145 format!("unknown client control op: {err}"),
2146 )?])
2147 }
2148 Err((err, ControlRequestBodyError::InvalidBody)) => {
2149 return Ok(vec![control_error_frame(
2150 &frame,
2151 "invalid_control_body",
2152 format!("malformed client control body: {err}"),
2153 )?])
2154 }
2155 };
2156 let op = client_control_request_op(&request);
2157 let corr = frame.header.corr;
2158 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2159 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2160 #[cfg(test)]
2161 if let Some(delay) = self.control_dispatch_delay {
2162 tokio::time::sleep(delay).await;
2163 }
2164 let result = self
2165 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2166 .await;
2167 if let Some(timings) = route_open_timings.as_mut() {
2168 timings.finish();
2169 }
2170 log_slow_control_dispatch(
2171 dispatch_started_at,
2172 op,
2173 ctx.connection_id,
2174 corr,
2175 route_open_timings.as_ref(),
2176 );
2177 result
2178 }
2179 FrameType::Push => {
2180 let Some(endpoint) = self
2181 .forwarding
2182 .module_endpoint_for_connection(ctx.connection_id)
2183 .map_err(RouterError::Forwarding)?
2184 else {
2185 return Ok(vec![control_error_frame(
2186 &frame,
2187 "unsupported_control_frame",
2188 "client-originated channel-0 PUSH is not supported",
2189 )?]);
2190 };
2191 self.handle_status_update(endpoint, frame)
2192 }
2193 FrameType::Response | FrameType::Error
2194 if self
2195 .forwarding
2196 .module_endpoint_for_connection(ctx.connection_id)
2197 .map_err(RouterError::Forwarding)?
2198 .is_some() =>
2199 {
2200 self.handle_module_relay_response(ctx.connection_id, frame)
2201 }
2202 ty => Ok(vec![control_error_frame(
2203 &frame,
2204 "unsupported_control_frame",
2205 format!("unsupported channel-0 frame {ty:?}"),
2206 )?]),
2207 }
2208 }
2209
2210 pub fn cleanup_connection(
2211 &self,
2212 connection_id: ConnectionId,
2213 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2214 self.local_connections
2215 .lock()
2216 .unwrap_or_else(|p| p.into_inner())
2217 .remove(&connection_id);
2218 self.caller_endpoints
2219 .lock()
2220 .unwrap_or_else(|p| p.into_inner())
2221 .remove(&connection_id);
2222 self.build_class_warned
2223 .lock()
2224 .unwrap_or_else(|p| p.into_inner())
2225 .remove(&connection_id);
2226 self.observed_peers
2227 .lock()
2228 .unwrap_or_else(|p| p.into_inner())
2229 .remove(&connection_id);
2230 self.operator_apps
2231 .lock()
2232 .unwrap_or_else(|p| p.into_inner())
2233 .remove(&connection_id);
2234 self.forwarding
2235 .operator_confirms()
2236 .connection_closed(connection_id);
2237 self.cleanup_connection_with_end_reason(
2238 connection_id,
2239 RegistrationEndReason::ConnectionClosed,
2240 )
2241 }
2242
2243 fn cleanup_connection_with_end_reason(
2244 &self,
2245 connection_id: ConnectionId,
2246 requested_reason: RegistrationEndReason,
2247 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2248 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2249 self.registry
2250 .get_module_by_connection(connection_id)?
2251 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2252 .and_then(|module| module.registration_end_reason().ok().flatten())
2253 .unwrap_or(requested_reason)
2254 } else {
2255 requested_reason
2256 };
2257 let crash_closed = self
2258 .registry
2259 .get_module_by_connection(connection_id)?
2260 .and_then(|registration| {
2261 self.forwarding
2262 .module_endpoint_for_connection(connection_id)
2263 .ok()
2264 .flatten()
2265 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2266 .map(|routes| (registration.manifest.module_id, routes))
2267 });
2268 let crash_closed = crash_closed.map(|(module_id, routes)| {
2269 let terminal = match self.supervisor.get(&module_id) {
2270 None => false,
2271 Some(module) => match module.will_recover_after_connection_loss() {
2272 Ok(will_recover) => !will_recover,
2273 Err(err) => {
2274 warn!(
2275 %module_id,
2276 error = %err,
2277 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2278 );
2279 false
2280 }
2281 },
2282 };
2283 let (reason, drained) = match self
2287 .forwarding
2288 .daemon_drain_result_for_connection(connection_id)
2289 {
2290 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2291 Ok(None) => (RouteCloseReason::Crash, false),
2292 Err(err) => {
2293 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2294 (RouteCloseReason::Crash, false)
2295 }
2296 };
2297 (module_id, routes, reason, drained, terminal)
2298 });
2299 let registrations = self.deregister_connection(connection_id, end_reason);
2300 let cleanup = if crash_closed.is_some() {
2301 self.forwarding.cleanup_connection_counted(connection_id)
2302 } else {
2303 self.forwarding
2309 .cleanup_connection(connection_id)
2310 .map(|released| crate::forwarding::ConnectionCleanup {
2311 released,
2312 abandoned_relays: 0,
2313 })
2314 };
2315 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2320 let abandoned = cleanup
2321 .as_ref()
2322 .map_or(0, |cleanup| cleanup.abandoned_relays);
2323 send_route_control_pushes(
2324 &self.forwarding,
2325 routes,
2326 ClientControlPush::RouteClosed {
2327 module_id,
2328 channels: Vec::new(),
2329 reason,
2330 drained,
2331 abandoned,
2332 excluded_subscriptions: 0,
2333 terminal: Some(terminal),
2334 },
2335 );
2336 }
2337 if let Ok(cleanup) = cleanup {
2338 self.emit_route_goodbyes(cleanup.released);
2339 }
2340 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2344 crate::supervise::notify_registration_release();
2345 self.capability_evaluator.wake_deadline_loop();
2346 self.refresh_capability_requirements();
2347 }
2348 self.supervisor.remove_spawn_subscribers(connection_id);
2349 self.activity_streams
2350 .lock()
2351 .unwrap_or_else(|p| p.into_inner())
2352 .retain(|(id, _), task| {
2353 if *id == connection_id {
2354 task.abort();
2355 false
2356 } else {
2357 true
2358 }
2359 });
2360 self.hello_launch_nonces
2363 .lock()
2364 .unwrap_or_else(|poisoned| poisoned.into_inner())
2365 .forget(connection_id);
2366 self.scopes
2367 .write()
2368 .unwrap_or_else(|poisoned| poisoned.into_inner())
2369 .release_connection(connection_id);
2370 registrations
2371 }
2372
2373 pub(crate) fn handle_route_goodbye(
2374 &self,
2375 connection_id: ConnectionId,
2376 route_channel: u16,
2377 route_epoch: u32,
2378 ) -> Result<bool, RouterError> {
2379 debug!(
2380 connection_id = connection_id.get(),
2381 route_channel, route_epoch, "handling route GOODBYE"
2382 );
2383 let RouteRelease::Removed(released_route) = self
2384 .forwarding
2385 .release_client_route(connection_id, route_channel, route_epoch)
2386 .map_err(RouterError::Forwarding)?
2387 else {
2388 return Ok(false);
2389 };
2390 self.emit_route_goodbyes(vec![released_route]);
2391 Ok(true)
2392 }
2393
2394 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2395 for released in released_routes {
2396 let frame = match Frame::build_with_version(
2397 released.negotiated_ver,
2398 FrameType::Goodbye,
2399 control_flags(),
2400 released.channel,
2401 released.epoch,
2402 0,
2403 Vec::new(),
2404 ) {
2405 Ok(frame) => frame,
2406 Err(err) => {
2407 warn!(
2408 route_channel = released.channel,
2409 error = %err,
2410 "failed to build route GOODBYE frame"
2411 );
2412 continue;
2413 }
2414 };
2415 if !released.close_on_delivery_failure() {
2416 crate::forwarding::send_module_route_goodbye(
2417 &self.counters,
2418 &released.sink,
2419 frame,
2420 released.module_id.as_deref(),
2421 "client route released",
2422 );
2423 continue;
2424 }
2425 if let Err(err) = released.sink.try_send(frame) {
2426 warn!(
2427 target_connection_id = released.connection_id.get(),
2428 route_channel = released.channel,
2429 error = %err,
2430 "route GOODBYE was not delivered to client; closing target connection"
2431 );
2432 if self
2433 .forwarding
2434 .escalate_client_delivery_failure(
2435 released.connection_id,
2436 released.channel,
2437 released.epoch,
2438 CloseReason::new(
2439 "route_goodbye_delivery_failed",
2440 format!(
2441 "failed to enqueue route GOODBYE for channel {}: {err}",
2442 released.channel
2443 ),
2444 ),
2445 crate::forwarding::UndeliveredFrame {
2446 module_id: released.module_id.as_deref(),
2447 sink: &released.sink,
2448 },
2449 )
2450 .unwrap_or(false)
2451 {
2452 self.counters.increment_goodbye_relay_client_failed();
2453 }
2454 }
2455 }
2456 }
2457
2458 fn send_abandoned_route_bind_goodbye(
2470 &self,
2471 module_sink: &crate::FrameSink,
2472 negotiated_ver: u8,
2473 module_channel: u16,
2474 module_epoch: u32,
2475 ) {
2476 let frame = match Frame::build_with_version(
2477 negotiated_ver,
2478 FrameType::Goodbye,
2479 control_flags(),
2480 module_channel,
2481 module_epoch,
2482 0,
2483 Vec::new(),
2484 ) {
2485 Ok(frame) => frame,
2486 Err(err) => {
2487 warn!(
2488 route_channel = module_channel,
2489 error = %err,
2490 "failed to build GOODBYE for abandoned route.bind"
2491 );
2492 return;
2493 }
2494 };
2495 crate::forwarding::send_module_route_goodbye(
2496 &self.counters,
2497 module_sink,
2498 frame,
2499 None,
2500 "abandoned route.bind",
2501 );
2502 }
2503
2504 fn handle_hello(
2505 &self,
2506 connection_id: ConnectionId,
2507 sink: Option<crate::FrameSink>,
2508 frame: Frame,
2509 ) -> Result<Vec<Frame>, RouterError> {
2510 debug!(
2511 connection_id = connection_id.get(),
2512 corr = frame.header.corr,
2513 "handling HELLO"
2514 );
2515 if self
2519 .registry
2520 .get_module_by_connection(connection_id)
2521 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2522 .is_some()
2523 {
2524 return Ok(vec![control_error_frame(
2525 &frame,
2526 "invalid_hello",
2527 "connection is already registered as a module",
2528 )?]);
2529 }
2530 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2531 Ok(value) => value,
2532 Err(err) => {
2533 return Ok(vec![control_error_frame(
2534 &frame,
2535 "invalid_hello",
2536 format!("malformed HELLO body: {err}"),
2537 )?])
2538 }
2539 };
2540 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2541 return Ok(vec![control_error_frame(
2542 &frame,
2543 "invalid_capability_grammar",
2544 err.to_string(),
2545 )?]);
2546 }
2547 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2548 return Ok(vec![control_error_frame(
2549 &frame,
2550 "invalid_manifest",
2551 err.to_string(),
2552 )?]);
2553 }
2554 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2555 return Ok(vec![control_error_frame(
2556 &frame,
2557 "invalid_event_declaration",
2558 err.to_string(),
2559 )?]);
2560 }
2561 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2562 return Ok(vec![control_error_frame(
2563 &frame,
2564 "invalid_manifest",
2565 err.to_string(),
2566 )?]);
2567 }
2568 if let Some(provenance) = hello_value
2569 .get("manifest")
2570 .and_then(|manifest| manifest.get("provenance"))
2571 {
2572 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2573 return Ok(vec![control_error_frame(
2574 &frame,
2575 "invalid_manifest",
2576 format!("malformed manifest provenance: {err}"),
2577 )?]);
2578 }
2579 }
2580 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2581 let module_id = hello_value["manifest"]["module_id"]
2582 .as_str()
2583 .unwrap_or("<unknown>");
2584 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2585 }
2586 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2587 let module_id = hello_value["manifest"]["module_id"]
2588 .as_str()
2589 .unwrap_or("<unknown>");
2590 warn!(module_id, %reason, "dropping flow_operations from HELLO");
2591 }
2592 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2593 let module_id = hello_value["manifest"]["module_id"]
2594 .as_str()
2595 .unwrap_or("<unknown>");
2596 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2597 }
2598 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2599 Ok(hello) => hello,
2600 Err(err) => {
2601 return Ok(vec![control_error_frame(
2602 &frame,
2603 "invalid_hello",
2604 format!("malformed HELLO body: {err}"),
2605 )?])
2606 }
2607 };
2608
2609 if hello.protocol_ver != hello.manifest.protocol_ver {
2610 return Ok(vec![control_error_frame(
2611 &frame,
2612 "invalid_manifest",
2613 format!(
2614 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2615 hello.protocol_ver, hello.manifest.protocol_ver
2616 ),
2617 )?]);
2618 }
2619
2620 if hello.manifest.module_id.trim().is_empty() {
2621 return Ok(vec![control_error_frame(
2622 &frame,
2623 "invalid_manifest",
2624 "manifest module_id must not be empty",
2625 )?]);
2626 }
2627
2628 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2629 Ok(negotiated_ver) => negotiated_ver,
2630 Err(message) => {
2631 return Ok(vec![control_error_frame(
2632 &frame,
2633 "version_unsupported",
2634 message,
2635 )?])
2636 }
2637 };
2638
2639 let swap_admission = self
2647 .supervisor
2648 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2649 if swap_admission == SwapHelloAdmission::Refused {
2650 warn!(
2651 module_id = %hello.manifest.module_id,
2652 connection_id = connection_id.get(),
2653 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2654 );
2655 return Ok(vec![control_error_frame(
2656 &frame,
2657 "swap_token_invalid",
2658 format!(
2659 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2660 hello.manifest.module_id
2661 ),
2662 )?]);
2663 }
2664 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2665
2666 let nonce_verified = if swap_candidate {
2671 true
2672 } else {
2673 match self.supervisor.reserved_hello_verification(
2674 &hello.manifest.module_id,
2675 hello.launch_nonce.as_deref(),
2676 ) {
2677 Ok(verified) => verified,
2678 Err(rejection) => {
2679 let message = match rejection {
2680 ReservedHelloRejection::Exact { module_id } => format!(
2681 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2682 ),
2683 ReservedHelloRejection::Prefix {
2684 prefix,
2685 owner_module_id,
2686 } => format!(
2687 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2688 hello.manifest.module_id
2689 ),
2690 };
2691 return Ok(vec![control_error_frame(
2692 &frame,
2693 "reserved_module",
2694 message,
2695 )?]);
2696 }
2697 }
2698 };
2699
2700 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2701 &hello.manifest.module_id,
2702 hello.manifest.capabilities.as_ref(),
2703 );
2704 if let Some(refusal) = reserved_capability_refusals.first() {
2705 let capability = refusal.capability.clone();
2706 let bound_module = refusal.claimants[0].clone();
2707 log_duplicate_claim_events(reserved_capability_refusals);
2708 return Ok(vec![control_error_frame(
2709 &frame,
2710 "reserved_capability",
2711 format!(
2712 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2713 capability, bound_module, hello.manifest.module_id
2714 ),
2715 )?]);
2716 }
2717
2718 if self
2721 .forwarding
2722 .connection_has_client_routes(connection_id)
2723 .map_err(RouterError::Forwarding)?
2724 {
2725 return Ok(vec![control_error_frame(
2726 &frame,
2727 "invalid_hello",
2728 "connection has open client routes and cannot also register as a module",
2729 )?]);
2730 }
2731
2732 self.hello_launch_nonces
2737 .lock()
2738 .unwrap_or_else(|poisoned| poisoned.into_inner())
2739 .record(connection_id, hello.launch_nonce.as_deref());
2740 let control_ops = effective_module_control_ops(hello.control_ops);
2741 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2744 if swap_candidate {
2745 return self.register_swap_candidate(
2746 connection_id,
2747 sink,
2748 &frame,
2749 hello.manifest,
2750 negotiated_ver,
2751 control_ops,
2752 hello_ack,
2753 );
2754 }
2755 let registration = match self
2762 .registry
2763 .register_unpublished_verified_with_control_ops(
2764 hello.manifest,
2765 negotiated_ver,
2766 connection_id,
2767 control_ops,
2768 nonce_verified,
2769 ) {
2770 Ok(registration) => registration,
2771 Err(RegistryError::DuplicateModuleId { module_id }) => {
2772 return Ok(vec![control_error_frame(
2773 &frame,
2774 "duplicate_module_id",
2775 format!(
2776 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2777 ),
2778 )?])
2779 }
2780 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2781 return Ok(vec![control_error_frame(
2782 &frame,
2783 "invalid_module_id",
2784 err.to_string(),
2785 )?])
2786 }
2787 Err(err) => {
2788 return Ok(vec![control_error_frame(
2789 &frame,
2790 "registry_error",
2791 err.to_string(),
2792 )?])
2793 }
2794 };
2795 #[cfg(test)]
2796 self.hold_between_registry_and_forwarding();
2797
2798 let reply = if let Some(sink) = sink {
2799 let concurrency = manifest_concurrency(®istration.manifest);
2815 if let Err(err) = self.forwarding.register_module_connection_acked(
2816 connection_id,
2817 registration.manifest.module_id.clone(),
2818 negotiated_ver,
2819 concurrency,
2820 sink,
2821 hello_ack,
2822 ) {
2823 if matches!(
2828 self.deregister_connection(
2829 connection_id,
2830 RegistrationEndReason::RegistrationFailed,
2831 ),
2832 Ok(r) if !r.is_empty()
2833 ) {
2834 crate::supervise::notify_registration_release();
2835 }
2836 return Ok(vec![control_error_frame(
2837 &frame,
2838 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2839 "invalid_hello"
2840 } else {
2841 forwarding_error_code(&err)
2842 },
2843 err.to_string(),
2844 )?]);
2845 }
2846 Vec::new()
2847 } else {
2848 vec![hello_ack]
2851 };
2852
2853 if !self
2858 .registry
2859 .publish_registration(connection_id)
2860 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2861 {
2862 debug!(
2863 module_id = %registration.manifest.module_id,
2864 connection_id = connection_id.get(),
2865 "registration was removed before it could be published"
2866 );
2867 }
2868
2869 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2878 info!(
2879 module_id = %registration.manifest.module_id,
2880 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2881 );
2882 }
2883
2884 self.apply_registration_capabilities(®istration);
2885
2886 self.supervisor
2887 .record_activity_registration(®istration.manifest.module_id);
2888
2889 info!(
2890 module_id = %registration.manifest.module_id,
2891 module_version = %registration.manifest.module_version,
2892 negotiated_ver,
2893 routable_provider = manifest_provides_routable_role(®istration.manifest),
2894 connection_id = connection_id.get(),
2895 "module registered"
2896 );
2897
2898 Ok(reply)
2899 }
2900
2901 #[allow(clippy::too_many_arguments)]
2918 fn register_swap_candidate(
2919 &self,
2920 connection_id: ConnectionId,
2921 sink: Option<crate::FrameSink>,
2922 frame: &Frame,
2923 manifest: ModuleManifest,
2924 negotiated_ver: u8,
2925 control_ops: Vec<String>,
2926 hello_ack: Frame,
2927 ) -> Result<Vec<Frame>, RouterError> {
2928 let module_id = manifest.module_id.clone();
2929 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2930 manifest,
2931 negotiated_ver,
2932 connection_id,
2933 control_ops,
2934 ) {
2935 Ok(registration) => registration,
2936 Err(RegistryError::DuplicateModuleId { module_id }) => {
2937 return Ok(vec![control_error_frame(
2938 frame,
2939 "duplicate_module_id",
2940 format!(
2941 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2942 ),
2943 )?])
2944 }
2945 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2946 return Ok(vec![control_error_frame(
2947 frame,
2948 "invalid_module_id",
2949 err.to_string(),
2950 )?])
2951 }
2952 Err(err) => {
2953 return Ok(vec![control_error_frame(
2954 frame,
2955 "registry_error",
2956 err.to_string(),
2957 )?])
2958 }
2959 };
2960 #[cfg(test)]
2961 self.hold_between_registry_and_forwarding();
2962 let reply = if let Some(sink) = sink {
2963 let concurrency = manifest_concurrency(®istration.manifest);
2967 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2968 connection_id,
2969 module_id.clone(),
2970 negotiated_ver,
2971 concurrency,
2972 sink,
2973 hello_ack,
2974 ) {
2975 if matches!(
2976 self.deregister_connection(
2977 connection_id,
2978 RegistrationEndReason::RegistrationFailed,
2979 ),
2980 Ok(r) if !r.is_empty()
2981 ) {
2982 crate::supervise::notify_registration_release();
2983 }
2984 return Ok(vec![control_error_frame(
2985 frame,
2986 forwarding_error_code(&err),
2987 err.to_string(),
2988 )?]);
2989 }
2990 Vec::new()
2991 } else {
2992 vec![hello_ack]
2993 };
2994 if !self
2995 .registry
2996 .publish_registration(connection_id)
2997 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2998 {
2999 debug!(
3000 module_id = %module_id,
3001 connection_id = connection_id.get(),
3002 "swap candidate registration was removed before it could be published"
3003 );
3004 }
3005 self.supervisor.mark_swap_candidate_admitted(&module_id);
3006 info!(
3007 module_id = %module_id,
3008 module_version = %registration.manifest.module_version,
3009 negotiated_ver,
3010 ready = registration.ready,
3011 connection_id = connection_id.get(),
3012 "swap candidate registered; not routable until cutover"
3013 );
3014 Ok(reply)
3015 }
3016
3017 fn build_hello_ack(
3018 &self,
3019 frame: &Frame,
3020 negotiated_ver: u8,
3021 module_id: &str,
3022 ) -> Result<Frame, RouterError> {
3023 let ack = ModuleHelloAckBody {
3024 negotiated_ver,
3025 subc_ops: module_subc_ops(),
3026 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
3027 storage: self
3028 .storage_config
3029 .as_ref()
3030 .map(|cfg| cfg.descriptor_for(module_id)),
3031 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3032 };
3033 let body = serde_json::to_vec(&ack).map_err(|err| {
3034 RouterError::backend(
3035 0,
3036 frame.header.corr,
3037 format!("failed to encode HELLO_ACK: {err}"),
3038 )
3039 })?;
3040
3041 Frame::build_with_version(
3042 negotiated_ver,
3043 FrameType::HelloAck,
3044 control_flags(),
3045 0,
3046 0,
3047 frame.header.corr,
3048 body,
3049 )
3050 .map_err(RouterError::FrameBuild)
3051 }
3052
3053 async fn handle_client_control_request(
3054 &self,
3055 ctx: &RouteCtx,
3056 frame: Frame,
3057 request: ClientControlRequest,
3058 route_open_timings: &mut Option<RouteOpenTimings>,
3059 ) -> Result<Vec<Frame>, RouterError> {
3060 if let Some(operation) = privileged_operation(&request) {
3061 let caller = self.resolve_caller(ctx.connection_id).await;
3062 let op = client_control_request_op(&request);
3063 let text = prompt_text(operation, &caller);
3064 match self.privileged_operator_prompt {
3065 PrivilegedOperatorPrompt::Log => {
3066 info!(target: "subc_daemon::privileged_prompt", operation = op,
3067 target = ?operation, caller = %caller.description(), summary = %text,
3068 "privileged request would prompt the operator");
3069 }
3070 PrivilegedOperatorPrompt::Enforce => {
3071 let key =
3072 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
3073 let stamp = self
3074 .operator_apps
3075 .lock()
3076 .unwrap_or_else(|p| p.into_inner())
3077 .get(&ctx.connection_id)
3078 .cloned();
3079 let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
3080 &self.operator_app_policy,
3081 &self.launch_table,
3082 stamp.as_ref(),
3083 )
3084 .then(|| self.shortcut_context(stamp, None, None));
3085 let outcome = self
3086 .forwarding
3087 .operator_confirms()
3088 .approve_privileged_with_shortcut(
3089 ctx,
3090 &frame,
3091 op,
3092 text,
3093 caller.description(),
3094 key,
3095 shortcut,
3096 )
3097 .await;
3098 if outcome.code != "confirmed" {
3099 return Ok(vec![outcome.frame(&frame)]);
3100 }
3101 }
3102 }
3103 }
3104 match request {
3105 ClientControlRequest::DaemonShutdownIntent { intent } => {
3106 if intent.as_str() != Some("restart") {
3107 return Ok(vec![control_error_frame(
3108 &frame,
3109 "invalid_params",
3110 "intent must be 'restart'",
3111 )?]);
3112 }
3113 let response = ClientControlResponse::DaemonShutdownIntent {
3114 intent: "restart".to_owned(),
3115 expires_at_ms: self.supervisor.announce_daemon_restart(),
3116 };
3117 Ok(vec![control_response_body_frame(
3118 &frame,
3119 &response,
3120 "ClientControlResponse::DaemonShutdownIntent",
3121 )?])
3122 }
3123 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
3124 ClientControlRequest::CatalogList { module_id } => {
3125 self.handle_catalog_list(frame, module_id)
3126 }
3127 ClientControlRequest::RouteOpen {
3128 target,
3129 identity,
3130 consumer_identity,
3131 consumer_capabilities,
3132 role_versions,
3133 busy_refusals,
3134 admission_facts,
3135 scope,
3136 ..
3137 } => {
3138 self.handle_route_open(
3139 ctx,
3140 frame,
3141 RouteOpenRequest {
3142 target,
3143 identity,
3144 consumer_identity,
3145 consumer_capabilities,
3146 role_versions,
3147 busy_refusals,
3148 admission_facts,
3149 scope,
3150 },
3151 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
3152 )
3153 .await
3154 }
3155 ClientControlRequest::RoutePoll {
3156 route_channel,
3157 route_epoch,
3158 kind,
3159 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
3160 ClientControlRequest::SupervisorList {} => {
3161 let handler = self.clone();
3163 let sink = ctx.egress.clone();
3164 tokio::spawn(
3165 async move {
3166 let result = handler.handle_supervisor_list(frame.clone()).await;
3167 let replies = match result {
3168 Ok(replies) => replies,
3169 Err(error) => {
3170 control_error_frame(&frame, "backend_error", error.to_string())
3171 .into_iter()
3172 .collect()
3173 }
3174 };
3175 for reply in replies {
3176 if sink.send(reply).await.is_err() {
3177 break;
3178 }
3179 }
3180 }
3181 .with_current_subscriber(),
3182 );
3183 Ok(Vec::new())
3184 }
3185 ClientControlRequest::SupervisorSpawnSnapshot {} => {
3186 self.handle_supervisor_spawn_snapshot(frame)
3187 }
3188 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
3189 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
3190 }
3191 ClientControlRequest::SupervisorRestart {
3192 module_id,
3193 drain_timeout_ms,
3194 } => {
3195 self.log_supervisor_request_received(
3196 ctx,
3197 frame.header.corr,
3198 ops::SUPERVISOR_RESTART,
3199 Some(&module_id),
3200 None,
3201 )?;
3202 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3203 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
3204 .await
3205 }
3206 ClientControlRequest::SupervisorSwap {
3207 module_id,
3208 ready_timeout_ms,
3209 } => {
3210 self.log_supervisor_request_received(
3211 ctx,
3212 frame.header.corr,
3213 ops::SUPERVISOR_SWAP,
3214 Some(&module_id),
3215 None,
3216 )?;
3217 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3218 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3219 .await
3220 }
3221 ClientControlRequest::SupervisorReload { module_id } => {
3222 self.log_supervisor_request_received(
3223 ctx,
3224 frame.header.corr,
3225 ops::SUPERVISOR_RELOAD,
3226 Some(&module_id),
3227 None,
3228 )?;
3229 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3230 self.handle_supervisor_reload(frame, module_id, requester)
3231 .await
3232 }
3233 ClientControlRequest::SupervisorRescan { preview } => {
3234 if !preview {
3235 self.log_supervisor_request_received(
3236 ctx,
3237 frame.header.corr,
3238 ops::SUPERVISOR_RESCAN,
3239 None,
3240 None,
3241 )?;
3242 }
3243 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3244 let operation = self.handle_supervisor_rescan(
3245 frame,
3246 if preview {
3247 RescanMode::Preview
3248 } else {
3249 RescanMode::Execute
3250 },
3251 requester,
3252 );
3253 self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
3254 .await
3255 }
3256 ClientControlRequest::SupervisorRescanChecked {
3257 preview,
3258 candidate,
3259 expect,
3260 } => {
3261 self.observe_control_wait(
3262 ctx,
3263 None,
3264 ops::SUPERVISOR_RESCAN,
3265 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
3266 )
3267 .await
3268 }
3269 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3270 self.log_supervisor_request_received(
3271 ctx,
3272 frame.header.corr,
3273 ops::SUPERVISOR_RELEASE_RESERVED,
3274 Some(&module_id),
3275 None,
3276 )?;
3277 self.handle_supervisor_release_reserved(frame, module_id)
3278 .await
3279 }
3280 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3281 self.log_supervisor_request_received(
3282 ctx,
3283 frame.header.corr,
3284 ops::SUPERVISOR_SET_ENABLED,
3285 Some(&module_id),
3286 Some(enabled),
3287 )?;
3288 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3289 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3290 .await
3291 }
3292 ClientControlRequest::SupervisorHealthProbe { module_id } => {
3293 self.handle_supervisor_health_probe(ctx, frame, module_id)
3294 .await
3295 }
3296 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3297 ClientControlRequest::SupervisorRoutes { module_id } => {
3298 self.handle_supervisor_routes(frame, module_id)
3299 }
3300 ClientControlRequest::SupervisorProvenance { module_id } => {
3301 self.handle_supervisor_provenance(ctx, frame, module_id)
3302 .await
3303 }
3304 ClientControlRequest::SupervisorStderrTail {
3305 module_id,
3306 max_lines,
3307 max_bytes,
3308 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3309 ClientControlRequest::SupervisorTerminals { module_id } => {
3310 self.handle_supervisor_terminals(ctx, frame, module_id)
3311 .await
3312 }
3313 }
3314 }
3315
3316 fn log_supervisor_request_received(
3317 &self,
3318 ctx: &RouteCtx,
3319 corr: u64,
3320 op: &'static str,
3321 module_id: Option<&str>,
3322 enabled: Option<bool>,
3323 ) -> Result<(), RouterError> {
3324 let caller = self.supervisor_requester(ctx, corr)?;
3325 let caller = principal_label(&caller);
3326 info!(
3329 target: "control",
3330 op,
3331 module_id,
3332 enabled,
3333 connection_id = ctx.connection_id.get(),
3334 caller = %caller,
3335 "supervisor request received"
3336 );
3337 Ok(())
3338 }
3339
3340 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3341 Ok(self
3342 .registry
3343 .get_module_by_connection(ctx.connection_id)
3344 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3345 .map(|registration| Principal::Reserved {
3346 module_id: registration.manifest.module_id,
3347 })
3348 .unwrap_or(Principal::Direct))
3349 }
3350
3351 fn handle_module_control_request(
3352 &self,
3353 connection_id: ConnectionId,
3354 frame: Frame,
3355 request: ModuleControlRequestFromModule,
3356 ) -> Result<Vec<Frame>, RouterError> {
3357 match request {
3358 ModuleControlRequestFromModule::CatalogUpdate {
3359 provides,
3360 capabilities,
3361 ready,
3362 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3363 ModuleControlRequestFromModule::LiveRoots {} => {
3364 let registered = self
3365 .registry
3366 .get_module_by_connection(connection_id)
3367 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3368 let Some(registration) = registered else {
3369 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3370 };
3371 let response = self
3372 .forwarding
3373 .live_roots(®istration.manifest.module_id)
3374 .map_err(RouterError::Forwarding)?;
3375 Ok(vec![control_response_body_frame(
3376 &frame,
3377 &response,
3378 "ModuleControlResponseToModule::LiveRoots",
3379 )?])
3380 }
3381 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3382 self.handle_scope_sync(connection_id, frame, generation, scopes)
3383 }
3384 ModuleControlRequestFromModule::ScopeApply {
3385 generation,
3386 upsert,
3387 end,
3388 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3389 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3390 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3391 }
3392 }
3393 }
3394
3395 fn handle_scope_served_by(
3396 &self,
3397 connection_id: ConnectionId,
3398 frame: Frame,
3399 ) -> Result<Vec<Frame>, RouterError> {
3400 let registered = self
3401 .registry
3402 .get_module_by_connection(connection_id)
3403 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3404 let Some(registered) = registered else {
3405 return Ok(vec![control_error_frame(
3406 &frame,
3407 "not_registered",
3408 "scope.served_by requires an active module registration owned by this connection",
3409 )?]);
3410 };
3411 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3412 {
3413 Ok(request) => request,
3414 Err(err) => {
3415 return Ok(vec![control_error_frame(
3416 &frame,
3417 "invalid_control_body",
3418 format!("malformed scope.served_by body: {err}"),
3419 )?])
3420 }
3421 };
3422 let answer = self
3423 .scopes
3424 .read()
3425 .unwrap_or_else(|poisoned| poisoned.into_inner())
3426 .served_by(
3427 ®istered.manifest.module_id,
3428 &request.scope_ref,
3429 request.scope_epoch,
3430 &request.module_id,
3431 );
3432 let reply = subc_control::ScopeServedByReply::new(
3433 answer,
3434 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3435 );
3436 Ok(vec![control_response_body_frame(
3437 &frame,
3438 &reply,
3439 "ScopeServedByReply",
3440 )?])
3441 }
3442
3443 fn handle_operator_confirm(
3444 &self,
3445 ctx: &RouteCtx,
3446 frame: Frame,
3447 ) -> Result<Vec<Frame>, RouterError> {
3448 use crate::operator_confirm::{audit, Outcome};
3449 let registration = self
3450 .registry
3451 .get_module_by_connection(ctx.connection_id)
3452 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3453 let Some(registration) = registration else {
3454 let outcome = Outcome::refusal("not_registered");
3455 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3456 return Ok(vec![outcome.frame(&frame)]);
3457 };
3458 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3459 Ok(request) => request,
3460 Err(_) => {
3461 let outcome = Outcome::refusal("invalid_control_body");
3462 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3463 return Ok(vec![outcome.frame(&frame)]);
3464 }
3465 };
3466 let declared = matches!(
3467 request.kind,
3468 Some(subc_protocol::session::OperatorConfirmKind::Operation)
3469 ) && request.operation.as_ref().is_some_and(|operation| {
3470 registration
3471 .manifest
3472 .confirm_operations
3473 .as_ref()
3474 .is_some_and(|operations| operations.contains(operation))
3475 });
3476 let module_id = registration.manifest.module_id;
3477 let nonce = self
3481 .hello_launch_nonces
3482 .lock()
3483 .unwrap_or_else(|p| p.into_inner())
3484 .nonce(ctx.connection_id)
3485 .map(str::to_owned);
3486 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3487 self.supervisor
3488 .spawned_consumer_authorized(&module_id, nonce)
3489 });
3490 let confirms = self.forwarding.operator_confirms();
3491 self.forwarding
3492 .with_operator_route(
3493 ctx.connection_id,
3494 request.route_channel,
3495 request.route_epoch,
3496 |binding| {
3497 let shortcut = binding
3498 .filter(|binding| {
3499 nonce_proven
3500 && crate::operator_confirm::valid_summary(&request.summary)
3501 && declared
3502 && crate::operator_confirm::ShortcutContext::route_eligible(
3503 &self.operator_app_policy,
3504 &self.launch_table,
3505 binding,
3506 )
3507 })
3508 .map(|binding| {
3509 let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
3510 &self.operator_app_policy,
3511 binding,
3512 );
3513 let stamp = if phone.is_some() {
3514 None
3515 } else {
3516 binding.operator_app.clone()
3517 };
3518 self.shortcut_context(
3519 stamp,
3520 phone,
3521 Some((request.route_channel, request.route_epoch)),
3522 )
3523 });
3524 confirms.admit_with_shortcut(
3525 ctx,
3526 frame,
3527 module_id,
3528 nonce_proven,
3529 request,
3530 binding,
3531 shortcut,
3532 )
3533 },
3534 )
3535 .map_err(RouterError::Forwarding)
3536 }
3537
3538 fn shortcut_context(
3539 &self,
3540 stamp: Option<crate::operator_apps::OperatorAppStamp>,
3541 phone_fingerprint: Option<String>,
3542 route: Option<(u16, u32)>,
3543 ) -> crate::operator_confirm::ShortcutContext {
3544 crate::operator_confirm::ShortcutContext {
3545 ledger: self.activity.clone(),
3546 forwarding: self.forwarding.clone(),
3547 launches: self.launch_table.clone(),
3548 stamp,
3549 phone_fingerprint,
3550 route,
3551 connections: self.operator_apps.clone(),
3552 #[cfg(test)]
3553 hooks: Default::default(),
3554 }
3555 }
3556
3557 #[cfg(any(test, feature = "test-support"))]
3559 pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
3560 self.operator_app_policy.phone_enabled = enabled;
3561 self
3562 }
3563
3564 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3567 let mut scopes = ScopeRecords::from(scopes);
3568 for (index, record) in scopes.records.iter_mut().enumerate() {
3569 if let Some(refusal) = Self::canonicalize_workdir(record) {
3570 scopes.refusals.insert(index, refusal);
3571 }
3572 }
3573 scopes
3574 }
3575
3576 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3577 let workdir = record.workdir.as_mut()?;
3578 let path = std::path::Path::new(workdir);
3579 if !path.is_absolute() {
3580 return Some((
3581 error_codes::INVALID_WORKDIR,
3582 "workdir must be absolute".to_string(),
3583 ));
3584 }
3585 match ProjectRootId::from_path_allowing_missing(path) {
3586 Ok(root) => match root.as_path().to_str() {
3587 Some(canonical) => {
3588 *workdir = canonical.to_string();
3589 None
3590 }
3591 None => Some((
3592 error_codes::INVALID_WORKDIR,
3593 "canonical workdir is not UTF-8".to_string(),
3594 )),
3595 },
3596 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3597 }
3598 }
3599
3600 fn handle_scope_sync(
3604 &self,
3605 connection_id: ConnectionId,
3606 frame: Frame,
3607 generation: u64,
3608 scopes: Vec<ScopeRecord>,
3609 ) -> Result<Vec<Frame>, RouterError> {
3610 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3611 }
3612
3613 fn handle_scope_change(
3614 &self,
3615 connection_id: ConnectionId,
3616 frame: Frame,
3617 generation: u64,
3618 scopes: Vec<ScopeRecord>,
3619 end: Option<Vec<ScopeEnd>>,
3620 ) -> Result<Vec<Frame>, RouterError> {
3621 let is_apply = end.is_some();
3622 let Some(registration) = self
3623 .registry
3624 .get_module_by_connection(connection_id)
3625 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3626 else {
3627 return Ok(vec![control_error_frame(
3628 &frame,
3629 "not_registered",
3630 "scope.sync and scope.apply require an active module registration owned by this connection",
3631 )?]);
3632 };
3633 let owner = registration.manifest.module_id;
3634 let scopes = Self::prepare_scope_records(scopes);
3635 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3636 let is_current_launch = |connection: ConnectionId| {
3637 self.hello_launch_nonces
3638 .lock()
3639 .unwrap_or_else(|poisoned| poisoned.into_inner())
3640 .presented(connection, current_nonce.as_deref())
3641 };
3642 let mut table = self
3647 .scopes
3648 .write()
3649 .unwrap_or_else(|poisoned| poisoned.into_inner());
3650 let outcome = match end {
3651 Some(end) => table.apply(
3652 &owner,
3653 connection_id,
3654 is_current_launch,
3655 generation,
3656 scopes,
3657 end,
3658 ),
3659 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3660 };
3661 let drained = match &outcome {
3662 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3663 Err(_) => Vec::new(),
3664 };
3665 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3666 drop(table);
3667 if has_deadlines {
3668 self.scope_deadline_added.notify_one();
3669 }
3670 match outcome {
3671 Ok(applied) => {
3672 let counts = ScopeOutcomeCounts::of(&applied.results);
3673 info!(
3674 owner = %owner,
3675 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3676 generation,
3677 records = applied.results.len(),
3678 created = counts.created,
3679 replaced = counts.replaced,
3680 updated = counts.updated,
3681 unchanged = counts.unchanged,
3682 refused = counts.refused,
3683 ended = applied.ended.len(),
3684 tag_changes = applied.tag_changes.len(),
3685 routes_closed = drained.len(),
3686 "scope change accepted"
3687 );
3688 for refused in applied
3694 .results
3695 .iter()
3696 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3697 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3698 {
3699 warn!(
3700 owner = %owner,
3701 generation,
3702 scope_ref = %refused.scope_ref,
3703 scope_epoch = refused.scope_epoch,
3704 code = refused.code.as_deref().unwrap_or(""),
3705 "scope record refused"
3706 );
3707 }
3708 self.close_scope_drained_routes(drained);
3709 let response = if is_apply {
3710 ModuleControlResponseToModule::ScopeApply {
3711 generation,
3712 results: applied.results,
3713 end_results: applied.end_results,
3714 ended: applied.ended,
3715 }
3716 } else {
3717 ModuleControlResponseToModule::ScopeSync {
3718 generation,
3719 results: applied.results,
3720 ended: applied.ended,
3721 }
3722 };
3723 Ok(vec![control_response_body_frame(
3724 &frame,
3725 &response,
3726 "scope change response",
3727 )?])
3728 }
3729 Err(refusal) => {
3730 info!(
3731 owner = %owner,
3732 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3733 generation,
3734 code = refusal.code,
3735 "scope change refused"
3736 );
3737 Ok(vec![control_error_frame(
3738 &frame,
3739 refusal.code,
3740 refusal.message,
3741 )?])
3742 }
3743 }
3744 }
3745
3746 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3752 if drained.is_empty() {
3753 return;
3754 }
3755 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3756 BTreeMap::new();
3757 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3758 for route in drained {
3759 warn!(
3760 module_id = %route.module_id,
3761 reason = ?route.reason,
3762 client_connection_id = route.client.connection_id.get(),
3763 route_channel = route.client.channel,
3764 "closing route because its scope changed"
3765 );
3766 pushes
3767 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3768 .or_insert_with(|| (route.reason, Vec::new()))
3769 .1
3770 .push(EndpointRoute {
3771 goodbye_target: route.client.clone(),
3772 principal: Principal::Unverified,
3773 bound_at: Instant::now(),
3774 draining: false,
3775 drain_reason: None,
3776 });
3777 goodbyes.push(route.module);
3778 goodbyes.push(route.client);
3779 }
3780 for ((module_id, _), (reason, routes)) in pushes {
3781 send_route_control_pushes(
3782 &self.forwarding,
3783 routes,
3784 ClientControlPush::RouteClosed {
3785 module_id,
3786 channels: Vec::new(),
3787 reason,
3788 drained: false,
3789 abandoned: 0,
3790 excluded_subscriptions: 0,
3791 terminal: Some(false),
3792 },
3793 );
3794 }
3795 self.emit_route_goodbyes(goodbyes);
3796 }
3797
3798 fn handle_scope_describe(
3801 &self,
3802 connection_id: ConnectionId,
3803 frame: Frame,
3804 owner: Principal,
3805 scope_ref: String,
3806 ) -> Result<Vec<Frame>, RouterError> {
3807 let registered = self
3808 .registry
3809 .get_module_by_connection(connection_id)
3810 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3811 let Some(registered) = registered else {
3812 return Ok(vec![control_error_frame(
3813 &frame,
3814 "not_registered",
3815 "scope.describe requires an active module registration owned by this connection",
3816 )?]);
3817 };
3818 let mut description = self
3819 .scopes
3820 .read()
3821 .unwrap_or_else(|poisoned| poisoned.into_inner())
3822 .describe(&owner, &scope_ref);
3823 if !registered
3826 .manifest
3827 .capabilities
3828 .as_ref()
3829 .is_some_and(|caps| {
3830 caps.provides
3831 .iter()
3832 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3833 })
3834 {
3835 if let Some(stamp) = description.stamp.as_mut() {
3836 stamp.attributes.on_behalf_of_agent = None;
3837 }
3838 }
3839 if !registered
3840 .manifest
3841 .capabilities
3842 .as_ref()
3843 .is_some_and(|caps| {
3844 caps.provides
3845 .iter()
3846 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3847 })
3848 {
3849 if let Some(stamp) = description.stamp.as_mut() {
3850 stamp.attributes.flow_version = None;
3851 stamp.attributes.flow_code_hash = None;
3852 stamp.attributes.flow_install_id = None;
3853 }
3854 }
3855 let owner_configured = match &owner {
3856 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3861 _ => false,
3862 };
3863 let response = ModuleControlResponseToModule::ScopeDescribe {
3864 status: description.status,
3865 scope_epoch: description.scope_epoch,
3866 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3867 owner_synced: description.owner_synced,
3868 owner_configured,
3869 scope: description.stamp,
3870 };
3871 Ok(vec![control_response_body_frame(
3872 &frame,
3873 &response,
3874 "ModuleControlResponseToModule::ScopeDescribe",
3875 )?])
3876 }
3877
3878 fn handle_catalog_update(
3879 &self,
3880 connection_id: ConnectionId,
3881 frame: Frame,
3882 provides: Vec<ProviderRole>,
3883 capabilities: Option<CapabilityDeclarations>,
3884 ready: Option<bool>,
3885 ) -> Result<Vec<Frame>, RouterError> {
3886 self.refresh_capability_requirements();
3887 let Some(registration) = self
3888 .registry
3889 .get_module_by_connection(connection_id)
3890 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3891 else {
3892 return Ok(vec![control_error_frame(
3893 &frame,
3894 "not_registered",
3895 "catalog.update requires an active module registration owned by this connection",
3896 )?]);
3897 };
3898
3899 if let Some(message) =
3900 catalog_update_frozen_field_message(®istration.manifest, &provides)
3901 {
3902 return Ok(vec![control_error_frame(
3903 &frame,
3904 "catalog_update_frozen_field",
3905 message,
3906 )?]);
3907 }
3908
3909 let mut candidate = registration.manifest.clone();
3910 candidate.provides = provides.clone();
3911 candidate.capabilities = capabilities
3912 .clone()
3913 .or_else(|| registration.manifest.capabilities.clone());
3914 if let Some(rich_ui) = &candidate.rich_ui {
3915 if let Err(err) = rich_ui.validate() {
3916 return Ok(vec![control_error_frame(
3917 &frame,
3918 "invalid_manifest",
3919 err.to_string(),
3920 )?]);
3921 }
3922 }
3923 if let Err(err) = candidate.validate_capability_grammar() {
3924 return Ok(vec![control_error_frame(
3925 &frame,
3926 "invalid_capability_grammar",
3927 err.to_string(),
3928 )?]);
3929 }
3930
3931 let mut conflicts = self
3934 .capability_evaluator
3935 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3936 if let Some(conflict) = conflicts.first() {
3937 let message = format!(
3938 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3939 conflict.capability, conflict.claimants[0], candidate.module_id
3940 );
3941 for conflict in &mut conflicts {
3942 conflict.source = DuplicateClaimSource::CatalogUpdate;
3943 }
3944 log_duplicate_claim_events(conflicts);
3945 return Ok(vec![control_error_frame(
3946 &frame,
3947 "reserved_capability",
3948 message,
3949 )?]);
3950 }
3951
3952 let updated = self
3953 .registry
3954 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3955 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3956 if updated.is_none() {
3957 return Ok(vec![control_error_frame(
3958 &frame,
3959 "not_registered",
3960 "catalog.update requires an active module registration owned by this connection",
3961 )?]);
3962 }
3963 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3964 log_duplicate_claim_events(
3965 self.capability_evaluator
3966 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3967 );
3968 }
3969 if capability_census_trigger(
3970 registration.manifest.capabilities.as_ref(),
3971 updated
3972 .as_ref()
3973 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3974 ) {
3975 self.enforce_capability_denies();
3976 }
3977 self.refresh_capability_requirements();
3978
3979 let response = ModuleControlResponseToModule::CatalogUpdate {};
3980 control_response_body_frame(
3981 &frame,
3982 &response,
3983 "ModuleControlResponseToModule::CatalogUpdate",
3984 )
3985 .map(|frame| vec![frame])
3986 }
3987
3988 pub(crate) fn with_activity_ledger_config(
3989 mut self,
3990 run_dir: &Path,
3991 incarnation: String,
3992 config: crate::activity::ActivityConfig,
3993 ) -> Self {
3994 self.activity = Some(crate::activity::ActivityLedger::open_config(
3995 run_dir.join("activity.jsonl"),
3996 incarnation,
3997 config,
3998 ));
3999 self.supervisor.set_activity(self.activity.clone());
4000 self
4001 }
4002
4003 pub(crate) async fn configure_activity(
4004 &self,
4005 config: crate::activity::ActivityConfig,
4006 ) -> Result<(), &'static str> {
4007 match &self.activity {
4008 Some(ledger) => ledger.configure(config).await,
4009 None => Ok(()),
4010 }
4011 }
4012
4013 fn cancel_activity_subscription(
4014 &self,
4015 ctx: &RouteCtx,
4016 frame: &Frame,
4017 ) -> Result<bool, RouterError> {
4018 let task = self
4019 .activity_streams
4020 .lock()
4021 .unwrap_or_else(|p| p.into_inner())
4022 .remove(&(ctx.connection_id, frame.header.corr));
4023 let Some(task) = task else {
4024 return Ok(false);
4025 };
4026 task.abort();
4027 let end = Frame::build_with_version(
4028 response_version(frame),
4029 FrameType::StreamEnd,
4030 control_flags(),
4031 0,
4032 0,
4033 frame.header.corr,
4034 Vec::new(),
4035 )
4036 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
4037 let sink = ctx.egress.clone();
4038 tokio::spawn(async move {
4039 let _ = sink.send(end).await;
4040 });
4041 Ok(true)
4042 }
4043
4044 async fn handle_activity_read(
4045 &self,
4046 ctx: &RouteCtx,
4047 frame: Frame,
4048 ) -> Result<Vec<Frame>, RouterError> {
4049 use subc_control::{
4050 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
4051 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
4052 };
4053 #[derive(Deserialize)]
4054 struct Identity {
4055 #[serde(default)]
4056 consumer_identity: Option<ConsumerIdentity>,
4057 }
4058 let principal = match self
4062 .forwarding
4063 .module_id_for_connection(ctx.connection_id)
4064 .map_err(RouterError::Forwarding)?
4065 {
4066 Some(module_id) => Ok(Principal::Reserved { module_id }),
4067 None => {
4068 let identity: Identity = match serde_json::from_slice(&frame.body) {
4069 Ok(identity) => identity,
4070 Err(_) => {
4071 return Ok(vec![control_error_frame(
4072 &frame,
4073 ACTIVITY_MALFORMED,
4074 "malformed activity read",
4075 )?])
4076 }
4077 };
4078 self.route_open_principal(&frame, identity.consumer_identity)?
4079 }
4080 };
4081 let admitted = matches!(principal, Ok(Principal::Direct))
4082 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
4083 if !admitted {
4084 return Ok(vec![control_error_frame(
4085 &frame,
4086 ACTIVITY_READ_FORBIDDEN,
4087 "activity reads require an operator or trusted mirror principal",
4088 )?]);
4089 }
4090 let Some(ledger) = &self.activity else {
4091 return Ok(vec![control_error_frame(
4092 &frame,
4093 ACTIVITY_UNAVAILABLE,
4094 "activity ledger unavailable",
4095 )?]);
4096 };
4097 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
4098 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
4099 {
4100 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
4101 Ok(request) => ledger.query(request).await,
4102 Err(_) => Err(ACTIVITY_MALFORMED),
4103 };
4104 return match result {
4105 Ok(reply) => Ok(vec![control_response_body_frame(
4106 &frame,
4107 &reply,
4108 ops::ACTIVITY_QUERY,
4109 )?]),
4110 Err(code) => Ok(vec![control_error_frame(
4111 &frame,
4112 code,
4113 "activity query refused",
4114 )?]),
4115 };
4116 }
4117 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
4118 Ok(request) => request,
4119 Err(_) => {
4120 return Ok(vec![control_error_frame(
4121 &frame,
4122 ACTIVITY_MALFORMED,
4123 "malformed activity subscription",
4124 )?])
4125 }
4126 };
4127 let mut subscription = match ledger.subscribe(request.since).await {
4128 Ok(subscription) => subscription,
4129 Err(code) => {
4130 return Ok(vec![control_error_frame(
4131 &frame,
4132 code,
4133 "activity subscription refused",
4134 )?])
4135 }
4136 };
4137 let key = (ctx.connection_id, frame.header.corr);
4138 let sink = ctx.egress.clone();
4139 let mut streams = self
4140 .activity_streams
4141 .lock()
4142 .unwrap_or_else(|p| p.into_inner());
4143 streams.retain(|_, task| !task.is_finished());
4144 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
4145 return Ok(vec![control_error_frame(
4146 &frame,
4147 "duplicate_subscription",
4148 "activity subscription correlation id already in use",
4149 )?]);
4150 }
4151 let task = tokio::spawn(async move {
4152 loop {
4153 let entry = if let Some(entry) = subscription.replay.pop_front() {
4154 Some(entry)
4155 } else {
4156 subscription.receiver.recv().await
4157 };
4158 let Some(entry) = entry else {
4159 break;
4160 };
4161 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
4162 return;
4163 };
4164 let Ok(next) = Frame::build_with_version(
4165 response_version(&frame),
4166 FrameType::StreamData,
4167 control_flags(),
4168 0,
4169 0,
4170 frame.header.corr,
4171 body,
4172 ) else {
4173 return;
4174 };
4175 if sink.send(next).await.is_err() {
4176 return;
4177 }
4178 }
4179 if let Some(code) = subscription.terminal_code() {
4182 if let Ok(end) =
4183 control_error_frame(&frame, code, "activity subscriber fell behind")
4184 {
4185 let _ = sink.send(end).await;
4186 }
4187 }
4188 });
4189 streams.insert(key, task.abort_handle());
4190 Ok(Vec::new())
4191 }
4192
4193 async fn handle_activity_record(
4194 &self,
4195 ctx: &RouteCtx,
4196 frame: Frame,
4197 ) -> Result<Vec<Frame>, RouterError> {
4198 use subc_control::{
4199 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
4200 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
4201 };
4202 let result = async {
4203 let module = self
4204 .forwarding
4205 .module_id_for_connection(ctx.connection_id)
4206 .ok()
4207 .flatten()
4208 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
4209 let request: ActivityRecordRequest =
4210 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
4211 crate::activity::validate(&module, &request)?;
4212 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
4213 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
4214 let at_ms = std::time::SystemTime::now()
4215 .duration_since(std::time::UNIX_EPOCH)
4216 .unwrap_or_default()
4217 .as_millis() as u64;
4218 ledger.record(module, request, actor, at_ms).await
4219 }
4220 .await;
4221 match result {
4222 Ok(entry) => Ok(vec![control_response_body_frame(
4223 &frame,
4224 &ActivityRecordResponse { seq: entry.seq },
4225 "activity.record",
4226 )?]),
4227 Err(code) => Ok(vec![control_error_frame(
4228 &frame,
4229 code,
4230 "activity recording refused",
4231 )?]),
4232 }
4233 }
4234
4235 fn activity_actor(
4236 &self,
4237 connection: ConnectionId,
4238 module: &str,
4239 on: Option<&subc_control::ActivityOn>,
4240 ) -> Result<subc_control::ActivityActor, &'static str> {
4241 use crate::forwarding::{DataRoute, DataRouteState};
4242 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
4243 let principal = |id: &str| Principal::Reserved {
4244 module_id: id.to_owned(),
4245 };
4246 let mut actor = ActivityActor {
4247 principal: principal(module),
4248 scope_owner: None,
4249 agent_id: None,
4250 run_id: None,
4251 flow_id: None,
4252 project_root: None,
4253 workdir: None,
4254 project_id: None,
4255 };
4256 let scope = match on {
4257 None => return Ok(actor),
4258 Some(ActivityOn::Scope(scope)) => {
4259 let selector = subc_protocol::scope::ScopeSelector::new(
4260 principal(module),
4261 scope.scope_ref.clone(),
4262 Some(scope.epoch),
4263 );
4264 self.scopes
4265 .read()
4266 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4267 .attribute(&principal(module), module, &selector, true)
4268 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4269 }
4270 Some(ActivityOn::Route(route)) => {
4271 let DataRoute::Module(DataRouteState::Bound(binding)) = self
4272 .forwarding
4273 .lookup_data_route(connection, route.channel, route.epoch)
4274 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4275 else {
4276 return Err(ACTIVITY_ACTOR_UNKNOWN);
4277 };
4278 if binding.module_id != module {
4279 return Err(ACTIVITY_ACTOR_UNKNOWN);
4280 }
4281 actor.principal = binding.principal.clone();
4282 actor.project_root = binding
4283 .project_root
4284 .as_ref()
4285 .map(|root| root.as_path().to_path_buf());
4286 let Some(bound) = &binding.scope else {
4287 return Ok(actor);
4288 };
4289 let owner = principal(&bound.owner);
4290 let selector = subc_protocol::scope::ScopeSelector::new(
4291 owner.clone(),
4292 bound.scope_ref.clone(),
4293 Some(bound.tag.scope_epoch),
4294 );
4295 let admitted = self
4296 .scopes
4297 .read()
4298 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4299 .attribute(&owner, module, &selector, true)
4300 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4301 if admitted.tag != bound.tag {
4302 return Err(ACTIVITY_ACTOR_UNKNOWN);
4303 }
4304 admitted
4305 }
4306 };
4307 actor.scope_owner = Some(scope.stamp.owner);
4308 actor.agent_id = scope.stamp.attributes.agent_id;
4309 actor.run_id = scope.stamp.attributes.run_id;
4310 actor.flow_id = scope.stamp.attributes.flow_id;
4311 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4312 actor.project_id = scope.stamp.project_id;
4313 Ok(actor)
4314 }
4315
4316 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4317 self.refresh_capability_requirements();
4318 let mut counters = self.counters.snapshot();
4324 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4325 obj.insert(
4326 "activity_unparseable_lines".into(),
4327 ledger.read_errors().into(),
4328 );
4329 obj.insert(
4330 "activity_lifecycle_append_failures".into(),
4331 ledger.lifecycle_failures().into(),
4332 );
4333 obj.insert(
4334 "activity_started_digest_skips".into(),
4335 ledger.lifecycle_digest_skips().into(),
4336 );
4337 }
4338 if let (Ok((connections_with_routes, max)), Some(obj)) = (
4339 self.forwarding.client_route_concentration(),
4340 counters.as_object_mut(),
4341 ) {
4342 obj.insert(
4343 "client_connections_with_routes".into(),
4344 connections_with_routes.into(),
4345 );
4346 obj.insert("max_routes_on_one_connection".into(), max.into());
4347 }
4348 if let (Some(open_breakers), Some(obj)) = (
4354 self.route_bind_breakers.open_snapshot(),
4355 counters.as_object_mut(),
4356 ) {
4357 obj.insert("route_bind_breakers_open".into(), open_breakers);
4358 }
4359 if let (Some(reason), Some(obj)) = (
4364 self.local_endpoint_unavailable.clone(),
4365 counters.as_object_mut(),
4366 ) {
4367 obj.insert("local_endpoint_unavailable".into(), reason.into());
4368 }
4369 let response = ClientControlResponse::ServerDescribe {
4370 protocol_ver: PROTOCOL_VERSION,
4371 subc_ops: subc_ops(),
4372 capabilities: self.describe_capabilities(),
4373 connected_clients: self.connected_clients.count(),
4374 counters: Some(counters),
4375 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4376 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4377 capability_requirements: self.capability_requirement_statuses(),
4378 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4379 config_path: self
4380 .rescan
4381 .as_ref()
4382 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4383 };
4384 Ok(vec![control_response_body_frame(
4385 &frame,
4386 &response,
4387 "ClientControlResponse::ServerDescribe",
4388 )?])
4389 }
4390
4391 fn handle_catalog_list(
4392 &self,
4393 frame: Frame,
4394 module_id: Option<String>,
4395 ) -> Result<Vec<Frame>, RouterError> {
4396 let (generation, modules) = self
4397 .registry
4398 .list_modules_with_nonce_verification()
4399 .map_err(|err| {
4400 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4401 })?;
4402 let entries = modules
4403 .into_iter()
4404 .filter(|(registration, _)| {
4405 module_id
4406 .as_deref()
4407 .map(|wanted| registration.manifest.module_id == wanted)
4408 .unwrap_or(true)
4409 })
4410 .map(|(registration, nonce_verified)| {
4411 let not_ready = self.not_ready_reason(®istration);
4412 let roles = registration.manifest.provides;
4413 CatalogEntry::new(
4414 registration.manifest.module_id,
4415 roles,
4416 registration.control_ops,
4417 )
4418 .with_ready(not_ready.is_none())
4419 .with_not_ready(not_ready)
4420 .with_module_version(Some(registration.manifest.module_version))
4421 .with_provenance(registration.manifest.provenance)
4422 .with_capabilities(registration.manifest.capabilities)
4423 .with_self_signals(registration.manifest.self_signals)
4424 .with_events(registration.manifest.events)
4425 .with_operation_delivery(registration.manifest.operation_delivery)
4426 .with_operation_payload(registration.manifest.operation_payload)
4427 .with_flow_operations(registration.manifest.flow_operations)
4428 .with_confirm_operations(registration.manifest.confirm_operations)
4429 .with_rich_ui(registration.manifest.rich_ui)
4430 .with_role_declarations(registration.manifest.role_declarations)
4431 .with_nonce_verified(nonce_verified)
4432 })
4433 .collect();
4434 let response = ClientControlResponse::CatalogList {
4435 generation,
4436 modules: entries,
4437 subc_ops: subc_ops(),
4438 };
4439 Ok(vec![control_response_body_frame(
4440 &frame,
4441 &response,
4442 "ClientControlResponse::CatalogList",
4443 )?])
4444 }
4445
4446 fn attest_route_open_consumer(
4451 &self,
4452 consumer_identity: Option<&ConsumerIdentity>,
4453 ) -> Option<Principal> {
4454 let Some(consumer_identity) = consumer_identity else {
4455 return Some(Principal::Direct);
4456 };
4457 self.supervisor
4458 .spawned_consumer_authorized(
4459 &consumer_identity.module_id,
4460 &consumer_identity.launch_nonce,
4461 )
4462 .then(|| Principal::Reserved {
4463 module_id: consumer_identity.module_id.clone(),
4464 })
4465 }
4466
4467 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4471 let Ok(ClientControlRequest::RouteOpen {
4472 identity,
4473 consumer_identity,
4474 scope,
4475 ..
4476 }) = parse_client_control_request(&frame.body)
4477 else {
4478 return RouteOpenCaller::unknown();
4479 };
4480 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4481 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4482 }
4483
4484 fn route_open_principal(
4485 &self,
4486 frame: &Frame,
4487 consumer_identity: Option<ConsumerIdentity>,
4488 ) -> Result<Result<Principal, Frame>, RouterError> {
4489 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4490 self.route_open_principal_attested(frame, consumer_identity, attested)
4491 }
4492
4493 fn route_open_principal_attested(
4497 &self,
4498 frame: &Frame,
4499 consumer_identity: Option<ConsumerIdentity>,
4500 attested: Option<Principal>,
4501 ) -> Result<Result<Principal, Frame>, RouterError> {
4502 if let Some(principal) = attested {
4503 return Ok(Ok(principal));
4504 }
4505 let Some(consumer_identity) = consumer_identity else {
4506 return Ok(Ok(Principal::Direct));
4507 };
4508
4509 Ok(Err(control_error_frame(
4510 frame,
4511 "bad_consumer_identity",
4512 format!(
4513 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4514 consumer_identity.module_id
4515 ),
4516 )?))
4517 }
4518
4519 #[allow(clippy::too_many_arguments)]
4526 fn route_open_refusal_frame(
4527 &self,
4528 ctx: &RouteCtx,
4529 caller: &RouteOpenCaller,
4530 frame: &Frame,
4531 module_id: &str,
4532 reason: &'static str,
4533 code: &'static str,
4534 message: impl Into<String>,
4535 ) -> Result<Frame, RouterError> {
4536 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4537 control_error_frame(frame, code, message.into())
4538 }
4539
4540 #[allow(clippy::too_many_arguments)]
4549 fn route_bind_rejection(
4550 &self,
4551 ctx: &RouteCtx,
4552 caller: &RouteOpenCaller,
4553 frame: &Frame,
4554 target_module_id: &str,
4555 body: ErrorBody,
4556 daemon_authored: bool,
4557 ) -> Result<Vec<Frame>, RouterError> {
4558 if daemon_authored {
4564 let scope_code = match body.code.as_str() {
4565 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4566 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4567 _ => None,
4568 };
4569 if let Some(code) = scope_code {
4570 self.observe_route_open_refusal(
4571 ctx,
4572 caller,
4573 target_module_id,
4574 "scope_changed_before_commit",
4575 code,
4576 );
4577 return Ok(vec![control_error_body_frame(frame, body)?]);
4578 }
4579 }
4580 self.counters
4581 .increment_route_open_refused("module_rejected");
4582 info!(
4583 target: "control",
4584 code = "module_rejected",
4585 module_code = ?body.code,
4586 module_id = ?target_module_id,
4587 connection_id = ctx.connection_id.get(),
4588 principal = %caller.principal,
4589 scope = %caller.scope,
4590 harness = %caller.harness,
4591 "route.open refused"
4592 );
4593 let refusal = if daemon_authored {
4594 control_error_body_frame(frame, body)?
4595 } else {
4596 control_relayed_error_body_frame(frame, body)?
4597 };
4598 Ok(vec![refusal])
4599 }
4600
4601 #[allow(clippy::too_many_arguments)]
4619 fn route_open_breaker_refusal_frame(
4620 &self,
4621 ctx: &RouteCtx,
4622 caller: &RouteOpenCaller,
4623 frame: &Frame,
4624 module_id: &str,
4625 consecutive_timeouts: u32,
4626 retry_in: Duration,
4627 probe_in_flight: bool,
4628 ) -> Result<Frame, RouterError> {
4629 self.counters
4630 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4631 debug!(
4632 target: "control",
4633 code = "module_timeout",
4634 module_id = ?module_id,
4635 connection_id = ctx.connection_id.get(),
4636 principal = %caller.principal,
4637 scope = %caller.scope,
4638 harness = %caller.harness,
4639 consecutive_timeouts,
4640 retry_in_ms = retry_in.as_millis() as u64,
4641 probe_in_flight,
4642 "route.open refused by open bind-relay breaker"
4643 );
4644 let detail = if probe_in_flight {
4651 "one test route is already being tried; retry once it settles".to_string()
4652 } else {
4653 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4654 };
4655 control_error_frame(
4656 frame,
4657 "module_timeout",
4658 format!(
4659 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4660 timed out in a row); {detail}; its established routes are unaffected"
4661 ),
4662 )
4663 }
4664
4665 fn observe_route_open_refusal(
4678 &self,
4679 ctx: &RouteCtx,
4680 caller: &RouteOpenCaller,
4681 module_id: &str,
4682 reason: &'static str,
4683 code: &'static str,
4684 ) {
4685 self.counters.increment_route_open_refused(code);
4686 info!(
4687 target: "control",
4688 code,
4689 reason,
4690 module_id = ?module_id,
4691 connection_id = ctx.connection_id.get(),
4692 principal = %caller.principal,
4693 scope = %caller.scope,
4694 harness = %caller.harness,
4695 "route.open refused"
4696 );
4697 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4698 self.route_outages.record_not_serving(module_id, reason);
4699 }
4700 }
4701
4702 fn observe_route_open_accept(
4747 &self,
4748 ctx: &RouteCtx,
4749 module_id: &str,
4750 principal: &str,
4751 scope: &str,
4752 ) {
4753 self.route_outages.record_accepted(module_id);
4754 self.counters.increment_route_open_accepted(principal);
4755 info!(
4756 target: "control",
4757 principal,
4758 module_id,
4759 connection_id = ctx.connection_id.get(),
4760 scope = %scope,
4761 "route.open accepted"
4762 );
4763 }
4764
4765 fn supervised_absent_route_open_refusal_frame(
4766 &self,
4767 ctx: &RouteCtx,
4768 caller: &RouteOpenCaller,
4769 frame: &Frame,
4770 module_id: &str,
4771 code: &'static str,
4772 status: &crate::supervise::ModuleStatus,
4773 ) -> Result<Frame, RouterError> {
4774 self.counters.increment_route_open_refused(code);
4775 info!(
4776 target: "control",
4777 code,
4778 reason = "supervised_not_registered",
4779 module_id = ?module_id,
4780 connection_id = ctx.connection_id.get(),
4781 principal = %caller.principal,
4782 scope = %caller.scope,
4783 harness = %caller.harness,
4784 state = %status.state,
4785 enabled = status.enabled,
4786 live = status.live,
4787 "route.open refused"
4788 );
4789 self.route_outages
4793 .record_not_serving(module_id, "supervised_not_registered");
4794 let message = if let Some(pid) = status.waiting_for_previous_pid {
4795 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4796 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4797 })
4798 } else {
4799 format!(
4800 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4801 status.state, status.enabled, status.live
4802 )
4803 };
4804 control_error_frame(frame, code, message)
4805 }
4806
4807 async fn handle_route_open(
4808 &self,
4809 ctx: &RouteCtx,
4810 frame: Frame,
4811 request: RouteOpenRequest,
4812 timings: &mut RouteOpenTimings,
4813 ) -> Result<Vec<Frame>, RouterError> {
4814 let RouteOpenRequest {
4815 target,
4816 mut identity,
4817 consumer_identity,
4818 consumer_capabilities,
4819 role_versions,
4820 busy_refusals,
4821 admission_facts,
4822 scope,
4823 } = request;
4824 let target_module_id = target_module_id(&target).to_string();
4825 timings.module_id.clone_from(&target_module_id);
4826 if self
4827 .registry
4828 .get_module_by_connection(ctx.connection_id)
4829 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4830 .is_some()
4831 {
4832 return Ok(vec![control_error_frame(
4833 &frame,
4834 "invalid_request",
4835 "module connections cannot open client routes",
4836 )?]);
4837 }
4838 debug!(
4839 connection_id = ctx.connection_id.get(),
4840 corr = frame.header.corr,
4841 module_id = %target_module_id,
4842 "handling route.open"
4843 );
4844
4845 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4850 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4851
4852 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4858 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4859 self.observe_route_open_refusal(
4860 ctx,
4861 &caller,
4862 &target_module_id,
4863 "invalid_role_versions",
4864 error_codes::INVALID_REQUEST,
4865 );
4866 return Ok(vec![control_error_body_frame(
4867 &frame,
4868 ErrorBody {
4869 code: error_codes::INVALID_REQUEST.to_string(),
4870 message: error.to_string(),
4871 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4872 },
4873 )?]);
4874 }
4875
4876 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4877 if status.waiting_for_previous_pid.is_some() {
4878 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4879 ctx,
4880 &caller,
4881 &frame,
4882 &target_module_id,
4883 "module_warming",
4884 &status,
4885 )?]);
4886 }
4887 }
4888
4889 let Some(registration) = self
4908 .registry
4909 .get_module(&target_module_id)
4910 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4911 else {
4912 if let Some((status, warming)) =
4913 self.supervisor_status(&target_module_id, frame.header.corr)?
4914 {
4915 if status.protocol == ModuleProtocol::None {
4922 return Ok(vec![self.route_open_refusal_frame(
4923 ctx,
4924 &caller,
4925 &frame,
4926 &target_module_id,
4927 "protocol_none",
4928 error_codes::MODULE_NO_PROTOCOL,
4929 format!(
4930 "module_id '{target_module_id}' is declared protocol: none; \
4931 it speaks no subc wire and serves no routes"
4932 ),
4933 )?]);
4934 }
4935 let code = if warming {
4936 "module_warming"
4937 } else {
4938 "target_unavailable"
4939 };
4940 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4941 ctx,
4942 &caller,
4943 &frame,
4944 &target_module_id,
4945 code,
4946 &status,
4947 )?]);
4948 }
4949 if let Some(removed_ago_ms) =
4950 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4951 {
4952 return Ok(vec![self.route_open_refusal_frame(
4953 ctx,
4954 &caller,
4955 &frame,
4956 &target_module_id,
4957 "removed",
4958 error_codes::MODULE_REMOVED,
4959 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4960 )?]);
4961 }
4962 return Ok(vec![self.route_open_refusal_frame(
4963 ctx,
4964 &caller,
4965 &frame,
4966 &target_module_id,
4967 "not_registered",
4968 error_codes::UNKNOWN_MODULE,
4969 format!("module_id '{target_module_id}' is not registered"),
4970 )?]);
4971 };
4972
4973 if !registration.ready {
4977 self.counters
4978 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4979 info!(
4980 target: "control",
4981 code = error_codes::MODULE_WARMING,
4982 module_id = ?target_module_id,
4983 connection_id = ctx.connection_id.get(),
4984 principal = %caller.principal,
4985 scope = %caller.scope,
4986 harness = %caller.harness,
4987 reason = "declared_not_ready",
4988 "route.open refused"
4989 );
4990 self.route_outages
4993 .record_not_serving(&target_module_id, "declared_not_ready");
4994 return Ok(vec![control_error_body_frame(
4995 &frame,
4996 ErrorBody {
4997 code: error_codes::MODULE_WARMING.to_string(),
4998 message: format!(
4999 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
5000 ),
5001 detail: Some(serde_json::json!({
5002 "reason": "declared_not_ready"
5003 })),
5004 },
5005 )?]);
5006 }
5007
5008 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
5027 self.counters
5028 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
5029 info!(
5030 target: "control",
5031 code = error_codes::MODULE_WARMING,
5032 module_id = ?target_module_id,
5033 connection_id = ctx.connection_id.get(),
5034 principal = %caller.principal,
5035 scope = %caller.scope,
5036 harness = %caller.harness,
5037 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5038 capability = %capability,
5039 "route.open refused"
5040 );
5041 return Ok(vec![control_error_body_frame(
5042 &frame,
5043 ErrorBody {
5044 code: error_codes::MODULE_WARMING.to_string(),
5045 message: format!(
5046 "module_id '{target_module_id}' requires capability '{capability}', \
5047 which no registered module provides; retry"
5048 ),
5049 detail: Some(serde_json::json!({
5050 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5051 "capability": capability,
5052 })),
5053 },
5054 )?]);
5055 }
5056
5057 if !target_has_required_role(&target, ®istration.manifest.provides) {
5058 return Ok(vec![self.route_open_refusal_frame(
5059 ctx,
5060 &caller,
5061 &frame,
5062 &target_module_id,
5063 "role_not_provided",
5064 "target_unavailable",
5065 format!("module_id '{target_module_id}' does not provide the requested target"),
5066 )?]);
5067 }
5068
5069 if registration.state != ChannelState::Active {
5070 return Ok(vec![self.route_open_refusal_frame(
5071 ctx,
5072 &caller,
5073 &frame,
5074 &target_module_id,
5075 "registration_not_active",
5076 "target_unavailable",
5077 format!("module_id '{target_module_id}' is not active"),
5078 )?]);
5079 }
5080
5081 if self
5082 .forwarding
5083 .module_is_draining(&target_module_id)
5084 .map_err(RouterError::Forwarding)?
5085 {
5086 return Ok(vec![self.route_open_refusal_frame(
5087 ctx,
5088 &caller,
5089 &frame,
5090 &target_module_id,
5091 "reloading",
5092 "module_reloading",
5093 format!("module_id '{target_module_id}' is reloading"),
5094 )?]);
5095 }
5096
5097 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
5098 process_liveness.process_live(&target_module_id) == Some(false)
5099 }) {
5100 if process_liveness.process_replacing(&target_module_id) {
5107 return Ok(vec![self.route_open_refusal_frame(
5108 ctx,
5109 &caller,
5110 &frame,
5111 &target_module_id,
5112 "reloading",
5113 "module_reloading",
5114 format!("module_id '{target_module_id}' is reloading"),
5115 )?]);
5116 }
5117 return Ok(vec![self.route_open_refusal_frame(
5118 ctx,
5119 &caller,
5120 &frame,
5121 &target_module_id,
5122 "supervisor_not_live",
5123 "target_unavailable",
5124 format!("module_id '{target_module_id}' is not live"),
5125 )?]);
5126 }
5127
5128 if !self
5129 .forwarding
5130 .has_live_module_connection(&target_module_id)
5131 .map_err(RouterError::Forwarding)?
5132 {
5133 return Ok(vec![self.route_open_refusal_frame(
5134 ctx,
5135 &caller,
5136 &frame,
5137 &target_module_id,
5138 "no_forwarding_connection",
5139 "target_unavailable",
5140 format!("module_id '{target_module_id}' has no live forwarding connection"),
5141 )?]);
5142 }
5143
5144 if let Some(error) =
5145 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
5146 {
5147 self.observe_route_open_refusal(
5148 ctx,
5149 &caller,
5150 &target_module_id,
5151 "op_not_allowed",
5152 "op_not_allowed",
5153 );
5154 return Ok(vec![error]);
5155 }
5156
5157 let principal =
5158 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
5159 Ok(principal) => principal,
5160 Err(error) => {
5161 self.observe_route_open_refusal(
5162 ctx,
5163 &caller,
5164 &target_module_id,
5165 "bad_consumer_identity",
5166 "bad_consumer_identity",
5167 );
5168 return Ok(vec![error]);
5169 }
5170 };
5171 timings.principal = principal_label(&principal);
5172
5173 let build_class = identity.build_class.take().filter(|build_class| {
5176 if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
5177 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
5178 } else {
5179 if self
5180 .build_class_warned
5181 .lock()
5182 .unwrap_or_else(|p| p.into_inner())
5183 .insert(ctx.connection_id)
5184 {
5185 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
5186 "ignoring build_class from a non-Callosum opener");
5187 }
5188 false
5189 }
5190 });
5191
5192 if let Principal::Reserved {
5196 module_id: opening_module_id,
5197 } = &principal
5198 {
5199 if let Some(opening_registration) = self
5200 .registry
5201 .get_module(opening_module_id)
5202 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5203 {
5204 if let Some(capability) =
5205 denied_capability(&opening_registration.manifest, ®istration.manifest)
5206 {
5207 warn!(
5208 opening_module_id,
5209 target_module_id,
5210 capability,
5211 "refusing route.open because an attested capability deny edge matches"
5212 );
5213 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5214 &frame,
5215 &target_module_id,
5216 "capability_deny_edge",
5217 "capability_forbidden",
5218 format!(
5219 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
5220 ),
5221 )?]);
5222 }
5223 }
5224 }
5225
5226 if admission_facts.is_some() {
5227 let carrier_matches = matches!(
5228 &principal,
5229 Principal::Reserved { module_id }
5230 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
5231 );
5232 if !carrier_matches {
5233 return Ok(vec![self.route_open_refusal_frame(
5234 ctx,
5235 &caller,
5236 &frame,
5237 &target_module_id,
5238 "admission_facts_carrier_not_permitted",
5239 "admission_facts_not_permitted",
5240 "admission facts may only be carried by the configured reserved module",
5241 )?]);
5242 }
5243
5244 let target_allowed = self
5245 .admission_facts_targets
5246 .as_ref()
5247 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
5248 if !target_allowed {
5249 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5250 &frame,
5251 &target_module_id,
5252 "admission_facts_target_not_listed",
5253 "admission_facts_target_not_allowed",
5254 format!(
5255 "admission facts are not permitted for target module_id '{target_module_id}'"
5256 ),
5257 )?]);
5258 }
5259
5260 }
5264
5265 let (bound_scope, mut scope_stamp) = match &scope {
5271 None => (None, None),
5272 Some(selector) => {
5273 let owner_configured = match &selector.owner {
5274 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5280 _ => false,
5281 };
5282 let mut scopes = self
5283 .scopes
5284 .write()
5285 .unwrap_or_else(|poisoned| poisoned.into_inner());
5286 let admitted = scopes
5287 .admit(&principal, &target_module_id, selector, owner_configured)
5288 .and_then(|admission| {
5289 crate::scopes::check_target_flow_support(
5290 &admission.stamp,
5291 &target_module_id,
5292 registration.manifest.capabilities.as_ref(),
5293 )?;
5294 crate::scopes::check_target_agent_run_support(
5295 &admission.stamp,
5296 &target_module_id,
5297 registration.manifest.capabilities.as_ref(),
5298 )?;
5299 let bound = scopes.reserve_bind(
5300 &admission,
5301 &principal,
5302 ctx.connection_id,
5303 selector.hold_presence,
5304 );
5305 Ok((admission, bound))
5306 });
5307 match admitted {
5308 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
5309 Err(refusal) => {
5310 return Ok(vec![self.route_open_refusal_frame(
5311 ctx,
5312 &caller,
5313 &frame,
5314 &target_module_id,
5315 refusal.code,
5316 refusal.code,
5317 refusal.message,
5318 )?]);
5319 }
5320 }
5321 }
5322 };
5323 if let Some(stamp) = &scope_stamp {
5326 caller.scope = route_open_scope_label(stamp);
5327 }
5328
5329 timings.enter(RouteOpenPhase::ProjectRoot);
5337 let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
5338 Ok(project) => project,
5339 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5340 };
5341 timings.enter(RouteOpenPhase::Admission);
5342 let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
5343 Ok(root) => root,
5344 Err(err) => {
5345 return Ok(vec![control_error_frame(
5346 &frame,
5347 "invalid_project_root",
5348 err.to_string(),
5349 )?])
5350 }
5351 };
5352 let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
5358
5359 let _concurrency_guard = match self
5370 .route_bind_concurrency
5371 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5372 {
5373 Ok(guard) => guard,
5374 Err(in_flight) => {
5375 return Ok(vec![self.route_open_target_capacity_refusal(
5376 ctx,
5377 &caller,
5378 &frame,
5379 &target_module_id,
5380 in_flight,
5381 )?]);
5382 }
5383 };
5384
5385 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5388 RouteBindAdmission::Admitted { guard, probe } => {
5389 if probe {
5390 info!(
5391 module_id = %target_module_id,
5392 connection_id = ctx.connection_id.get(),
5393 "route.bind breaker half-open: admitting one probe"
5394 );
5395 }
5396 guard
5397 }
5398 RouteBindAdmission::Refused {
5399 consecutive_timeouts,
5400 retry_in,
5401 probe_in_flight,
5402 } => {
5403 return Ok(vec![self.route_open_breaker_refusal_frame(
5404 ctx,
5405 &caller,
5406 &frame,
5407 &target_module_id,
5408 consecutive_timeouts,
5409 retry_in,
5410 probe_in_flight,
5411 )?]);
5412 }
5413 };
5414
5415 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5420 let relay_deadline = Instant::now() + route_bind_relay_timeout;
5421 let operator_app = self
5422 .operator_apps
5423 .lock()
5424 .unwrap_or_else(|p| p.into_inner())
5425 .get(&ctx.connection_id)
5426 .cloned();
5427 timings.enter(RouteOpenPhase::EgressReservation);
5428 let pending = match self
5429 .forwarding
5430 .begin_route_bind_relay_for(
5431 ctx.connection_id,
5432 ctx.egress.clone(),
5433 response_version(&frame),
5434 frame.header.corr,
5435 &target_module_id,
5436 principal.clone(),
5437 bound_scope,
5438 project_root,
5439 identity.harness.clone(),
5440 build_class,
5441 operator_app.clone(),
5442 relay_deadline,
5443 busy_refusals == Some(true),
5444 )
5445 .await
5446 {
5447 Ok(pending) => pending,
5448 Err(err) => {
5449 return Ok(vec![self.route_open_refusal_frame(
5450 ctx,
5451 &caller,
5452 &frame,
5453 &target_module_id,
5454 "relay_reservation_failed",
5455 forwarding_error_code(&err),
5456 err.to_string(),
5457 )?])
5458 }
5459 };
5460 timings.enter(RouteOpenPhase::Admission);
5461 let crate::forwarding::PendingRouteBindRelay {
5462 endpoint,
5463 module_sink,
5464 negotiated_ver,
5465 client_channel,
5466 client_epoch,
5467 module_channel,
5468 module_epoch,
5469 corr: relay_corr,
5470 receiver,
5471 } = pending;
5472 let mut reservation =
5473 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5474
5475 let relay_registration = self
5480 .registry
5481 .get_module_by_connection(endpoint.connection_id)
5482 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5483 let relay_capabilities = relay_registration
5484 .as_ref()
5485 .and_then(|registration| registration.manifest.capabilities.as_ref());
5486 if let Some(selector) = &scope {
5487 if let Err(refusal) = self
5490 .scopes
5491 .read()
5492 .unwrap_or_else(|poisoned| poisoned.into_inner())
5493 .check_bind_proof(&principal, selector)
5494 {
5495 reservation.release_and_disarm();
5496 return Ok(vec![self.route_open_refusal_frame(
5497 ctx,
5498 &caller,
5499 &frame,
5500 &target_module_id,
5501 refusal.code,
5502 refusal.code,
5503 refusal.message,
5504 )?]);
5505 }
5506 }
5507 if let Some(stamp) = scope_stamp
5508 .as_ref()
5509 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5510 {
5511 if let Err(refusal) = crate::scopes::check_target_flow_support(
5512 stamp,
5513 &target_module_id,
5514 relay_capabilities,
5515 )
5516 .and_then(|()| {
5517 crate::scopes::check_target_agent_run_support(
5518 stamp,
5519 &target_module_id,
5520 relay_capabilities,
5521 )
5522 }) {
5523 reservation.release_and_disarm();
5524 return Ok(vec![self.route_open_refusal_frame(
5525 ctx,
5526 &caller,
5527 &frame,
5528 &target_module_id,
5529 refusal.code,
5530 refusal.code,
5531 refusal.message,
5532 )?]);
5533 }
5534 }
5535
5536 if !relay_capabilities.is_some_and(|caps| {
5539 caps.provides
5540 .iter()
5541 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5542 }) {
5543 if let Some(stamp) = scope_stamp.as_mut() {
5544 stamp.workdir = None;
5545 stamp.project_id = None;
5546 }
5547 }
5548 let worker_run = scope_stamp
5553 .as_ref()
5554 .is_some_and(crate::scopes::is_worker_run)
5555 && relay_capabilities.is_some_and(|caps| {
5556 caps.provides
5557 .iter()
5558 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5559 });
5560 if !worker_run
5561 && !relay_capabilities.is_some_and(|caps| {
5562 caps.provides
5563 .iter()
5564 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5565 })
5566 {
5567 if let Some(stamp) = scope_stamp.as_mut() {
5568 stamp.attributes.on_behalf_of_agent = None;
5569 }
5570 }
5571 if !relay_capabilities.is_some_and(|caps| {
5572 caps.provides
5573 .iter()
5574 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5575 }) {
5576 if let Some(stamp) = scope_stamp.as_mut() {
5577 stamp.attributes.flow_version = None;
5578 stamp.attributes.flow_code_hash = None;
5579 stamp.attributes.flow_install_id = None;
5580 }
5581 }
5582 let call_mode = if worker_run
5583 || relay_capabilities
5584 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5585 {
5586 Some(derive_call_mode(scope_stamp.as_ref()))
5587 } else {
5588 if let Some(stamp) = scope_stamp.as_mut() {
5589 stamp.expires_at_ms = None;
5590 }
5591 None
5592 };
5593
5594 debug!(
5595 connection_id = ctx.connection_id.get(),
5596 client_channel,
5597 client_epoch,
5598 module_channel,
5599 module_epoch,
5600 "reserved route handle pair"
5601 );
5602 let route_principal_label = principal_label(&principal);
5605 let relay = ModuleControlRequest::RouteBind {
5606 route_channel: module_channel,
5607 epoch: module_epoch,
5608 target,
5609 identity,
5610 principal: Some(principal),
5611 consumer_capabilities,
5612 role_versions,
5613 admission_facts,
5614 scope: scope_stamp,
5615 call_mode,
5616 };
5617 let peer = RouteBindPeer::new()
5618 .with_observed_peer(
5619 relay_capabilities
5620 .is_some_and(|caps| {
5621 caps.provides
5622 .iter()
5623 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5624 })
5625 .then(|| {
5626 self.observed_peers
5627 .lock()
5628 .unwrap_or_else(|p| p.into_inner())
5629 .get(&ctx.connection_id)
5630 .cloned()
5631 .unwrap_or_else(ObservedPeer::unobserved)
5632 }),
5633 )
5634 .with_operator_app(
5635 self.connection_operator_app(operator_app.as_ref())
5636 .filter(|_| {
5637 relay_capabilities.is_some_and(|caps| {
5638 caps.provides
5639 .iter()
5640 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5641 })
5642 }),
5643 );
5644 let relay_body = crate::bind_compat::encode_bind(
5645 &relay,
5646 &peer,
5647 &legacy_bind_project,
5648 relay_capabilities,
5649 )
5650 .map_err(|err| {
5651 RouterError::backend(
5652 0,
5653 frame.header.corr,
5654 format!("failed to encode route.bind request: {err}"),
5655 )
5656 })?;
5657 let relay_frame = Frame::build_with_version(
5658 negotiated_ver,
5659 FrameType::Request,
5660 control_flags(),
5661 0,
5662 0,
5663 relay_corr,
5664 relay_body,
5665 )
5666 .map_err(RouterError::FrameBuild)?;
5667
5668 timings.enter(RouteOpenPhase::BindSend);
5669 let sent = module_sink.send(relay_frame).await;
5670 timings.enter(RouteOpenPhase::Admission);
5671 if let Err(err) = sent {
5672 reservation.release_and_disarm();
5673 return Ok(vec![self.route_open_refusal_frame(
5674 ctx,
5675 &caller,
5676 &frame,
5677 &target_module_id,
5678 "relay_send_failed",
5679 "target_unavailable",
5680 err.to_string(),
5681 )?]);
5682 }
5683
5684 #[cfg(test)]
5685 if let Some(hold) = &self.route_bind_relay_sent_hold {
5686 hold.wait().await;
5690 hold.wait().await;
5691 }
5692
5693 if self
5698 .forwarding
5699 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5700 .map_err(RouterError::Forwarding)?
5701 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5702 {
5703 self.send_abandoned_route_bind_goodbye(
5704 &module_sink,
5705 negotiated_ver,
5706 module_channel,
5707 module_epoch,
5708 );
5709 }
5710
5711 timings.enter(RouteOpenPhase::BindAckWait);
5712 let outcome = timeout_at(relay_deadline, receiver).await;
5713 timings.enter(RouteOpenPhase::Admission);
5714 match outcome {
5715 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5716 reservation.disarm();
5717 if breaker.record_accepted() {
5718 info!(
5719 module_id = %target_module_id,
5720 "route.bind breaker closed: the probe was accepted"
5721 );
5722 }
5723 self.observe_route_open_accept(
5724 ctx,
5725 &target_module_id,
5726 &route_principal_label,
5727 &caller.scope,
5728 );
5729 Ok(Vec::new())
5730 }
5731 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5732 reservation.release_and_disarm();
5733 breaker.record_inconclusive();
5737 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5738 }
5739 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5740 reservation.release_and_disarm();
5741 breaker.record_inconclusive();
5742 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5743 }
5744 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5745 reservation.release_and_disarm();
5746 breaker.record_inconclusive();
5747 tracing::warn!(
5754 module_id = %target_module_id,
5755 "route.bind relay abandoned: {message}"
5756 );
5757 Ok(vec![self.route_open_refusal_frame(
5758 ctx,
5759 &caller,
5760 &frame,
5761 &target_module_id,
5762 "relay_abandoned",
5763 "target_unavailable",
5764 message,
5765 )?])
5766 }
5767 Ok(Err(_)) => {
5768 reservation.release_and_disarm();
5769 breaker.record_inconclusive();
5770 Ok(vec![self.route_open_refusal_frame(
5771 ctx,
5772 &caller,
5773 &frame,
5774 &target_module_id,
5775 "relay_waiter_canceled",
5776 "target_unavailable",
5777 "route.bind relay waiter was canceled before the module responded",
5778 )?])
5779 }
5780 Err(_) => {
5781 reservation.release_and_disarm();
5782 if let Some(opened) = breaker.record_timeout(
5786 self.route_bind_breaker_threshold,
5787 self.route_bind_breaker_cooldown,
5788 ) {
5789 warn!(
5790 module_id = %target_module_id,
5791 consecutive_timeouts = opened.consecutive_timeouts,
5792 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5793 reopened_after_probe = opened.reopened_after_probe,
5794 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5795 );
5796 }
5797 tracing::warn!(
5805 module_id = %target_module_id,
5806 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5807 "route.bind relay timed out: module did not ack within budget"
5808 );
5809 Ok(vec![self.route_open_refusal_frame(
5810 ctx,
5811 &caller,
5812 &frame,
5813 &target_module_id,
5814 "relay_timed_out",
5815 "module_timeout",
5816 format!(
5817 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5818 route_bind_relay_timeout
5819 ),
5820 )?])
5821 }
5822 }
5823 }
5824
5825 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5826 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5827 snapshot: self.supervisor.spawn_snapshot(),
5828 };
5829 Ok(vec![control_response_body_frame(
5830 &frame,
5831 &response,
5832 "ClientControlResponse::SupervisorSpawnSnapshot",
5833 )?])
5834 }
5835
5836 fn handle_supervisor_spawn_subscribe(
5837 &self,
5838 ctx: &RouteCtx,
5839 frame: Frame,
5840 since: Option<SpawnCursor>,
5841 ) -> Result<Vec<Frame>, RouterError> {
5842 match self.supervisor.subscribe_spawns(
5843 ctx.connection_id,
5844 frame.header.corr,
5845 response_version(&frame),
5846 since,
5847 ctx.egress.clone(),
5848 ) {
5849 Ok(()) => Ok(Vec::new()),
5850 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5851 Ok(vec![control_error_body_frame(
5852 &frame,
5853 ErrorBody {
5854 code: "spawn_cursor_incarnation_mismatch".to_string(),
5855 message: "spawn cursor belongs to a different daemon incarnation"
5856 .to_string(),
5857 detail: Some(serde_json::json!({
5858 "current_daemon_incarnation": current
5859 })),
5860 },
5861 )?])
5862 }
5863 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5864 &frame,
5865 ErrorBody {
5866 code: "spawn_cursor_too_old".to_string(),
5867 message: "spawn cursor predates the retained event ring".to_string(),
5868 detail: Some(serde_json::json!({
5869 "oldest_retained_cursor": oldest
5870 })),
5871 },
5872 )?]),
5873 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5874 0,
5875 frame.header.corr,
5876 format!("failed to open supervisor spawn subscription: {error}"),
5877 )),
5878 }
5879 }
5880
5881 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5882 let generation = self
5883 .registry
5884 .generation()
5885 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5886 let supervised = self.supervisor.list();
5887 let targets = supervised
5888 .iter()
5889 .map(|module| module.subtree_target())
5890 .collect();
5891 let subtree = tokio::spawn(crate::subtree_resources::read(targets));
5892 let mut modules = Vec::new();
5893 for module in supervised {
5894 let status = module.status_for_control("list").map_err(|err| {
5895 RouterError::backend(
5896 0,
5897 frame.header.corr,
5898 format!("failed to read supervisor status: {err}"),
5899 )
5900 })?;
5901 let (configured, _) = module.configuration().map_err(|err| {
5902 RouterError::backend(
5903 0,
5904 frame.header.corr,
5905 format!("failed to read module configuration: {err}"),
5906 )
5907 })?;
5908 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5909 RouterError::backend(
5910 0,
5911 frame.header.corr,
5912 format!("failed to read retry time: {err}"),
5913 )
5914 })?;
5915 let image = module.running_image_agreement().await;
5917 let resources = Some(crate::subtree_resources::read_process(module.clone()).await);
5920 let mut verdict =
5921 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5922 verdict.spec = Some(launch_spec_agreement(
5923 &configured,
5924 status.process_alive,
5925 status.launch_spec_digest.as_ref(),
5926 ));
5927 let pending_reload = Some(verdict);
5928 let configured_enabled = self
5929 .capability_evaluator
5930 .configured_enabled(&status.module_id);
5931 modules.push(
5934 SupervisorEntry::new(
5935 status.module_id,
5936 status.state.to_string(),
5937 status.enabled,
5938 status.live,
5939 status.health.status,
5940 )
5941 .with_configured_enabled(configured_enabled)
5942 .with_next_retry_at_ms(next_retry_at_ms)
5943 .with_launch_nonce_env(Some(!cfg!(unix)))
5944 .with_launch_nonce_source(status.launch_nonce_source)
5945 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5946 .with_protocol(status.protocol)
5947 .with_pending_reload(pending_reload)
5948 .with_last_probe_ms(status.health.last_probe_ms)
5949 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5950 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5951 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5952 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5953 .with_restart_count(Some(status.restart_count))
5954 .with_max_restarts(Some(status.max_restarts))
5955 .with_lifetime_restarts(Some(status.lifetime_restarts))
5956 .with_spawn_generation(Some(status.spawn_generation))
5957 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5958 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5959 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5960 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5961 .with_resources(resources),
5962 );
5963 }
5964 let readings = subtree
5965 .await
5966 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5967 for (module, reading) in modules.iter_mut().zip(readings) {
5968 if reading.reason.as_deref() == Some("timeout") {
5969 crate::subtree_resources::log_timeout(&module.module_id, "subtree");
5970 }
5971 module.subtree = Some(reading);
5972 }
5973 let response = ClientControlResponse::SupervisorList {
5974 generation,
5975 modules,
5976 };
5977 Ok(vec![control_response_body_frame(
5978 &frame,
5979 &response,
5980 "ClientControlResponse::SupervisorList",
5981 )?])
5982 }
5983
5984 fn handle_supervisor_stderr_tail(
5985 &self,
5986 frame: Frame,
5987 module_id: String,
5988 max_lines: Option<u32>,
5989 max_bytes: Option<u32>,
5990 ) -> Result<Vec<Frame>, RouterError> {
5991 let Some(module) = self.supervisor.get(&module_id) else {
5992 return Ok(vec![control_error_frame(
5993 &frame,
5994 "unknown_module",
5995 format!("module_id '{module_id}' is not supervised"),
5996 )?]);
5997 };
5998
5999 let snapshot = module.stderr_tail(
6000 max_lines.map(|value| value as usize),
6001 max_bytes.map(|value| value as usize),
6002 );
6003
6004 let response = ClientControlResponse::SupervisorStderrTail {
6005 module_id,
6006 tail: StderrTail {
6007 capture: match snapshot.capture {
6008 CaptureState::Captured => StderrCaptureState::Captured,
6009 CaptureState::Incomplete { reason } => {
6010 StderrCaptureState::Incomplete { reason }
6011 }
6012 CaptureState::NotCaptured { reason } => {
6013 StderrCaptureState::NotCaptured { reason }
6014 }
6015 },
6016 entries: snapshot
6017 .entries
6018 .into_iter()
6019 .map(|entry| match entry {
6020 TailEntry::Line {
6021 text,
6022 truncated,
6023 at_ms,
6024 } => StderrTailEntry::Line {
6025 text,
6026 truncated,
6027 at_ms,
6028 },
6029 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
6030 })
6031 .collect(),
6032 dropped_lines: snapshot.dropped_lines,
6033 },
6034 };
6035 Ok(vec![control_response_body_frame(
6036 &frame,
6037 &response,
6038 "ClientControlResponse::SupervisorStderrTail",
6039 )?])
6040 }
6041
6042 async fn observe_control_wait<F: std::future::Future>(
6043 &self,
6044 ctx: &RouteCtx,
6045 target: Option<&str>,
6046 kind: &'static str,
6047 future: F,
6048 ) -> F::Output {
6049 self.counters
6050 .observe_reader_wait(
6051 || {
6052 let module = self
6053 .forwarding
6054 .module_id_for_connection(ctx.connection_id)
6055 .ok()
6056 .flatten();
6057 let principal = if module.is_some() { "module" } else { "direct" };
6058 (
6059 self.resolve_reader_wait_target(kind, target)
6060 .or(module)
6061 .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
6062 principal.into(),
6063 )
6064 },
6065 kind,
6066 future,
6067 )
6068 .await
6069 }
6070
6071 pub(crate) fn resolve_reader_wait_target(
6074 &self,
6075 op: &str,
6076 selector: Option<&str>,
6077 ) -> Option<String> {
6078 let selector = selector?;
6079 match op {
6080 ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
6081 .supervisor
6082 .get(selector)
6083 .map(|module| module.module_id().to_owned()),
6084 ops::SUPERVISOR_HEALTH_PROBE => self
6085 .registry
6086 .get_module(selector)
6087 .ok()
6088 .flatten()
6089 .map(|registration| registration.manifest.module_id),
6090 _ => None,
6091 }
6092 }
6093
6094 async fn handle_supervisor_terminals(
6095 &self,
6096 ctx: &RouteCtx,
6097 frame: Frame,
6098 module_id: String,
6099 ) -> Result<Vec<Frame>, RouterError> {
6100 let Some(module) = self.supervisor.get(&module_id) else {
6101 return Ok(vec![control_error_frame(
6102 &frame,
6103 "unknown_module",
6104 format!("module_id '{module_id}' is not supervised"),
6105 )?]);
6106 };
6107
6108 let terminals = self
6111 .observe_control_wait(
6112 ctx,
6113 Some(module.module_id()),
6114 ops::SUPERVISOR_TERMINALS,
6115 module.read_durable_terminal_history(),
6116 )
6117 .await
6118 .map_err(|error| {
6119 RouterError::backend(
6120 0,
6121 frame.header.corr,
6122 format!("failed to read terminal history: {error}"),
6123 )
6124 })?;
6125 let response = ClientControlResponse::SupervisorTerminals {
6126 module_id,
6127 terminals,
6128 };
6129 Ok(vec![control_response_body_frame(
6130 &frame,
6131 &response,
6132 "ClientControlResponse::SupervisorTerminals",
6133 )?])
6134 }
6135
6136 fn handle_supervisor_routes(
6137 &self,
6138 frame: Frame,
6139 module_id: Option<String>,
6140 ) -> Result<Vec<Frame>, RouterError> {
6141 let modules = self
6142 .forwarding
6143 .route_census(module_id.as_deref())
6144 .map_err(RouterError::Forwarding)?
6145 .into_iter()
6146 .map(|(module_id, routes)| SupervisorRouteModule {
6147 module_id,
6148 routes: routes
6149 .into_iter()
6150 .map(|route| SupervisorRoute {
6151 consumer: match route.principal {
6152 Principal::Reserved { module_id } => {
6153 SupervisorRouteConsumer::Reserved { module_id }
6154 }
6155 Principal::Direct | Principal::Unverified => {
6156 SupervisorRouteConsumer::Direct {
6157 connection_id: route.goodbye_target.connection_id.get(),
6158 }
6159 }
6160 },
6161 age_ms: Instant::now()
6162 .saturating_duration_since(route.bound_at)
6163 .as_millis()
6164 .try_into()
6165 .unwrap_or(u64::MAX),
6166 draining: route.draining,
6167 drain_reason: route.drain_reason,
6168 })
6169 .collect(),
6170 })
6171 .collect();
6172 let response = ClientControlResponse::SupervisorRoutes { modules };
6173 Ok(vec![control_response_body_frame(
6174 &frame,
6175 &response,
6176 "ClientControlResponse::SupervisorRoutes",
6177 )?])
6178 }
6179
6180 async fn handle_supervisor_provenance(
6181 &self,
6182 ctx: &RouteCtx,
6183 frame: Frame,
6184 module_id: Option<String>,
6185 ) -> Result<Vec<Frame>, RouterError> {
6186 let targeted = module_id.is_some();
6187 let selected = if let Some(module_id) = module_id {
6188 let Some(module) = self.supervisor.get(&module_id) else {
6189 return Ok(vec![control_error_frame(
6190 &frame,
6191 "unknown_module",
6192 format!("module_id '{module_id}' is not supervised"),
6193 )?]);
6194 };
6195 vec![module]
6196 } else {
6197 self.supervisor.list()
6198 };
6199
6200 let target = targeted.then(|| selected[0].module_id());
6201 self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
6202 let mut modules = Vec::with_capacity(selected.len());
6203 for module in &selected {
6204 let (status, observed_image) = module
6205 .status_and_running_image_agreement()
6206 .await
6207 .map_err(|err| {
6208 RouterError::backend(
6209 0,
6210 frame.header.corr,
6211 format!("failed to read supervisor status: {err}"),
6212 )
6213 })?;
6214 let module_declared = self
6215 .registry
6216 .get_module(&status.module_id)
6217 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6218 .and_then(|registration| registration.manifest.provenance)
6219 .map(|build| ModuleDeclaredProvenance::Reported { build })
6220 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
6221 #[cfg(test)]
6222 let running_image = match &self.provenance_probe_override {
6223 Some(result) => result.clone(),
6224 None => observed_image,
6225 };
6226 #[cfg(not(test))]
6227 let running_image = observed_image;
6228 modules.push(SupervisorModuleProvenance {
6229 module_id: status.module_id,
6230 module_declared,
6231 daemon_observed: SupervisorObservedProcess {
6232 launch_nonce_source: status.launch_nonce_source,
6233 pid: status.pid,
6234 spawned_at_ms: status.spawned_at_ms,
6235 spawned_from: status.spawned_from,
6236 running_image,
6237 },
6238 });
6239 }
6240 let daemon = SupervisorDaemonProvenance {
6241 daemon_build: self.daemon_provenance.build.clone(),
6242 daemon_observed: DaemonObservedProcess {
6243 pid: self.daemon_provenance.pid,
6244 started_at_ms: self
6245 .daemon_provenance
6246 .start_clock
6247 .map(|clock| clock.started_at_ms())
6248 .or(self.daemon_provenance.started_at_ms),
6249 running_image: self
6250 .daemon_provenance
6251 .probe
6252 .observe(
6253 self.daemon_provenance.pid,
6254 self.daemon_provenance.executable_path.as_deref(),
6255 self.daemon_provenance.executable_identity.clone(),
6256 self.daemon_provenance.process_start_time,
6257 )
6258 .await,
6259 },
6260 };
6261 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
6262 Ok(vec![control_response_body_frame(
6263 &frame,
6264 &response,
6265 "ClientControlResponse::SupervisorProvenance",
6266 )?])
6267 })
6268 .await
6269 }
6270
6271 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6272 self.refresh_capability_requirements();
6273 let generation = self
6274 .registry
6275 .generation()
6276 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6277 let modules = self
6278 .supervisor
6279 .list()
6280 .into_iter()
6281 .map(|module| {
6282 let status = module.status_for_control("health").map_err(|err| {
6283 RouterError::backend(
6284 0,
6285 frame.header.corr,
6286 format!("failed to read supervisor health: {err}"),
6287 )
6288 })?;
6289 let module_id = status.module_id;
6290 let capability_detail = self
6291 .capability_evaluator
6292 .required_problem_detail(&module_id);
6293 Ok(SupervisorHealthEntry {
6294 module_id,
6295 status: status.health.status,
6296 detail: append_capability_problem_detail(
6297 status.health.detail,
6298 capability_detail,
6299 ),
6300 metrics: status.health.metrics,
6301 consecutive_failures: status.health.consecutive_failures,
6302 late_answer_count: status.health.late_answer_count,
6303 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
6304 last_action: status.health.last_action,
6305 last_action_ms: status.health.last_action_ms,
6306 last_probe_ms: status.health.last_probe_ms,
6307 })
6308 })
6309 .collect::<Result<Vec<_>, RouterError>>()?;
6310 let response = ClientControlResponse::SupervisorHealth {
6311 generation,
6312 modules,
6313 };
6314 Ok(vec![control_response_body_frame(
6315 &frame,
6316 &response,
6317 "ClientControlResponse::SupervisorHealth",
6318 )?])
6319 }
6320
6321 async fn handle_supervisor_restart(
6322 &self,
6323 frame: Frame,
6324 module_id: String,
6325 drain_timeout_ms: Option<u64>,
6326 requester: Principal,
6327 ) -> Result<Vec<Frame>, RouterError> {
6328 let operation_lock = self.supervisor.operation_lock();
6329 let _operation_guard = operation_lock.lock().await;
6330 let Some(module) = self.supervisor.get(&module_id) else {
6331 return Ok(vec![control_error_frame(
6332 &frame,
6333 "unknown_module",
6334 format!("module_id '{module_id}' is not supervised"),
6335 )?]);
6336 };
6337
6338 self.route_outages.mark_operator_action(&module_id);
6339 if let Err(err) = module
6340 .restart_requested(drain_timeout_ms, Some(requester))
6341 .await
6342 {
6343 self.route_outages
6344 .operator_action_ended_unrefused(&module_id);
6345 let (code, message) = match err {
6346 crate::supervise::SuperviseError::Disabled { .. } => {
6347 ("module_disabled", err.to_string())
6348 }
6349 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6350 ("swap_in_progress", err.to_string())
6351 }
6352 _ => (
6353 "target_unavailable",
6354 format!("failed to restart module_id '{module_id}': {err}"),
6355 ),
6356 };
6357 return Ok(vec![control_error_frame(&frame, code, message)?]);
6358 }
6359
6360 let response = ClientControlResponse::SupervisorAck {
6361 module_id,
6362 applied: true,
6363 };
6364 Ok(vec![control_response_body_frame(
6365 &frame,
6366 &response,
6367 "ClientControlResponse::SupervisorAck",
6368 )?])
6369 }
6370
6371 async fn handle_supervisor_swap(
6375 &self,
6376 frame: Frame,
6377 module_id: String,
6378 ready_timeout_ms: Option<u64>,
6379 requester: Principal,
6380 ) -> Result<Vec<Frame>, RouterError> {
6381 let module = {
6388 let operation_lock = self.supervisor.operation_lock();
6389 let _operation_guard = operation_lock.lock().await;
6390 self.supervisor.get(&module_id)
6391 };
6392 let Some(module) = module else {
6393 return Ok(vec![control_error_frame(
6394 &frame,
6395 "unknown_module",
6396 format!("module_id '{module_id}' is not supervised"),
6397 )?]);
6398 };
6399
6400 self.route_outages.mark_operator_action(&module_id);
6401 if let Err(err) = module
6402 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6403 .await
6404 {
6405 self.route_outages
6406 .operator_action_ended_unrefused(&module_id);
6407 use crate::supervise::SuperviseError;
6408 let message = err.to_string();
6409 let error = match err {
6410 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6411 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6412 code: "swap_refused".to_string(),
6413 message,
6414 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6415 },
6416 SuperviseError::SwapFailed {
6417 arm,
6418 candidate_exit,
6419 ..
6420 } => ErrorBody {
6421 code: "swap_failed".to_string(),
6422 message,
6423 detail: Some(serde_json::json!({
6424 "arm": arm.as_str(),
6425 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6426 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6427 })),
6428 },
6429 _ => ErrorBody::new(
6430 "target_unavailable",
6431 format!("failed to swap module_id '{module_id}': {message}"),
6432 ),
6433 };
6434 return Ok(vec![control_error_body_frame(&frame, error)?]);
6435 }
6436 self.route_outages
6440 .operator_action_ended_unrefused(&module_id);
6441
6442 let response = ClientControlResponse::SupervisorAck {
6443 module_id,
6444 applied: true,
6445 };
6446 Ok(vec![control_response_body_frame(
6447 &frame,
6448 &response,
6449 "ClientControlResponse::SupervisorAck",
6450 )?])
6451 }
6452
6453 async fn handle_supervisor_reload(
6454 &self,
6455 frame: Frame,
6456 module_id: String,
6457 requester: Principal,
6458 ) -> Result<Vec<Frame>, RouterError> {
6459 let operation_lock = self.supervisor.operation_lock();
6460 let _operation_guard = operation_lock.lock().await;
6461 let Some(module) = self.supervisor.get(&module_id) else {
6462 return Ok(vec![control_error_frame(
6463 &frame,
6464 "unknown_module",
6465 format!("module_id '{module_id}' is not supervised"),
6466 )?]);
6467 };
6468
6469 self.route_outages.mark_operator_action(&module_id);
6470 if let Err(err) = module.reload_requested(Some(requester)).await {
6471 self.route_outages
6472 .operator_action_ended_unrefused(&module_id);
6473 let (code, message) = match err {
6474 crate::supervise::SuperviseError::Disabled { .. } => {
6475 ("module_disabled", err.to_string())
6476 }
6477 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6478 ("swap_in_progress", err.to_string())
6479 }
6480 _ => (
6481 "reload_failed",
6482 format!("failed to reload module_id '{module_id}': {err}"),
6483 ),
6484 };
6485 return Ok(vec![control_error_frame(&frame, code, message)?]);
6486 }
6487
6488 let response = ClientControlResponse::SupervisorAck {
6489 module_id,
6490 applied: true,
6491 };
6492 Ok(vec![control_response_body_frame(
6493 &frame,
6494 &response,
6495 "ClientControlResponse::SupervisorAck",
6496 )?])
6497 }
6498
6499 async fn handle_supervisor_rescan(
6503 &self,
6504 frame: Frame,
6505 mode: RescanMode,
6506 requester: Principal,
6507 ) -> Result<Vec<Frame>, RouterError> {
6508 let preview = mode.previews();
6509 let Some(context) = self.rescan.clone() else {
6510 return Ok(vec![control_error_frame(
6511 &frame,
6512 "rescan_unavailable",
6513 "the daemon was not started with a reloadable config path".to_string(),
6514 )?]);
6515 };
6516
6517 let operation_lock = self.supervisor.operation_lock();
6518 let _operation_guard = operation_lock.lock().await;
6519 let loaded = match self.load_rescan_config(&mode, &context) {
6523 Ok(config) => config,
6524 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6525 };
6526 let Some((config, activity_config)) = loaded else {
6534 return Ok(vec![control_error_frame(
6535 &frame,
6536 "invalid_daemon_config",
6537 format!(
6538 "daemon config not found at {}; refusing to rescan (an absent config would \
6539 retire every supervised module)",
6540 context.config_path.display()
6541 ),
6542 )?]);
6543 };
6544 let (
6545 configured_port,
6546 storage_config,
6547 admission_facts_carrier_module_id,
6548 admission_facts_targets,
6549 scope_authority_owners,
6550 privileged_operator_prompt,
6551 modules,
6552 reserved_capabilities,
6553 ) = (
6554 config.port,
6555 config.storage,
6556 config.admission_facts_carrier_module_id,
6557 config.admission_facts_targets,
6558 config.scope_authority_owners,
6559 config.privileged_operator_prompt,
6560 config.modules,
6561 config.reserved_capabilities,
6562 );
6563
6564 let mut restart_required = Vec::new();
6573 for section in RestartRequiredSection::ALL {
6574 let changed = match section {
6575 RestartRequiredSection::Port => configured_port != context.configured_port,
6576 RestartRequiredSection::Storage => storage_config != context.storage_config,
6577 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6578 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6579 }
6580 RestartRequiredSection::AdmissionFactsTargets => {
6581 admission_facts_targets != context.admission_facts_targets
6582 }
6583 RestartRequiredSection::ScopeAuthorityOwners => {
6584 scope_authority_owners != context.scope_authority_owners
6585 }
6586 RestartRequiredSection::PrivilegedOperatorPrompt => {
6587 privileged_operator_prompt != context.privileged_operator_prompt
6588 }
6589 };
6590 if changed {
6591 restart_required.push(section.label().to_string());
6592 }
6593 }
6594 if !restart_required.is_empty() {
6595 warn!(
6596 config_path = %context.config_path.display(),
6597 sections = %restart_required.join(", "),
6598 "daemon config changed outside the modules section; restart the daemon to apply those changes"
6599 );
6600 }
6601
6602 for configured in &modules {
6603 if let Err(err) = validate_spec(&configured.module_spec()) {
6604 return Ok(vec![control_error_frame(
6605 &frame,
6606 "invalid_daemon_config",
6607 format!("supervisor rescan rejected daemon config: {err}"),
6608 )?]);
6609 }
6610 }
6611
6612 let configured_capabilities = modules
6613 .iter()
6614 .map(|module| (module.module_id.clone(), module.enabled))
6615 .collect::<Vec<_>>();
6616 let preview_capability_warnings = if preview {
6617 let (_, registrations) = self.runtime_capability_snapshot()?;
6618 let current_modules = self
6619 .supervisor
6620 .list()
6621 .into_iter()
6622 .map(|module| module.module_id().to_string())
6623 .collect::<BTreeSet<_>>();
6624 let resulting_modules = configured_capabilities.clone();
6625 let removed = current_modules
6626 .into_iter()
6627 .filter(|module_id| {
6628 !resulting_modules
6629 .iter()
6630 .any(|(configured_id, _)| configured_id == module_id)
6631 })
6632 .collect::<Vec<_>>();
6633 let disabled = match &mode {
6636 RescanMode::Candidate(_) => resulting_modules
6637 .iter()
6638 .filter(|(_, enabled)| !enabled)
6639 .map(|(module_id, _)| module_id.clone())
6640 .collect::<Vec<_>>(),
6641 _ => Vec::new(),
6642 };
6643 self.capability_evaluator.preview_warnings(
6644 resulting_modules,
6645 &removed,
6646 &disabled,
6647 ®istrations,
6648 )
6649 } else {
6650 Vec::new()
6651 };
6652 let reconciled = match &mode {
6653 RescanMode::Candidate(candidate) => self
6654 .preview_candidate(&context, modules, candidate)
6655 .map(|result| (result, Vec::new())),
6656 _ => {
6657 self.reconcile_supervised_modules(
6658 &context.supervisor,
6659 modules,
6660 preview,
6661 requester.clone(),
6662 )
6663 .await
6664 }
6665 };
6666 let (result, changed) = match reconciled {
6667 Ok(result) => result,
6668 Err(message) => {
6669 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6670 }
6671 };
6672 if !preview {
6673 if let Some(ledger) = &self.activity {
6674 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6675 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6676 }
6677 if let Err(code) = self.configure_activity(activity_config).await {
6678 warn!(
6679 code,
6680 "activity retention could not be applied during rescan"
6681 );
6682 }
6683 self.capability_evaluator
6684 .configure(configured_capabilities, reserved_capabilities);
6685 self.capability_evaluator.wake_deadline_loop();
6686 self.refresh_capability_requirements();
6687 }
6688 let mut result = result;
6689 result.restart_required = restart_required;
6690 result.capability_warnings = preview_capability_warnings;
6691 let response = ClientControlResponse::SupervisorRescan { result };
6692 Ok(vec![control_response_body_frame(
6693 &frame,
6694 &response,
6695 "ClientControlResponse::SupervisorRescan",
6696 )?])
6697 }
6698
6699 async fn handle_supervisor_release_reserved(
6700 &self,
6701 frame: Frame,
6702 module_id: String,
6703 ) -> Result<Vec<Frame>, RouterError> {
6704 let Some(context) = self.rescan.clone() else {
6705 return Ok(vec![control_error_frame(
6706 &frame,
6707 "release_unavailable",
6708 "reserved-id release requires a daemon started with a reloadable config path",
6709 )?]);
6710 };
6711 let operation_lock = self.supervisor.operation_lock();
6712 let _operation_guard = operation_lock.lock().await;
6713 let loaded = match crate::daemon_config::load(&context.config_path) {
6714 Ok(Some(config)) => config,
6715 Ok(None) => {
6716 return Ok(vec![control_error_frame(
6717 &frame,
6718 "invalid_daemon_config",
6719 format!(
6720 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6721 context.config_path.display()
6722 ),
6723 )?])
6724 }
6725 Err(err) => {
6726 return Ok(vec![control_error_frame(
6727 &frame,
6728 "invalid_daemon_config",
6729 format!("unable to verify reserved-id release against daemon config: {err}"),
6730 )?])
6731 }
6732 };
6733 if loaded
6734 .modules
6735 .iter()
6736 .any(|configured| configured.module_id == module_id)
6737 {
6738 return Ok(vec![control_error_frame(
6739 &frame,
6740 "reserved_module_configured",
6741 format!(
6742 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6743 ),
6744 )?]);
6745 }
6746 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6747 return Ok(vec![control_error_frame(
6748 &frame,
6749 "reserved_gate_not_retained",
6750 format!(
6751 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6752 ),
6753 )?]);
6754 }
6755
6756 let response = ClientControlResponse::SupervisorAck {
6757 module_id,
6758 applied: true,
6759 };
6760 Ok(vec![control_response_body_frame(
6761 &frame,
6762 &response,
6763 "ClientControlResponse::SupervisorAck",
6764 )?])
6765 }
6766
6767 fn plan_rescan(
6771 &self,
6772 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6773 ) -> Result<RescanPlan, String> {
6774 let mut current = BTreeMap::new();
6775 for module in self.supervisor.list() {
6776 let (spec, health) = module.configuration().map_err(|err| {
6777 format!(
6778 "failed to read configuration for module_id '{}': {err}",
6779 module.module_id()
6780 )
6781 })?;
6782 let enabled = module
6783 .status()
6784 .map_err(|err| {
6785 format!(
6786 "failed to read status for module_id '{}': {err}",
6787 module.module_id()
6788 )
6789 })?
6790 .enabled;
6791 current.insert(
6792 module.module_id().to_string(),
6793 (module, spec, health, enabled),
6794 );
6795 }
6796 let configured = configured_modules
6797 .into_iter()
6798 .map(|module| (module.module_id.clone(), module))
6799 .collect::<BTreeMap<_, _>>();
6800
6801 let added = configured
6802 .keys()
6803 .filter(|module_id| !current.contains_key(*module_id))
6804 .cloned()
6805 .collect::<Vec<_>>();
6806 let removed = current
6807 .keys()
6808 .filter(|module_id| !configured.contains_key(*module_id))
6809 .cloned()
6810 .collect::<Vec<_>>();
6811 let mut changed_pending_reload = Vec::new();
6812 let mut configuration_changes = BTreeSet::new();
6813 let mut enabled_changes = BTreeSet::new();
6814 let mut unchanged = 0_u32;
6815
6816 for (module_id, configured_module) in &configured {
6817 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6818 else {
6819 continue;
6820 };
6821 let launch_changed = *current_spec != configured_module.module_spec();
6825 let configuration_changed =
6826 launch_changed || *current_health != configured_module.health;
6827 let enabled_changed = *current_enabled != configured_module.enabled;
6828 if configuration_changed {
6829 configuration_changes.insert(module_id.clone());
6830 }
6831 if launch_changed {
6832 changed_pending_reload.push(module_id.clone());
6833 }
6834 if enabled_changed {
6835 enabled_changes.insert(module_id.clone());
6836 }
6837 if !configuration_changed && !enabled_changed {
6838 unchanged = unchanged.saturating_add(1);
6839 }
6840 }
6841 Ok(RescanPlan {
6842 current,
6843 configured,
6844 added,
6845 removed,
6846 changed_pending_reload,
6847 configuration_changes,
6848 enabled_changes,
6849 unchanged,
6850 })
6851 }
6852
6853 async fn reconcile_supervised_modules(
6862 &self,
6863 supervisor: &Supervisor,
6864 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6865 preview: bool,
6866 requester: Principal,
6867 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6868 let plan = self.plan_rescan(configured_modules)?;
6869
6870 if preview {
6876 return Ok((plan.preview_result(), Vec::new()));
6877 }
6878 let changed = plan
6879 .configuration_changes
6880 .union(&plan.enabled_changes)
6881 .cloned()
6882 .collect::<Vec<_>>();
6883 let RescanPlan {
6884 current,
6885 configured,
6886 added,
6887 removed,
6888 changed_pending_reload,
6889 configuration_changes,
6890 enabled_changes,
6891 unchanged,
6892 } = plan;
6893 if !(added.is_empty()
6898 && removed.is_empty()
6899 && configuration_changes.is_empty()
6900 && enabled_changes.is_empty())
6901 {
6902 if let Some(context) = &self.rescan {
6903 context
6904 .generation
6905 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6906 }
6907 }
6908
6909 for module_id in &removed {
6910 let module = ¤t
6911 .get(module_id)
6912 .expect("removed module came from current supervisor state")
6913 .0;
6914 module
6915 .retire_requested(Some(requester.clone()))
6916 .await
6917 .map_err(|err| {
6918 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6919 })?;
6920 self.supervisor.record_rescan_removal(module_id);
6954 self.supervisor.retire(module_id);
6955 self.route_outages.forget(module_id);
6956 }
6957
6958 for module_id in configured.keys() {
6959 let Some((module, _, _, _)) = current.get(module_id) else {
6960 continue;
6961 };
6962 let configured_module = configured
6963 .get(module_id)
6964 .expect("configured module id came from configured map");
6965 if configuration_changes.contains(module_id) {
6966 module
6967 .update_configuration(
6968 configured_module.module_spec(),
6969 configured_module.health.clone(),
6970 configured_module.drain_timeout_ms,
6971 )
6972 .await
6973 .map_err(|err| {
6974 format!(
6975 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6976 )
6977 })?;
6978 }
6979 if enabled_changes.contains(module_id) {
6980 self.route_outages.mark_operator_action(module_id);
6983 module
6984 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6985 .await
6986 .map_err(|err| {
6987 self.route_outages.operator_action_ended_unrefused(module_id);
6988 format!(
6989 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6990 configured_module.enabled
6991 )
6992 })?;
6993 }
6994 }
6995
6996 for module_id in &added {
6997 let configured_module = configured
6998 .get(module_id)
6999 .expect("added module id came from configured map");
7000 supervisor
7001 .supervise_configured_requested(
7002 configured_module.module_spec(),
7003 configured_module.enabled,
7004 configured_module.health.clone(),
7005 configured_module.drain_timeout_ms,
7006 configured_module.restart,
7007 configured_module.register_timeout,
7008 Some(requester.clone()),
7009 )
7010 .map_err(|err| {
7011 format!("failed to add module_id '{module_id}' during rescan: {err}")
7012 })?;
7013 }
7014
7015 Ok((
7016 SupervisorRescanResult {
7017 added,
7018 removed,
7019 changed_pending_reload,
7020 enabled_changes: enabled_changes.iter().cloned().collect(),
7021 unchanged,
7022 preview: false,
7023 restart_required: Vec::new(),
7026 capability_warnings: Vec::new(),
7027 module_actions: Vec::new(),
7028 earlier_pending: Vec::new(),
7029 expect: None,
7030 },
7031 changed,
7032 ))
7033 }
7034
7035 async fn handle_supervisor_set_enabled(
7036 &self,
7037 frame: Frame,
7038 module_id: String,
7039 enabled: bool,
7040 requester: Principal,
7041 ) -> Result<Vec<Frame>, RouterError> {
7042 let operation_lock = self.supervisor.operation_lock();
7043 let _operation_guard = operation_lock.lock().await;
7044 let Some(module) = self.supervisor.get(&module_id) else {
7045 return Ok(vec![control_error_frame(
7046 &frame,
7047 "unknown_module",
7048 format!("module_id '{module_id}' is not supervised"),
7049 )?]);
7050 };
7051
7052 self.route_outages.mark_operator_action(&module_id);
7055 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
7056 Ok(applied) => applied,
7057 Err(err) => {
7058 self.route_outages
7059 .operator_action_ended_unrefused(&module_id);
7060 return Ok(vec![control_error_frame(
7061 &frame,
7062 "target_unavailable",
7063 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
7064 )?]);
7065 }
7066 };
7067 if !applied {
7068 self.route_outages
7071 .operator_action_ended_unrefused(&module_id);
7072 }
7073
7074 self.capability_evaluator.wake_deadline_loop();
7075 self.refresh_capability_requirements();
7076 let response = ClientControlResponse::SupervisorAck { module_id, applied };
7077 Ok(vec![control_response_body_frame(
7078 &frame,
7079 &response,
7080 "ClientControlResponse::SupervisorAck",
7081 )?])
7082 }
7083
7084 async fn handle_supervisor_health_probe(
7085 &self,
7086 ctx: &RouteCtx,
7087 frame: Frame,
7088 module_id: String,
7089 ) -> Result<Vec<Frame>, RouterError> {
7090 self.refresh_capability_requirements();
7091 let Some(registration) = self
7092 .registry
7093 .get_module(&module_id)
7094 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
7095 else {
7096 return Ok(vec![control_error_frame(
7097 &frame,
7098 "unknown_module",
7099 format!("module_id '{module_id}' is not registered"),
7100 )?]);
7101 };
7102
7103 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
7116 {
7117 return Ok(vec![control_error_frame(
7118 &frame,
7119 "health_not_advertised",
7120 format!("module_id '{module_id}' did not advertise health.check"),
7121 )?]);
7122 }
7123
7124 self.observe_control_wait(
7125 ctx,
7126 Some(®istration.manifest.module_id),
7127 ops::SUPERVISOR_HEALTH_PROBE,
7128 async {
7129 let deadline = Instant::now() + self.health_probe_timeout;
7130 let pending = match self.forwarding.begin_module_control_rpc_for(
7131 &module_id,
7132 MODULE_CONTROL_OP_HEALTH_CHECK,
7133 deadline,
7134 ) {
7135 Ok(pending) => pending,
7136 Err(err) => {
7137 return Ok(vec![control_error_frame(
7138 &frame,
7139 forwarding_error_code(&err),
7140 err.to_string(),
7141 )?])
7142 }
7143 };
7144
7145 let PendingModuleControlRpc {
7146 endpoint,
7147 module_sink,
7148 negotiated_ver,
7149 corr: probe_corr,
7150 receiver,
7151 } = pending;
7152 let mut guard =
7153 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
7154 let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
7155 .map_err(|err| {
7156 RouterError::backend(
7157 0,
7158 frame.header.corr,
7159 format!("failed to encode health.check request: {err}"),
7160 )
7161 })?;
7162 let probe_frame = Frame::build_with_version(
7163 negotiated_ver,
7164 FrameType::Request,
7165 control_flags(),
7166 0,
7167 0,
7168 probe_corr,
7169 probe_body,
7170 )
7171 .map_err(RouterError::FrameBuild)?;
7172
7173 match timeout_at(deadline, module_sink.send(probe_frame)).await {
7176 Ok(Ok(())) => {}
7177 Ok(Err(err)) => {
7178 return Ok(vec![control_error_frame(
7179 &frame,
7180 "target_unavailable",
7181 err.to_string(),
7182 )?]);
7183 }
7184 Err(_) => {
7185 return Ok(vec![control_error_frame(
7186 &frame,
7187 "module_timeout",
7188 format!(
7189 "module_id '{module_id}' did not answer health.check within {:?}",
7190 self.health_probe_timeout
7191 ),
7192 )?]);
7193 }
7194 }
7195
7196 match timeout_at(deadline, receiver).await {
7197 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
7198 guard.disarm();
7199 let Some(report) = response.health_report() else {
7200 return Ok(vec![control_error_frame(
7201 &frame,
7202 "invalid_control_body",
7203 "health.check RPC returned a non-health response",
7204 )?]);
7205 };
7206 let HealthReport {
7211 status,
7212 detail,
7213 metrics,
7214 } = report;
7215 let capability_detail = self
7216 .capability_evaluator
7217 .required_problem_detail(&module_id);
7218 let response = ClientControlResponse::SupervisorHealthProbe {
7219 module_id,
7220 status,
7221 detail: append_capability_problem_detail(detail, capability_detail),
7222 metrics,
7223 };
7224 Ok(vec![control_response_body_frame(
7225 &frame,
7226 &response,
7227 "ClientControlResponse::SupervisorHealthProbe",
7228 )?])
7229 }
7230 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
7231 guard.disarm();
7232 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
7237 }
7238 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
7239 guard.disarm();
7240 Ok(vec![control_error_frame(
7241 &frame,
7242 "target_unavailable",
7243 message,
7244 )?])
7245 }
7246 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
7247 guard.disarm();
7248 Ok(vec![control_error_frame(
7249 &frame,
7250 "invalid_control_body",
7251 message,
7252 )?])
7253 }
7254 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
7255 guard.disarm();
7256 Ok(vec![control_error_frame(
7257 &frame,
7258 "invalid_control_body",
7259 format!("expected module-control op '{expected}', got '{actual}'"),
7260 )?])
7261 }
7262 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
7263 guard.disarm();
7264 Ok(vec![control_error_frame(
7265 &frame,
7266 "module_timeout",
7267 format!(
7268 "module_id '{module_id}' answered health.check after {:?}",
7269 self.health_probe_timeout
7270 ),
7271 )?])
7272 }
7273 Ok(Err(_)) => Ok(vec![control_error_frame(
7274 &frame,
7275 "target_unavailable",
7276 "health.check waiter was canceled before the module responded",
7277 )?]),
7278 Err(_) => Ok(vec![control_error_frame(
7279 &frame,
7280 "module_timeout",
7281 format!(
7282 "module_id '{module_id}' did not answer health.check within {:?}",
7283 self.health_probe_timeout
7284 ),
7285 )?]),
7286 }
7287 },
7288 )
7289 .await
7290 }
7291
7292 fn supervisor_status(
7293 &self,
7294 module_id: &str,
7295 corr: u64,
7296 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
7297 self.supervisor
7298 .get(module_id)
7299 .map(|module| {
7300 let warming = module.is_warming_for_control("status").map_err(|err| {
7301 RouterError::backend(
7302 0,
7303 corr,
7304 format!(
7305 "failed to read supervisor warming state for module_id '{module_id}': {err}"
7306 ),
7307 )
7308 })?;
7309 module.status_for_control("status").map_err(|err| {
7310 RouterError::backend(
7311 0,
7312 corr,
7313 format!(
7314 "failed to read supervisor status for module_id '{module_id}': {err}"
7315 ),
7316 )
7317 }).map(|status| (status, warming))
7318 })
7319 .transpose()
7320 }
7321
7322 fn guard_module_control_op(
7323 &self,
7324 frame: &Frame,
7325 module_id: &str,
7326 op: &str,
7327 ) -> Result<Option<Frame>, RouterError> {
7328 if self.module_grants_op(module_id, op, frame.header.corr)? {
7329 return Ok(None);
7330 }
7331
7332 Ok(Some(control_error_frame(
7333 frame,
7334 "op_not_allowed",
7335 format!("module_id '{module_id}' did not grant control op '{op}'"),
7336 )?))
7337 }
7338
7339 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
7340 let Some(registration) = self
7341 .registry
7342 .get_module(module_id)
7343 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
7344 else {
7345 return Ok(false);
7346 };
7347 Ok(module_registration_grants_op(®istration.control_ops, op))
7348 }
7349
7350 fn handle_status_update(
7351 &self,
7352 endpoint: ModuleEndpointId,
7353 frame: Frame,
7354 ) -> Result<Vec<Frame>, RouterError> {
7355 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
7356 Ok(update) => update,
7357 Err(err) => {
7358 if is_known_module_push_op(&frame.body) {
7363 return Ok(vec![control_error_frame(
7364 &frame,
7365 "invalid_control_body",
7366 format!("malformed module control push body: {err}"),
7367 )?]);
7368 }
7369 return Ok(Vec::new());
7370 }
7371 };
7372
7373 match update {
7374 ModuleControlPush::RouteStatus {
7375 route_channel,
7376 route_epoch,
7377 status,
7378 } => {
7379 self.forwarding
7380 .cache_status(endpoint, route_channel, route_epoch, status)
7381 .map_err(RouterError::Forwarding)?;
7382 }
7383 }
7384 Ok(Vec::new())
7385 }
7386
7387 fn handle_route_poll(
7388 &self,
7389 ctx: &RouteCtx,
7390 frame: Frame,
7391 route_channel: u16,
7392 route_epoch: u32,
7393 kind: PollKind,
7394 ) -> Result<Vec<Frame>, RouterError> {
7395 let snapshot = self
7396 .forwarding
7397 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7398 .map_err(RouterError::Forwarding)?;
7399 let response = match (kind, snapshot) {
7400 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7401 ClientControlResponse::RoutePoll {
7402 route_channel,
7403 route_epoch,
7404 status,
7405 live: None,
7406 }
7407 }
7408 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7409 route_channel,
7410 route_epoch,
7411 status: None,
7412 live: None,
7413 },
7414 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7415 let live = self
7441 .process_liveness
7442 .as_ref()
7443 .and_then(|source| source.process_live(&module_id))
7444 .unwrap_or(true);
7445 ClientControlResponse::RoutePoll {
7446 route_channel,
7447 route_epoch,
7448 status: None,
7449 live: Some(live),
7450 }
7451 }
7452 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7453 route_channel,
7454 route_epoch,
7455 status: None,
7456 live: Some(false),
7457 },
7458 };
7459
7460 Ok(vec![control_response_body_frame(
7461 &frame,
7462 &response,
7463 "ClientControlResponse::RoutePoll",
7464 )?])
7465 }
7466
7467 pub(crate) fn observe_module_control_completion(
7468 &self,
7469 completion: ModuleControlRpcCompletion,
7470 ) -> bool {
7471 match completion {
7472 ModuleControlRpcCompletion::Unknown => false,
7473 ModuleControlRpcCompletion::Settled => true,
7474 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7475 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7476 info!(
7477 module_id = %module_id,
7478 latency_ms,
7479 "late health.check answer proves the module is alive"
7480 );
7481 match self
7482 .supervisor
7483 .record_late_health_answer(&module_id, latency_ms)
7484 {
7485 Ok(true) => {}
7486 Ok(false) => debug!(
7487 module_id = %module_id,
7488 latency_ms,
7489 "late health.check answer has no active supervisor snapshot"
7490 ),
7491 Err(err) => warn!(
7492 module_id = %module_id,
7493 latency_ms,
7494 error = %err,
7495 "failed to record late health.check answer"
7496 ),
7497 }
7498 true
7499 }
7500 }
7501 }
7502
7503 fn refuse_to_end_module_connection_for_a_client(
7524 &self,
7525 module_connection_id: ConnectionId,
7526 corr: u64,
7527 err: ForwardingError,
7528 ) -> Result<(), RouterError> {
7529 if let ForwardingError::ConnectionClosing { connection_id } = err {
7530 if connection_id != module_connection_id {
7531 warn!(
7532 module_connection_id = module_connection_id.get(),
7533 client_connection_id = connection_id.get(),
7534 corr,
7535 "dropping a route.bind response for a closing client; the module connection keeps serving"
7536 );
7537 return Ok(());
7538 }
7539 }
7540 Err(RouterError::Forwarding(err))
7541 }
7542
7543 fn handle_module_relay_response(
7544 &self,
7545 connection_id: ConnectionId,
7546 frame: Frame,
7547 ) -> Result<Vec<Frame>, RouterError> {
7548 let mut secondary_error = None;
7549 let outcome = match frame.header.ty {
7550 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7551 Ok(probe) if probe.op == "route.bind" => {
7552 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7553 Ok(ModuleControlResponse::RouteBindAck {}) => {
7554 RouteBindRelayOutcome::Accepted
7555 }
7556 Ok(other) => {
7557 let message =
7558 format!("route.bind response carried unexpected body: {other:?}");
7559 secondary_error = Some(control_error_frame(
7560 &frame,
7561 "invalid_control_body",
7562 message.clone(),
7563 )?);
7564 RouteBindRelayOutcome::ModuleGone(message)
7565 }
7566 Err(err) => {
7567 let message = format!("malformed route.bind response body: {err}");
7568 secondary_error = Some(control_error_frame(
7569 &frame,
7570 "invalid_control_body",
7571 message.clone(),
7572 )?);
7573 RouteBindRelayOutcome::ModuleGone(message)
7574 }
7575 }
7576 }
7577 Ok(probe) => {
7578 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7579 {
7580 Ok(response) => ModuleControlRpcOutcome::Response(response),
7581 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7582 "malformed {} response body: {err}",
7583 probe.op
7584 )),
7585 };
7586 let completion = self
7587 .forwarding
7588 .complete_module_control_rpc(
7589 connection_id,
7590 frame.header.corr,
7591 Some(&probe.op),
7592 outcome,
7593 )
7594 .map_err(RouterError::Forwarding)?;
7595 if !self.observe_module_control_completion(completion) {
7596 debug!(
7597 connection_id = connection_id.get(),
7598 corr = frame.header.corr,
7599 op = %probe.op,
7600 "dropping late or unknown module-control RPC response"
7601 );
7602 }
7603 return Ok(Vec::new());
7604 }
7605 Err(err) => {
7606 if let Some(expected_op) = self
7607 .forwarding
7608 .pending_module_control_op(connection_id, frame.header.corr)
7609 .map_err(RouterError::Forwarding)?
7610 {
7611 let completion = self
7612 .forwarding
7613 .complete_module_control_rpc(
7614 connection_id,
7615 frame.header.corr,
7616 None,
7617 ModuleControlRpcOutcome::MalformedResponse(format!(
7618 "malformed {expected_op} response body: {err}"
7619 )),
7620 )
7621 .map_err(RouterError::Forwarding)?;
7622 if !self.observe_module_control_completion(completion) {
7623 debug!(
7624 connection_id = connection_id.get(),
7625 corr = frame.header.corr,
7626 "dropping late malformed module-control RPC response"
7627 );
7628 }
7629 return Ok(Vec::new());
7630 }
7631 let message = format!("malformed route.bind response body: {err}");
7632 secondary_error = Some(control_error_frame(
7633 &frame,
7634 "invalid_control_body",
7635 message.clone(),
7636 )?);
7637 RouteBindRelayOutcome::ModuleGone(message)
7638 }
7639 },
7640 FrameType::Error => {
7641 if self
7642 .forwarding
7643 .pending_module_control_op(connection_id, frame.header.corr)
7644 .map_err(RouterError::Forwarding)?
7645 .is_some()
7646 {
7647 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7648 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7649 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7650 "malformed module-control ERROR body: {err}"
7651 )),
7652 };
7653 let completion = self
7654 .forwarding
7655 .complete_module_control_rpc(
7656 connection_id,
7657 frame.header.corr,
7658 None,
7659 outcome,
7660 )
7661 .map_err(RouterError::Forwarding)?;
7662 if !self.observe_module_control_completion(completion) {
7663 debug!(
7664 connection_id = connection_id.get(),
7665 corr = frame.header.corr,
7666 "dropping late or unknown module-control RPC error"
7667 );
7668 }
7669 return Ok(Vec::new());
7670 }
7671 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7672 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7673 Err(err) => {
7674 let message = format!("malformed route.bind ERROR body: {err}");
7675 secondary_error = Some(control_error_frame(
7676 &frame,
7677 "invalid_control_body",
7678 message.clone(),
7679 )?);
7680 RouteBindRelayOutcome::ModuleGone(message)
7681 }
7682 }
7683 }
7684 ty => {
7685 return Ok(vec![control_error_frame(
7686 &frame,
7687 "unsupported_control_frame",
7688 format!("unsupported module channel-0 frame {ty:?}"),
7689 )?])
7690 }
7691 };
7692
7693 let settled =
7694 self.forwarding
7695 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7696 let completion = match settled {
7697 Ok(completion) => completion,
7698 Err(err) => {
7699 self.refuse_to_end_module_connection_for_a_client(
7700 connection_id,
7701 frame.header.corr,
7702 err,
7703 )?;
7704 return Ok(secondary_error.into_iter().collect());
7705 }
7706 };
7707 if let Some(target) = completion.abandoned.as_ref() {
7708 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7709 }
7710 if !completion.settled {
7711 debug!(
7712 connection_id = connection_id.get(),
7713 corr = frame.header.corr,
7714 frame_type = ?frame.header.ty,
7715 "dropping late or unknown route.bind relay response"
7716 );
7717 }
7718 Ok(secondary_error.into_iter().collect())
7719 }
7720
7721 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7722 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7723 self.cleanup_connection_with_end_reason(
7727 connection_id,
7728 RegistrationEndReason::ExplicitGoodbye,
7729 )
7730 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7731 Ok(Vec::new())
7732 }
7733}
7734
7735impl Default for ControlHandler {
7736 fn default() -> Self {
7737 Self::new(Arc::new(Registry::default()))
7738 }
7739}
7740
7741impl crate::supervise::SwapPromotionObserver for ControlHandler {
7742 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7743 self.apply_registration_capabilities(registration);
7744 }
7745}
7746
7747fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7748 CapabilityRequirementStatus {
7749 consumer: status.consumer,
7750 capability: status.capability,
7751 need: match status.need {
7752 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7753 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7754 },
7755 verdict: status.verdict.as_str().to_string(),
7756 episode_seq: status.episode_seq,
7757 config_satisfiable: status.config_satisfiable,
7758 runtime_available: status.runtime_available,
7759 detail: status.detail,
7760 }
7761}
7762
7763fn append_capability_problem_detail(
7764 detail: Option<String>,
7765 capability_detail: Option<String>,
7766) -> Option<String> {
7767 match (detail, capability_detail) {
7768 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7769 (Some(detail), None) => Some(detail),
7770 (None, Some(capability_detail)) => Some(capability_detail),
7771 (None, None) => None,
7772 }
7773}
7774
7775fn subc_ops() -> Vec<String> {
7776 SUBC_CONTROL_OPS
7777 .iter()
7778 .map(|op| (*op).to_string())
7779 .collect()
7780}
7781
7782fn module_subc_ops() -> Vec<String> {
7783 SUBC_CONTROL_OPS
7784 .iter()
7785 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7786 .map(|op| (*op).to_string())
7787 .collect()
7788}
7789
7790#[cfg(test)]
7791fn module_baseline_control_ops() -> Vec<String> {
7792 MODULE_BASELINE_CONTROL_OPS
7793 .iter()
7794 .map(|op| (*op).to_string())
7795 .collect()
7796}
7797
7798fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7799 let mut seen = HashSet::new();
7800 let mut effective = Vec::new();
7801 for op in MODULE_BASELINE_CONTROL_OPS {
7802 if seen.insert((*op).to_string()) {
7803 effective.push((*op).to_string());
7804 }
7805 }
7806 for op in declared.unwrap_or_default() {
7807 if seen.insert(op.clone()) {
7808 effective.push(op);
7809 }
7810 }
7811 effective
7812}
7813
7814fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7815 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7816}
7817
7818fn target_module_id(target: &RouteTarget) -> &str {
7819 match target {
7820 RouteTarget::ToolProvider { module_id }
7821 | RouteTarget::ManagementSurface { module_id }
7822 | RouteTarget::InternalService { module_id, .. } => module_id,
7823 }
7824}
7825
7826fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7827 roles.iter().any(|role| match (target, role) {
7828 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7829 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7830 (
7831 RouteTarget::InternalService { service_id, .. },
7832 ProviderRole::InternalService {
7833 service_id: provided,
7834 ..
7835 },
7836 ) => service_id == provided,
7837 _ => false,
7838 })
7839}
7840
7841fn is_routable_role(role: &ProviderRole) -> bool {
7842 matches!(
7843 role,
7844 ProviderRole::ToolProvider { .. }
7845 | ProviderRole::ManagementSurface { .. }
7846 | ProviderRole::InternalService { .. }
7847 )
7848}
7849
7850#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7851enum ControlRequestBodyError {
7852 UnknownOp,
7853 InvalidBody,
7854}
7855
7856#[derive(Debug, Deserialize)]
7857struct ControlOpProbe {
7858 op: String,
7859}
7860
7861const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7864
7865fn is_known_module_push_op(body: &[u8]) -> bool {
7866 serde_json::from_slice::<ControlOpProbe>(body)
7867 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7868 .unwrap_or(false)
7869}
7870
7871fn is_known_module_request_op(body: &[u8]) -> bool {
7872 serde_json::from_slice::<ControlOpProbe>(body)
7873 .map(|probe| is_module_to_subc_op(&probe.op))
7874 .unwrap_or(false)
7875}
7876
7877fn is_module_to_subc_op(op: &str) -> bool {
7878 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7879}
7880
7881fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7882 debug!(
7883 op = %op,
7884 connection_id = connection_id.get(),
7885 corr,
7886 "control dispatch"
7887 );
7888}
7889
7890fn log_slow_control_dispatch(
7891 dispatch_started_at: Option<StdInstant>,
7892 op: &'static str,
7893 connection_id: ConnectionId,
7894 corr: u64,
7895 route_open_timings: Option<&RouteOpenTimings>,
7896) {
7897 let Some(dispatch_started_at) = dispatch_started_at else {
7898 return;
7899 };
7900 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7902 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7903 if let Some(timings) = route_open_timings {
7904 warn!(
7905 op = %op,
7906 connection_id = connection_id.get(),
7907 corr,
7908 elapsed_ms = elapsed.as_millis() as u64,
7909 module_id = ?timings.module_id,
7910 principal = %timings.principal,
7911 admission_ms = timings.admission.as_millis() as u64,
7912 project_root_ms = timings.project_root.as_millis() as u64,
7913 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7914 bind_send_ms = timings.bind_send.as_millis() as u64,
7915 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7916 "slow control dispatch"
7917 );
7918 return;
7919 }
7920 warn!(
7921 op = %op,
7922 connection_id = connection_id.get(),
7923 corr,
7924 elapsed_ms = elapsed.as_millis() as u64,
7925 "slow control dispatch"
7926 );
7927 }
7928}
7929
7930#[cfg(test)]
7932type TestCallerLookup =
7933 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7934
7935fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7937 match request {
7938 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7939 Some(PrivilegedOperation::Restart { module_id })
7940 }
7941 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7942 Some(PrivilegedOperation::Swap { module_id })
7943 }
7944 ClientControlRequest::SupervisorReload { module_id } => {
7945 Some(PrivilegedOperation::Reload { module_id })
7946 }
7947 ClientControlRequest::SupervisorRescan { preview } => {
7948 if *preview {
7949 None
7950 } else {
7951 Some(PrivilegedOperation::Rescan)
7952 }
7953 }
7954 ClientControlRequest::SupervisorRescanChecked {
7959 preview,
7960 candidate,
7961 expect,
7962 } => (!*preview && candidate.is_none() && expect.is_some())
7963 .then_some(PrivilegedOperation::Rescan),
7964 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7965 Some(PrivilegedOperation::SetEnabled {
7966 module_id,
7967 enabled: *enabled,
7968 })
7969 }
7970 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7971 Some(PrivilegedOperation::ReleaseReserved { module_id })
7972 }
7973 ClientControlRequest::ServerDescribe {}
7974 | ClientControlRequest::DaemonShutdownIntent { .. }
7977 | ClientControlRequest::CatalogList { .. }
7978 | ClientControlRequest::RouteOpen { .. }
7979 | ClientControlRequest::RoutePoll { .. }
7980 | ClientControlRequest::SupervisorList {}
7981 | ClientControlRequest::SupervisorHealth {}
7982 | ClientControlRequest::SupervisorHealthProbe { .. }
7983 | ClientControlRequest::SupervisorRoutes { .. }
7984 | ClientControlRequest::SupervisorProvenance { .. }
7985 | ClientControlRequest::SupervisorStderrTail { .. }
7986 | ClientControlRequest::SupervisorTerminals { .. }
7987 | ClientControlRequest::SupervisorSpawnSnapshot {}
7988 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7989 }
7990}
7991
7992fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7993 match request {
7994 ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
7995 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7996 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7997 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7998 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7999 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
8000 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
8001 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
8002 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
8003 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
8004 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
8005 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
8006 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
8007 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
8008 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
8009 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
8010 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
8011 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
8012 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
8013 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
8014 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
8015 }
8016}
8017
8018struct RouteOpenCaller {
8026 principal: String,
8030 scope: String,
8033 harness: String,
8036}
8037
8038impl RouteOpenCaller {
8039 fn new(
8040 principal: Option<&Principal>,
8041 scope: Option<&ScopeSelector>,
8042 identity: &BindIdentity,
8043 ) -> Self {
8044 Self {
8045 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
8046 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
8047 harness: bounded_log_text(&identity.harness),
8048 }
8049 }
8050
8051 fn unknown() -> Self {
8054 Self {
8055 principal: "unknown".to_string(),
8056 scope: "unknown".to_string(),
8057 harness: "unknown".to_string(),
8058 }
8059 }
8060}
8061
8062const MAX_LOGGED_TEXT_CHARS: usize = 64;
8065
8066fn bounded_log_text(value: &str) -> String {
8070 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
8071 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
8072 bounded.push('…');
8073 }
8074 format!("{bounded:?}")
8075}
8076
8077fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
8083 use subc_protocol::scope::ScopeKind;
8084 let kind = match stamp.kind {
8085 ScopeKind::Head => "head",
8086 ScopeKind::Worker => "worker",
8087 ScopeKind::Ephemeral => "ephemeral",
8088 };
8089 format!(
8090 "{kind}/{}/{}@{}",
8091 principal_label(&stamp.owner),
8092 bounded_log_text(&stamp.scope_ref),
8093 stamp.scope_epoch
8094 )
8095}
8096
8097fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
8104 let epoch = selector
8105 .scope_epoch
8106 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
8107 format!(
8108 "requested/{}/{}@{epoch}",
8109 bounded_log_text(&principal_label(&selector.owner)),
8110 bounded_log_text(&selector.scope_ref),
8111 )
8112}
8113
8114fn principal_label(principal: &Principal) -> String {
8115 match principal {
8116 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
8117 Principal::Direct => "direct".to_string(),
8118 other => format!("{other:?}"),
8119 }
8120}
8121
8122fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
8123 match request {
8124 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
8125 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
8126 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
8127 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
8128 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
8129 }
8130}
8131
8132fn parse_client_control_request(
8133 body: &[u8],
8134) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
8135 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
8136 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8137 Ok(probe)
8138 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
8139 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
8140 {
8141 ControlRequestBodyError::InvalidBody
8142 }
8143 Ok(_) => ControlRequestBodyError::UnknownOp,
8144 Err(_) => ControlRequestBodyError::InvalidBody,
8145 };
8146 (err, classification)
8147 })
8148}
8149
8150fn parse_module_control_request_from_module(
8151 body: &[u8],
8152) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
8153 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
8154 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8155 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
8156 Ok(_) => ControlRequestBodyError::UnknownOp,
8157 Err(_) => ControlRequestBodyError::InvalidBody,
8158 };
8159 (err, classification)
8160 })
8161}
8162
8163#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
8164enum ProviderRoleKind {
8165 ToolProvider,
8166 PipelineStage,
8167 ManagementSurface,
8168 InternalService,
8169}
8170
8171fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
8172 match role {
8173 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
8174 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
8175 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
8176 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
8177 }
8178}
8179
8180fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
8181 roles.iter().map(provider_role_kind).collect()
8182}
8183
8184const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
8187
8188#[derive(Debug, Default, PartialEq, Eq)]
8190struct ScopeOutcomeCounts {
8191 created: usize,
8192 replaced: usize,
8193 updated: usize,
8194 unchanged: usize,
8195 refused: usize,
8196}
8197
8198impl ScopeOutcomeCounts {
8199 fn of(results: &[ScopeRecordResult]) -> Self {
8200 let mut counts = Self::default();
8201 for result in results {
8202 let slot = match result.outcome {
8203 ScopeRecordOutcome::Created => &mut counts.created,
8204 ScopeRecordOutcome::Replaced => &mut counts.replaced,
8205 ScopeRecordOutcome::Updated => &mut counts.updated,
8206 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
8207 ScopeRecordOutcome::Refused => &mut counts.refused,
8208 };
8209 *slot += 1;
8210 }
8211 counts
8212 }
8213}
8214
8215#[cfg(test)]
8216mod scope_outcome_count_tests {
8217 use super::*;
8218
8219 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
8220 ScopeRecordResult {
8221 scope_ref: "r".to_string(),
8222 scope_epoch: 1,
8223 outcome,
8224 code: None,
8225 message: None,
8226 version: None,
8227 parent_state: None,
8228 }
8229 }
8230
8231 #[test]
8234 fn every_outcome_is_counted_in_its_own_field() {
8235 let results = [
8236 result(ScopeRecordOutcome::Created),
8237 result(ScopeRecordOutcome::Created),
8238 result(ScopeRecordOutcome::Replaced),
8239 result(ScopeRecordOutcome::Updated),
8240 result(ScopeRecordOutcome::Unchanged),
8241 result(ScopeRecordOutcome::Refused),
8242 result(ScopeRecordOutcome::Refused),
8243 result(ScopeRecordOutcome::Refused),
8244 ];
8245 assert_eq!(
8246 ScopeOutcomeCounts::of(&results),
8247 ScopeOutcomeCounts {
8248 created: 2,
8249 replaced: 1,
8250 updated: 1,
8251 unchanged: 1,
8252 refused: 3,
8253 }
8254 );
8255 }
8256}
8257
8258fn capability_census_trigger(
8262 old: Option<&CapabilityDeclarations>,
8263 new: Option<&CapabilityDeclarations>,
8264) -> bool {
8265 let old_provides = old
8266 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
8267 .unwrap_or_default();
8268 let old_denies = old
8269 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
8270 .unwrap_or_default();
8271 let new = new.cloned().unwrap_or(CapabilityDeclarations {
8272 provides: Vec::new(),
8273 requires: Vec::new(),
8274 must_never_reach: Vec::new(),
8275 });
8276
8277 new.provides
8278 .iter()
8279 .any(|capability| !old_provides.contains(capability))
8280 || new
8281 .must_never_reach
8282 .iter()
8283 .any(|capability| !old_denies.contains(capability))
8284}
8285
8286fn denied_capability<'a>(
8289 opening_manifest: &'a ModuleManifest,
8290 target_manifest: &ModuleManifest,
8291) -> Option<&'a str> {
8292 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
8293 let target_capabilities = target_manifest.capabilities.as_ref()?;
8294 opening_capabilities
8295 .must_never_reach
8296 .iter()
8297 .find(|denied| {
8298 target_capabilities
8299 .provides
8300 .iter()
8301 .any(|provided| provided == *denied)
8302 })
8303 .map(String::as_str)
8304}
8305
8306fn catalog_update_frozen_field_message(
8307 registered: &ModuleManifest,
8308 provides: &[ProviderRole],
8309) -> Option<String> {
8310 let old_has_provides = !registered.provides.is_empty();
8311 let new_has_provides = !provides.is_empty();
8312 if old_has_provides != new_has_provides {
8313 return Some(format!(
8314 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
8315 registered.module_id
8316 ));
8317 }
8318
8319 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
8320 return Some(format!(
8321 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
8322 registered.module_id
8323 ));
8324 }
8325
8326 let registered_concurrency = manifest_concurrency(registered);
8327 let mut candidate = registered.clone();
8328 candidate.provides = provides.to_vec();
8329 let candidate_concurrency = manifest_concurrency(&candidate);
8330 if candidate_concurrency != registered_concurrency {
8331 return Some(format!(
8332 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
8333 registered.module_id, registered_concurrency, candidate_concurrency
8334 ));
8335 }
8336
8337 None
8340}
8341
8342fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
8343 manifest.provides.iter().any(is_routable_role)
8344}
8345
8346fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
8352 manifest
8353 .provides
8354 .iter()
8355 .find_map(|provider| match provider {
8356 ProviderRole::ToolProvider { concurrency, .. }
8357 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
8358 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
8359 })
8360 .unwrap_or(Concurrency::ModuleManaged)
8361}
8362
8363fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
8370 let has_management_surface = manifest
8371 .provides
8372 .iter()
8373 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
8374 if !has_management_surface {
8375 return false;
8376 }
8377 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
8378 return false;
8379 };
8380 let Some(provides) = raw
8381 .get("manifest")
8382 .and_then(|manifest| manifest.get("provides"))
8383 .and_then(serde_json::Value::as_array)
8384 else {
8385 return false;
8386 };
8387 provides.iter().any(|role| {
8392 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8393 && role.get("concurrency").is_none()
8394 })
8395}
8396
8397fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8398 if peer_version != PROTOCOL_VERSION {
8399 return Err(format!(
8400 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8401 ));
8402 }
8403 Ok(PROTOCOL_VERSION)
8404}
8405
8406fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8407 Frame::build_with_version(
8408 response_version(frame),
8409 FrameType::Pong,
8410 frame.header.flags,
8411 0,
8412 0,
8413 frame.header.corr,
8414 Vec::new(),
8415 )
8416 .map_err(RouterError::FrameBuild)
8417}
8418
8419pub(crate) fn control_error_frame(
8420 frame: &Frame,
8421 code: &'static str,
8422 message: impl Into<String>,
8423) -> Result<Frame, RouterError> {
8424 control_error_body_frame(
8425 frame,
8426 ErrorBody {
8427 code: code.to_string(),
8428 message: message.into(),
8429 detail: None,
8430 },
8431 )
8432}
8433
8434fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8442 control_error_frame_with_flags(frame, error, daemon_error_flags())
8443}
8444
8445fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8451 control_error_frame_with_flags(frame, error, control_flags())
8452}
8453
8454fn control_error_frame_with_flags(
8455 frame: &Frame,
8456 error: ErrorBody,
8457 flags: Flags,
8458) -> Result<Frame, RouterError> {
8459 let body = serde_json::to_vec(&error).map_err(|err| {
8460 RouterError::backend(
8461 0,
8462 frame.header.corr,
8463 format!("failed to encode control ERROR: {err}"),
8464 )
8465 })?;
8466
8467 Frame::build_with_version(
8468 response_version(frame),
8469 FrameType::Error,
8470 flags,
8471 0,
8472 0,
8473 frame.header.corr,
8474 body,
8475 )
8476 .map_err(RouterError::FrameBuild)
8477}
8478
8479pub(crate) fn control_response_body_frame<T: Serialize>(
8480 frame: &Frame,
8481 reply: &T,
8482 label: &'static str,
8483) -> Result<Frame, RouterError> {
8484 let body = serde_json::to_vec(reply).map_err(|err| {
8485 RouterError::backend(
8486 0,
8487 frame.header.corr,
8488 format!("failed to encode {label}: {err}"),
8489 )
8490 })?;
8491
8492 Frame::build_with_version(
8493 response_version(frame),
8494 FrameType::Response,
8495 control_flags(),
8496 0,
8497 0,
8498 frame.header.corr,
8499 body,
8500 )
8501 .map_err(RouterError::FrameBuild)
8502}
8503
8504fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8534 match err {
8535 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8536 ForwardingError::NoModuleConnection => "target_unavailable",
8537 ForwardingError::ModuleReloading { .. } => "module_reloading",
8538 ForwardingError::ClientRouteChannelExhausted { .. }
8539 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8540 ForwardingError::StaleModuleEndpoint
8541 | ForwardingError::UnknownReservation { .. }
8542 | ForwardingError::ConnectionClosing { .. }
8543 | ForwardingError::ClientEgressClosed { .. }
8544 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8545 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8548 ForwardingError::RelayCorrelationExhausted
8549 | ForwardingError::RouteOpenBuild(_)
8550 | ForwardingError::Poisoned => "forwarding_error",
8551 }
8552}
8553
8554fn response_version(frame: &Frame) -> u8 {
8555 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8556 frame.header.ver
8557 } else {
8558 PROTOCOL_VERSION
8559 }
8560}
8561
8562fn control_flags() -> Flags {
8563 Flags::new(false, Priority::Passive, false)
8564}
8565
8566fn send_goodbye_target_best_effort(
8571 counters: &DaemonCounters,
8572 target: &GoodbyeTarget,
8573 context: &'static str,
8574) {
8575 let Ok(frame) = Frame::build_with_version(
8576 target.negotiated_ver,
8577 FrameType::Goodbye,
8578 control_flags(),
8579 target.channel,
8580 target.epoch,
8581 0,
8582 Vec::new(),
8583 ) else {
8584 return;
8585 };
8586 crate::forwarding::send_module_route_goodbye(
8587 counters,
8588 &target.sink,
8589 frame,
8590 target.module_id.as_deref(),
8591 context,
8592 );
8593}
8594
8595pub(crate) fn send_route_control_pushes(
8596 forwarding: &ForwardingTable,
8597 routes: Vec<EndpointRoute>,
8598 push: ClientControlPush,
8599) {
8600 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8601 for route in routes {
8602 let target = route.goodbye_target;
8603 if let Some((existing, channels)) = targets
8604 .iter_mut()
8605 .find(|(existing, _)| existing.connection_id == target.connection_id)
8606 {
8607 debug_assert_eq!(
8608 existing.negotiated_ver, target.negotiated_ver,
8609 "one connection cannot negotiate multiple frame versions"
8610 );
8611 if !channels.contains(&target.channel) {
8612 channels.push(target.channel);
8613 }
8614 continue;
8615 }
8616 let channel = target.channel;
8617 targets.push((target, vec![channel]));
8618 }
8619 for (target, mut channels) in targets {
8620 channels.sort_unstable();
8621 let mut push = push.clone();
8622 match &mut push {
8623 ClientControlPush::RouteClosing {
8624 channels: covered, ..
8625 }
8626 | ClientControlPush::RouteClosed {
8627 channels: covered, ..
8628 } => *covered = channels,
8629 }
8630 let body = match serde_json::to_vec(&push) {
8631 Ok(body) => body,
8632 Err(err) => {
8633 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8634 continue;
8635 }
8636 };
8637 let frame = match Frame::build_with_version(
8638 target.negotiated_ver,
8639 FrameType::Push,
8640 control_flags(),
8641 0,
8642 0,
8643 0,
8644 body.clone(),
8645 ) {
8646 Ok(frame) => frame,
8647 Err(err) => {
8648 warn!(
8649 route_channel = target.channel,
8650 error = %err,
8651 "failed to build route lifecycle control PUSH frame"
8652 );
8653 continue;
8654 }
8655 };
8656 if let Err(err) = target.sink.try_send(frame) {
8657 if target.close_on_delivery_failure() {
8658 warn!(
8659 target_connection_id = target.connection_id.get(),
8660 route_channel = target.channel,
8661 error = %err,
8662 "route lifecycle control PUSH was not delivered to client; closing target connection"
8663 );
8664 let _ = forwarding.escalate_client_delivery_failure(
8665 target.connection_id,
8666 target.channel,
8667 target.epoch,
8668 CloseReason::new(
8669 "route_lifecycle_push_delivery_failed",
8670 format!(
8671 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8672 target.channel
8673 ),
8674 ),
8675 crate::forwarding::UndeliveredFrame {
8676 module_id: target.module_id.as_deref(),
8677 sink: &target.sink,
8678 },
8679 );
8680 }
8681 }
8682 }
8683}
8684
8685#[cfg(test)]
8686mod tests {
8687 #[tokio::test]
8688 async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
8689 let registry = Arc::new(Registry::default());
8690 let handler = ControlHandler::new(registry);
8691 assert_eq!(
8692 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8693 None
8694 );
8695 let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
8696 hello_via_sink(
8697 &handler,
8698 &ctx,
8699 &mut rx,
8700 hello_frame("daemon", PROTOCOL_VERSION, 1),
8701 )
8702 .await;
8703 assert_eq!(
8704 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8705 Some("daemon".into())
8706 );
8707 assert_eq!(
8708 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
8709 None
8710 );
8711 assert_eq!(
8712 handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
8713 None,
8714 "a registered module is not necessarily supervised"
8715 );
8716 assert_eq!(
8717 handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
8718 None,
8719 "aggregate provenance has no single target"
8720 );
8721 assert_eq!(
8722 handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
8723 None
8724 );
8725 }
8726
8727 mod peer_manifest {
8728 include!("peer_manifest_tests.rs");
8729 }
8730 mod activity_read {
8731 include!("activity_read_tests.rs");
8732 }
8733
8734 mod lifecycle_activity {
8735 include!("control_lifecycle_tests.rs");
8736 }
8737 #[cfg(unix)]
8738 #[tokio::test]
8739 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8740 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8741 let path = dir.join("subc.jsonc");
8742 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8743 "program":"/bin/sleep","args":["60"],"protocol":"none",
8744 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8745 }}}).to_string()).unwrap();
8746 let mut configured = crate::daemon_config::load(&path)
8747 .unwrap()
8748 .unwrap()
8749 .modules
8750 .pop()
8751 .unwrap();
8752 let registry = std::sync::Arc::new(crate::Registry::default());
8753 let handle = crate::SupervisorHandle::new();
8754 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8755 .with_handle(handle.clone());
8756 let module = supervisor
8757 .supervise_configured_with_health(
8758 configured.module_spec(),
8759 true,
8760 configured.health.clone(),
8761 None,
8762 configured.restart,
8763 )
8764 .unwrap();
8765 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8766 let before = module.status().unwrap().pid;
8767 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8768 configured.health.cadence = std::time::Duration::from_secs(3600);
8769 let health_only = handler
8770 .reconcile_supervised_modules(
8771 &supervisor,
8772 vec![configured.clone()],
8773 false,
8774 Principal::Direct,
8775 )
8776 .await
8777 .unwrap();
8778 assert!(
8779 health_only.0.changed_pending_reload.is_empty(),
8780 "health policy is already applied live"
8781 );
8782 assert_eq!(module.status().unwrap().pid, before);
8783 assert_eq!(
8784 module.configuration().unwrap().1.http,
8785 configured.health.http
8786 );
8787 configured.args = vec!["61".into()];
8788 let launch = handler
8789 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8790 .await
8791 .unwrap();
8792 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8793 assert_eq!(
8794 module.status().unwrap().pid,
8795 before,
8796 "a launch edit is stored until reload"
8797 );
8798 module.drain().await.unwrap();
8799 }
8800 use cortexkit_test_support::ScratchDir;
8801 use std::{
8802 collections::BTreeMap,
8803 fmt,
8804 path::PathBuf,
8805 sync::{Arc, Mutex},
8806 time::Duration,
8807 };
8808
8809 use serde_json::{json, Value};
8810 use subc_protocol::{
8811 manifest::{
8812 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8813 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8814 },
8815 session::HealthStatus,
8816 FrameType,
8817 };
8818
8819 use super::*;
8820 use crate::{
8821 forwarding::{DataRoute, DataRouteState},
8822 registry::ChannelState,
8823 router::FrameSink,
8824 stderr_tail::DEFAULT_MAX_LINE_BYTES,
8825 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8826 RouteCtx, Router,
8827 };
8828 use tokio::{
8829 sync::mpsc,
8830 time::{sleep, Instant},
8831 };
8832 use tracing::{
8833 field::{Field, Visit},
8834 Event, Subscriber,
8835 };
8836 use tracing_subscriber::{layer::Context, prelude::*, Layer};
8837
8838 fn fake_aft_stub_path() -> PathBuf {
8857 let mut path = std::env::current_exe().expect("current_exe available in tests");
8858 path.pop(); path.pop(); path.push(if cfg!(windows) {
8861 "fake-aft-stub.exe"
8862 } else {
8863 "fake-aft-stub"
8864 });
8865 assert!(
8866 path.exists(),
8867 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8868 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8869 path.display()
8870 );
8871 path
8872 }
8873
8874 fn client_retries(code: &str) -> bool {
8879 subc_protocol::error_codes::is_retryable_route_open(code)
8880 }
8881
8882 #[test]
8893 fn retryability_of_forwarding_codes_matches_the_failure() {
8894 let transient = [
8897 ForwardingError::NoModuleConnection,
8898 ForwardingError::ModuleReloading {
8899 module_id: "m".into(),
8900 },
8901 ForwardingError::StaleModuleEndpoint,
8902 ForwardingError::UnknownReservation {
8903 client_channel: 1,
8904 module_channel: 1,
8905 },
8906 ForwardingError::ConnectionClosing {
8907 connection_id: ConnectionId::new(1),
8908 },
8909 ForwardingError::ClientEgressClosed {
8910 connection_id: ConnectionId::new(1),
8911 },
8912 ForwardingError::ModuleEgressUnavailable {
8913 connection_id: ConnectionId::new(1),
8914 },
8915 ];
8916 for err in transient {
8917 let code = forwarding_error_code(&err);
8918 assert!(
8919 client_retries(code),
8920 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8921 );
8922 }
8923
8924 let permanent = [
8929 ForwardingError::ConnectionRoleConflict {
8930 connection_id: ConnectionId::new(1),
8931 },
8932 ForwardingError::ClientRouteChannelExhausted {
8933 connection_id: ConnectionId::new(1),
8934 },
8935 ForwardingError::ModuleRouteChannelExhausted {
8936 endpoint: ModuleEndpointId {
8937 connection_id: ConnectionId::new(1),
8938 generation: 1,
8939 },
8940 },
8941 ForwardingError::RelayCorrelationExhausted,
8942 ForwardingError::RouteOpenBuild("x".into()),
8943 ForwardingError::Poisoned,
8944 ];
8945 for err in permanent {
8946 let code = forwarding_error_code(&err);
8947 assert!(
8948 !client_retries(code),
8949 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8950 );
8951 }
8952 }
8953
8954 #[tokio::test]
8982 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8983 let handler = ControlHandler::default();
8984 let frame =
8985 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8986
8987 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8991 assert_eq!(
8992 stamped,
8993 Principal::Direct,
8994 "a caller that proved nothing must not be stamped as a supervised module"
8995 );
8996
8997 let forged = handler
9002 .route_open_principal(
9003 &frame,
9004 Some(ConsumerIdentity {
9005 module_id: "aft".to_string(),
9006 launch_nonce: "not-a-real-nonce".to_string(),
9007 }),
9008 )
9009 .unwrap();
9010 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
9011 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
9012 }
9013
9014 #[test]
9031 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
9032 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
9033 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
9034 let ClientControlRequest::RouteOpen {
9035 consumer_identity, ..
9036 } = parsed
9037 else {
9038 panic!("route.open body must parse as RouteOpen");
9039 };
9040 assert_eq!(
9041 consumer_identity,
9042 Some(ConsumerIdentity {
9043 module_id: "aft".to_string(),
9044 launch_nonce: "n".to_string(),
9045 }),
9046 "the wire field name must reach the value route_open_principal reads"
9047 );
9048 }
9049
9050 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
9051 ModuleManifest::builder(module_id, "0.1.0")
9052 .protocol_ver(protocol_ver)
9053 .provides(vec![ProviderRole::ToolProvider {
9054 tools: vec![Tool {
9055 name: "read".to_string(),
9056 description: None,
9057 execution_mode: ExecutionMode::Pure,
9058 schema: json!({"type": "object"}),
9059 }],
9060 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
9061 concurrency: Concurrency::ModuleManaged,
9062 emits_push: true,
9063 sub_supervises: true,
9064 }])
9065 .build()
9066 }
9067
9068 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
9069 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
9070 }
9071
9072 fn hello_frame_with_control_ops(
9073 module_id: &str,
9074 protocol_ver: u8,
9075 corr: u64,
9076 control_ops: Option<Vec<String>>,
9077 ) -> Frame {
9078 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
9079 }
9080
9081 fn hello_frame_with_nonce(
9082 module_id: &str,
9083 protocol_ver: u8,
9084 corr: u64,
9085 launch_nonce: Option<&str>,
9086 ) -> Frame {
9087 hello_frame_full(
9088 module_id,
9089 protocol_ver,
9090 corr,
9091 None,
9092 launch_nonce.map(ToOwned::to_owned),
9093 )
9094 }
9095
9096 fn hello_frame_full(
9097 module_id: &str,
9098 protocol_ver: u8,
9099 corr: u64,
9100 control_ops: Option<Vec<String>>,
9101 launch_nonce: Option<String>,
9102 ) -> Frame {
9103 let body = serde_json::to_vec(&ModuleHelloBody {
9104 manifest: manifest(module_id, protocol_ver),
9105 protocol_ver,
9106 control_ops,
9107 launch_nonce,
9108 })
9109 .unwrap();
9110 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9111 }
9112
9113 fn hello_frame_with_rich_ui(
9114 module_id: &str,
9115 protocol_ver: u8,
9116 corr: u64,
9117 launch_nonce: Option<&str>,
9118 rich_ui: Value,
9119 ) -> Frame {
9120 let mut body = serde_json::to_value(ModuleHelloBody {
9121 manifest: manifest(module_id, protocol_ver),
9122 protocol_ver,
9123 control_ops: None,
9124 launch_nonce: launch_nonce.map(str::to_string),
9125 })
9126 .unwrap();
9127 body["manifest"]["rich_ui"] = rich_ui;
9128 Frame::build(
9129 FrameType::Hello,
9130 control_flags(),
9131 0,
9132 0,
9133 corr,
9134 serde_json::to_vec(&body).unwrap(),
9135 )
9136 .unwrap()
9137 }
9138
9139 fn non_routable_hello_frame_with_control_ops(
9140 module_id: &str,
9141 corr: u64,
9142 control_ops: Option<Vec<String>>,
9143 ) -> Frame {
9144 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9145 manifest.provides.clear();
9146 let body = serde_json::to_vec(&ModuleHelloBody {
9147 manifest,
9148 protocol_ver: PROTOCOL_VERSION,
9149 control_ops,
9150 launch_nonce: None,
9151 })
9152 .unwrap();
9153 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9154 }
9155
9156 fn capability_grammar_hello_frame(
9157 capabilities: Value,
9158 runtime_computed: Option<Value>,
9159 corr: u64,
9160 ) -> Frame {
9161 let mut body = serde_json::to_value(ModuleHelloBody {
9162 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
9163 protocol_ver: PROTOCOL_VERSION,
9164 control_ops: None,
9165 launch_nonce: None,
9166 })
9167 .expect("HELLO body serializes");
9168 body["manifest"]["capabilities"] = capabilities;
9169 if let Some(runtime_computed) = runtime_computed {
9170 body["runtime_computed"] = runtime_computed;
9171 }
9172 Frame::build(
9173 FrameType::Hello,
9174 control_flags(),
9175 0,
9176 0,
9177 corr,
9178 serde_json::to_vec(&body).expect("HELLO body reserializes"),
9179 )
9180 .expect("HELLO frame builds")
9181 }
9182
9183 fn channel_request(channel: u16, corr: u64) -> Frame {
9184 Frame::build(
9185 FrameType::Request,
9186 Flags::new(true, Priority::Interactive, false),
9187 channel,
9188 0,
9189 corr,
9190 b"opaque".to_vec(),
9191 )
9192 .unwrap()
9193 }
9194
9195 fn route_ctx(
9196 connection_id: ConnectionId,
9197 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
9198 let (tx, rx) = mpsc::channel(8);
9199 (
9200 RouteCtx {
9201 connection_id,
9202 egress: FrameSink::new(tx),
9203 },
9204 rx,
9205 )
9206 }
9207
9208 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
9209 serde_json::from_slice(&frame.body).unwrap()
9210 }
9211
9212 async fn hello_via_sink(
9217 handler: &ControlHandler,
9218 ctx: &RouteCtx,
9219 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9220 hello: Frame,
9221 ) -> Frame {
9222 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
9223 assert!(
9224 replies.is_empty(),
9225 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
9226 );
9227 let ack = rx
9228 .try_recv()
9229 .expect("HELLO_ACK is queued on the module sink")
9230 .frame;
9231 assert_eq!(ack.header.ty, FrameType::HelloAck);
9232 ack
9233 }
9234
9235 fn parse_error(frame: &Frame) -> Value {
9236 serde_json::from_slice(&frame.body).unwrap()
9237 }
9238
9239 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
9240 serde_json::from_slice(&frame.body).unwrap()
9241 }
9242
9243 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
9244 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
9245 route_channel,
9246 route_epoch: 0,
9247 kind,
9248 })
9249 .unwrap();
9250 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9251 }
9252
9253 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
9254 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
9255 module_id: module_id.to_string(),
9256 })
9257 .unwrap();
9258 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9259 }
9260
9261 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
9262 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
9263 }
9264
9265 fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
9267 let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
9268 let mut frame = route_open_frame(corr, module_id, root);
9269 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
9270 body["identity"]["project_root"] = serde_json::json!(path.as_path());
9271 frame.body = serde_json::to_vec(&body).unwrap();
9272 frame
9273 }
9274
9275 fn route_open_frame_with_consumer_capabilities(
9276 corr: u64,
9277 module_id: &str,
9278 _project_root: ScratchDir,
9279 consumer_capabilities: Option<Vec<String>>,
9280 ) -> Frame {
9281 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9282 busy_refusals: None,
9283 target: RouteTarget::ToolProvider {
9284 module_id: module_id.to_string(),
9285 },
9286 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
9287 consumer_identity: None,
9288 consumer_capabilities,
9289 role_versions: None,
9290 admission_facts: None,
9291 scope: None,
9292 })
9293 .unwrap();
9294 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9295 }
9296
9297 fn route_open_frame_with_role_versions(
9298 corr: u64,
9299 module_id: &str,
9300 _project_root: ScratchDir,
9301 role_versions: Option<BTreeMap<String, String>>,
9302 ) -> Frame {
9303 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9304 busy_refusals: None,
9305 target: RouteTarget::ToolProvider {
9306 module_id: module_id.to_string(),
9307 },
9308 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9309 consumer_identity: None,
9310 consumer_capabilities: None,
9311 role_versions,
9312 admission_facts: None,
9313 scope: None,
9314 })
9315 .unwrap();
9316 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9317 }
9318
9319 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
9320 entries
9321 .iter()
9322 .map(|(role, version)| (role.to_string(), version.to_string()))
9323 .collect()
9324 }
9325
9326 fn route_open_frame_with_admission_facts(
9327 corr: u64,
9328 module_id: &str,
9329 _project_root: ScratchDir,
9330 consumer_identity: Option<subc_control::ConsumerIdentity>,
9331 facts: Option<Value>,
9332 ) -> Frame {
9333 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9334 busy_refusals: None,
9335 target: RouteTarget::ToolProvider {
9336 module_id: module_id.to_string(),
9337 },
9338 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9339 consumer_identity,
9340 consumer_capabilities: None,
9341 role_versions: None,
9342 admission_facts: facts,
9343 scope: None,
9344 })
9345 .unwrap();
9346 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9347 }
9348
9349 #[derive(Clone, Default)]
9350 struct EventCapture {
9351 events: Arc<Mutex<Vec<CapturedEvent>>>,
9352 }
9353
9354 #[derive(Clone, Debug)]
9355 struct CapturedEvent {
9356 target: String,
9357 level: tracing::Level,
9358 fields: BTreeMap<String, String>,
9359 }
9360
9361 impl EventCapture {
9362 fn events(&self) -> Vec<CapturedEvent> {
9363 self.events.lock().unwrap().clone()
9364 }
9365 }
9366
9367 impl<S> Layer<S> for EventCapture
9368 where
9369 S: Subscriber,
9370 {
9371 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
9372 let mut visitor = EventFieldVisitor::default();
9373 event.record(&mut visitor);
9374 self.events.lock().unwrap().push(CapturedEvent {
9375 target: event.metadata().target().to_string(),
9376 level: *event.metadata().level(),
9377 fields: visitor.fields,
9378 });
9379 }
9380 }
9381
9382 #[derive(Default)]
9383 struct EventFieldVisitor {
9384 fields: BTreeMap<String, String>,
9385 }
9386
9387 impl Visit for EventFieldVisitor {
9388 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
9389 self.fields
9390 .insert(field.name().to_string(), format!("{value:?}"));
9391 }
9392 }
9393
9394 fn health_response(corr: u64, status: HealthStatus) -> Frame {
9395 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
9396 status,
9397 detail: Some("warming".to_string()),
9398 metrics: Some(json!({"queue_depth": 3})),
9399 })
9400 .unwrap();
9401 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9402 }
9403
9404 fn route_bind_ack(corr: u64) -> Frame {
9405 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
9406 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9407 }
9408
9409 fn unique_project_root(label: &str) -> ScratchDir {
9410 ScratchDir::new(label)
9411 }
9412
9413 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
9414 match parse_route_poll(frame) {
9415 ClientControlResponse::RoutePoll {
9416 status: None,
9417 live: Some(live),
9418 ..
9419 } => assert_eq!(live, expected_live),
9420 other => panic!("unexpected route.poll response: {other:?}"),
9421 }
9422 }
9423
9424 fn bind_liveness_route(
9425 registry: &Registry,
9426 forwarding: &ForwardingTable,
9427 module_id: &str,
9428 ) -> (RouteCtx, u16, u32) {
9429 let module_connection = ConnectionId::new(101);
9430 let client_connection = ConnectionId::new(202);
9431 let registration = registry
9432 .register_with_control_ops(
9433 manifest(module_id, PROTOCOL_VERSION),
9434 PROTOCOL_VERSION,
9435 module_connection,
9436 module_baseline_control_ops(),
9437 )
9438 .unwrap();
9439 let (module_tx, _module_rx) = mpsc::channel(8);
9440 let endpoint = forwarding
9441 .register_module_connection(
9442 module_connection,
9443 module_id.to_string(),
9444 PROTOCOL_VERSION,
9445 manifest_concurrency(®istration.manifest),
9446 FrameSink::new(module_tx),
9447 )
9448 .unwrap();
9449 let (client_ctx, _client_rx) = route_ctx(client_connection);
9450 let pending = forwarding
9451 .begin_route_bind_relay_for_test(
9452 client_connection,
9453 client_ctx.egress.clone(),
9454 1,
9455 module_id,
9456 )
9457 .unwrap();
9458 assert_eq!(pending.endpoint, endpoint);
9459 let route_channel = pending.client_channel;
9460 let route_epoch = pending.client_epoch;
9461 forwarding
9462 .complete_pending_relay(
9463 module_connection,
9464 pending.corr,
9465 RouteBindRelayOutcome::Accepted,
9466 )
9467 .unwrap();
9468 (client_ctx, route_channel, route_epoch)
9469 }
9470
9471 struct FakeProcessLiveness {
9472 live: Option<bool>,
9473 }
9474
9475 impl ModuleProcessLiveness for FakeProcessLiveness {
9476 fn process_live(&self, _module_id: &str) -> Option<bool> {
9477 self.live
9478 }
9479 }
9480
9481 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9482 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9483 {
9484 let registry = Arc::new(Registry::default());
9485 let supervisor_handle = SupervisorHandle::new();
9486 let supervisor = Supervisor::new_for_test(
9487 Arc::clone(®istry),
9488 RestartPolicy::new(1, Duration::from_millis(10)),
9489 )
9490 .with_handle(supervisor_handle.clone());
9491 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9492 let module = supervisor
9493 .spawn(ModuleSpec {
9494 module_id: "stderr-tail-wire".to_string(),
9495 program: fake_aft_stub_path(),
9496 args: Vec::new(),
9497 env: vec![
9498 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9499 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9500 ],
9501 reserved: false,
9502 reserved_prefixes: Vec::new(),
9503 protocol: ModuleProtocol::Subc,
9504 overlap: Default::default(),
9505 })
9506 .unwrap();
9507
9508 let deadline = Instant::now() + Duration::from_secs(5);
9509 loop {
9510 let tail = module.stderr_tail(None, None);
9511 if tail
9512 .entries
9513 .iter()
9514 .any(|entry| matches!(entry, TailEntry::ProcessStart))
9515 && tail.entries.iter().any(|entry| {
9516 matches!(
9517 entry,
9518 TailEntry::Line {
9519 truncated: true,
9520 ..
9521 }
9522 )
9523 })
9524 {
9525 break;
9526 }
9527 assert!(
9528 Instant::now() < deadline,
9529 "module did not produce a truncated line and restart boundary: {tail:?}"
9530 );
9531 sleep(Duration::from_millis(10)).await;
9532 }
9533
9534 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9535 let request = ClientControlRequest::SupervisorStderrTail {
9536 module_id: "stderr-tail-wire".to_string(),
9537 max_lines: None,
9538 max_bytes: None,
9539 };
9540 let frame = Frame::build(
9541 FrameType::Request,
9542 control_flags(),
9543 0,
9544 0,
9545 1,
9546 serde_json::to_vec(&request).unwrap(),
9547 )
9548 .unwrap();
9549 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9550 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9551 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9552 serde_json::from_slice(&responses[0].body).unwrap()
9553 else {
9554 panic!("expected supervisor.stderr_tail response");
9555 };
9556
9557 assert!(
9558 tail.entries
9559 .iter()
9560 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9561 "the control response lost the restart boundary"
9562 );
9563 let Some(StderrTailEntry::Line {
9564 text,
9565 truncated,
9566 at_ms,
9567 }) = tail.entries.iter().find(|entry| {
9568 matches!(
9569 entry,
9570 StderrTailEntry::Line {
9571 truncated: true,
9572 ..
9573 }
9574 )
9575 })
9576 else {
9577 panic!("the control response lost the truncated line");
9578 };
9579 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9580 assert!(*truncated);
9581 assert!(
9582 at_ms.is_some(),
9583 "the control response lost the line's capture time"
9584 );
9585 }
9586
9587 #[tokio::test(flavor = "current_thread")]
9592 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9593 let dir = ScratchDir::new("terminals-off-worker");
9594 let journal_path = dir.join("terminals.jsonl");
9595 let registry = Arc::new(Registry::default());
9596 let supervisor_handle = SupervisorHandle::new();
9597 let supervisor =
9598 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9599 .with_handle(supervisor_handle.clone())
9600 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9601 let module = supervisor
9602 .spawn(ModuleSpec {
9603 module_id: "terminal-off-worker".to_string(),
9604 program: fake_aft_stub_path(),
9605 args: Vec::new(),
9606 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9607 reserved: false,
9608 reserved_prefixes: Vec::new(),
9609 protocol: ModuleProtocol::Subc,
9610 overlap: Default::default(),
9611 })
9612 .unwrap();
9613 let deadline = Instant::now() + Duration::from_secs(5);
9614 while module.terminal_history().entries.len() != 2 {
9615 assert!(Instant::now() < deadline, "module did not record two exits");
9616 sleep(Duration::from_millis(10)).await;
9617 }
9618
9619 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9620 let handler =
9621 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
9622 let frame = Frame::build(
9623 FrameType::Request,
9624 control_flags(),
9625 0,
9626 0,
9627 1,
9628 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9629 module_id: "terminal-off-worker".to_string(),
9630 })
9631 .unwrap(),
9632 )
9633 .unwrap();
9634 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9635 let spawned_at = std::time::Instant::now();
9636 let read = tokio::spawn({
9637 let handler = Arc::clone(&handler);
9638 async move { handler.handle_control_frame(&ctx, frame).await }
9639 });
9640 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9643 .await
9644 .unwrap()
9645 .expect("the history read reached its pause");
9646 let elapsed = spawned_at.elapsed();
9647 assert!(
9648 elapsed < Duration::from_secs(2) && !read.is_finished(),
9649 "this task could not run while the history read was paused \
9650 (resumed after {elapsed:?}, read finished: {})",
9651 read.is_finished()
9652 );
9653
9654 drop(release);
9655 let responses = read.await.unwrap().unwrap();
9656 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9657 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9658 panic!("expected supervisor.terminals response");
9659 };
9660 assert_eq!(terminals.entries.len(), 2);
9661 assert_eq!(terminals.journal_skipped_lines, 0);
9662 assert_eq!(terminals.journal_read_errors, 0);
9663 }
9664
9665 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9666 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9667 let registry = Arc::new(Registry::default());
9668 let supervisor_handle = SupervisorHandle::new();
9669 let supervisor =
9670 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9671 .with_handle(supervisor_handle.clone());
9672 let module = supervisor
9673 .spawn(ModuleSpec {
9674 module_id: "terminal-golden".to_string(),
9675 program: fake_aft_stub_path(),
9676 args: Vec::new(),
9677 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9678 reserved: false,
9679 reserved_prefixes: Vec::new(),
9680 protocol: ModuleProtocol::Subc,
9681 overlap: Default::default(),
9682 })
9683 .unwrap();
9684
9685 let deadline = Instant::now() + Duration::from_secs(5);
9686 while module.terminal_history().entries.len() != 2 {
9687 assert!(
9688 Instant::now() < deadline,
9689 "module did not retain two terminal exits: {:?}",
9690 module.terminal_history()
9691 );
9692 sleep(Duration::from_millis(10)).await;
9693 }
9694
9695 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9696 let request = ClientControlRequest::SupervisorTerminals {
9697 module_id: "terminal-golden".to_string(),
9698 };
9699 let frame = Frame::build(
9700 FrameType::Request,
9701 control_flags(),
9702 0,
9703 0,
9704 1,
9705 serde_json::to_vec(&request).unwrap(),
9706 )
9707 .unwrap();
9708 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9709 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9710 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9711 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9712 panic!("expected supervisor.terminals response");
9713 };
9714 assert_eq!(terminals.entries.len(), 2);
9715 assert_eq!(terminals.dropped, 0);
9716
9717 let mut rendered = serde_json::to_value(response).unwrap();
9718 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9721 for (index, entry) in rendered["entries"]
9722 .as_array_mut()
9723 .expect("terminal response entries array")
9724 .iter_mut()
9725 .enumerate()
9726 {
9727 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9728 }
9729
9730 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9731 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9732 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9733 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9734 std::fs::write(&golden_path, &serialized).unwrap();
9735 }
9736 let expected: Value =
9737 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9738 assert_eq!(rendered, expected);
9739 }
9740
9741 #[test]
9742 fn hello_registers_manifest_and_returns_ack() {
9743 let registry = Arc::new(Registry::default());
9744 let handler = ControlHandler::new(Arc::clone(®istry));
9745 let conn = ConnectionId::new(1);
9746
9747 let responses = handler
9748 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9749 .unwrap();
9750
9751 assert_eq!(responses.len(), 1);
9752 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9753 assert_eq!(responses[0].header.channel, 0);
9754 assert_eq!(responses[0].header.corr, 7);
9755 let ack = parse_ack(&responses[0]);
9756 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9757 assert!(ack
9758 .subc_capabilities
9759 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9760 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9761 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9762 assert!(ack
9763 .subc_ops
9764 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9765 assert!(ack
9766 .subc_ops
9767 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9768
9769 let registration = registry.get_module("aft").unwrap().unwrap();
9770 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9771 assert_eq!(registration.state, ChannelState::Active);
9772 assert_eq!(registration.connection_id, conn);
9773 assert_eq!(registration.control_ops, module_baseline_control_ops());
9774 }
9775
9776 #[test]
9777 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9778 let invalid_identifiers = [
9779 ("case_change", "credentials-Provider/v1"),
9780 ("leading_zero", "credentials-provider/v01"),
9781 ("trailing_hyphen", "credentials-provider-/v1"),
9782 ("consecutive_hyphens", "credentials--provider/v1"),
9783 ("uppercase", "Credentials-provider/v1"),
9784 ("missing_v", "credentials-provider/1"),
9785 ("whitespace", "credentials provider/v1"),
9786 ("zero_version", "credentials-provider/v0"),
9787 ("out_of_range_version", "credentials-provider/v4294967296"),
9788 (
9789 "overlength_name",
9790 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9791 ),
9792 ];
9793 let mut cases = invalid_identifiers
9794 .into_iter()
9795 .map(|(name, identifier)| {
9796 (
9797 format!("identifier_{name}"),
9798 "capabilities.provides[0]".to_string(),
9799 identifier.to_string(),
9800 json!({ "provides": [identifier] }),
9801 None,
9802 )
9803 })
9804 .collect::<Vec<_>>();
9805 cases.extend([
9806 (
9807 "unknown_need".to_string(),
9808 "capabilities.requires[0].need".to_string(),
9809 "deferred".to_string(),
9810 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9811 None,
9812 ),
9813 (
9814 "duplicate_provides".to_string(),
9815 "capabilities.provides[1]".to_string(),
9816 "credentials-provider/v1".to_string(),
9817 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9818 None,
9819 ),
9820 (
9821 "duplicate_must_never_reach".to_string(),
9822 "capabilities.must_never_reach[1]".to_string(),
9823 "credentials-provider/v1".to_string(),
9824 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9825 None,
9826 ),
9827 (
9828 "duplicate_requires_same_need".to_string(),
9829 "capabilities.requires[1]".to_string(),
9830 "credentials-provider/v1".to_string(),
9831 json!({ "requires": [
9832 { "capability": "credentials-provider/v1", "need": "required" },
9833 { "capability": "credentials-provider/v1", "need": "required" }
9834 ] }),
9835 None,
9836 ),
9837 (
9838 "duplicate_requires_conflicting_need".to_string(),
9839 "capabilities.requires[1]".to_string(),
9840 "credentials-provider/v1".to_string(),
9841 json!({ "requires": [
9842 { "capability": "credentials-provider/v1", "need": "required" },
9843 { "capability": "credentials-provider/v1", "need": "optional" }
9844 ] }),
9845 None,
9846 ),
9847 (
9848 "capabilities_root_pointer".to_string(),
9849 "runtime_computed[0]".to_string(),
9850 "/capabilities".to_string(),
9851 json!({}),
9852 Some(json!(["/capabilities"])),
9853 ),
9854 (
9855 "capabilities_descendant_pointer".to_string(),
9856 "runtime_computed[0]".to_string(),
9857 "/capabilities/provides".to_string(),
9858 json!({}),
9859 Some(json!(["/capabilities/provides"])),
9860 ),
9861 (
9862 "malformed_pointer_without_leading_slash".to_string(),
9863 "runtime_computed[0]".to_string(),
9864 "capabilities".to_string(),
9865 json!({}),
9866 Some(json!(["capabilities"])),
9867 ),
9868 (
9869 "malformed_pointer_escape".to_string(),
9870 "runtime_computed[0]".to_string(),
9871 "/roles/~2/tools".to_string(),
9872 json!({}),
9873 Some(json!(["/roles/~2/tools"])),
9874 ),
9875 (
9876 "unknown_capabilities_field".to_string(),
9877 "capabilities.future".to_string(),
9878 "<array>".to_string(),
9879 json!({ "future": [] }),
9880 None,
9881 ),
9882 ]);
9883
9884 for (index, (name, field, value, capabilities, runtime_computed)) in
9885 cases.into_iter().enumerate()
9886 {
9887 let registry = Arc::new(Registry::default());
9888 let handler = ControlHandler::new(Arc::clone(®istry));
9889 let response = handler
9890 .handle_control(
9891 ConnectionId::new((index + 1) as u64),
9892 capability_grammar_hello_frame(
9893 capabilities,
9894 runtime_computed,
9895 index as u64 + 1,
9896 ),
9897 )
9898 .expect("invalid HELLO returns a refusal");
9899
9900 assert_eq!(response.len(), 1, "{name} must emit one refusal");
9901 let error = parse_error(&response[0]);
9902 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9903 let message = error["message"]
9904 .as_str()
9905 .expect("error message is a string");
9906 assert!(
9907 message.contains(&field),
9908 "{name}: field missing from {message}"
9909 );
9910 assert!(
9911 message.contains(&value),
9912 "{name}: value missing from {message}"
9913 );
9914 assert_eq!(
9915 registry
9916 .active_registration_count()
9917 .expect("registry reads"),
9918 0,
9919 "{name}: refused HELLO must not create a catalog entry"
9920 );
9921 }
9922 }
9923
9924 #[test]
9925 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9926 let registry = Arc::new(Registry::default());
9927 let handler = ControlHandler::new(Arc::clone(®istry));
9928 let capabilities = json!({
9929 "provides": ["credentials-provider/v1"],
9930 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9931 "must_never_reach": ["federation-transport/v1"]
9932 });
9933 let response = handler
9934 .handle_control(
9935 ConnectionId::new(99),
9936 capability_grammar_hello_frame(
9937 capabilities.clone(),
9938 Some(json!(["/roles/0/tools"])),
9939 99,
9940 ),
9941 )
9942 .expect("valid HELLO registers");
9943 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9944
9945 let request = Frame::build(
9946 FrameType::Request,
9947 control_flags(),
9948 0,
9949 0,
9950 100,
9951 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9952 .expect("catalog request serializes"),
9953 )
9954 .expect("catalog request frame builds");
9955 let response = handler
9956 .handle_catalog_list(request, None)
9957 .expect("catalog list succeeds");
9958 let ClientControlResponse::CatalogList { modules, .. } =
9959 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9960 else {
9961 panic!("catalog request must return catalog.list");
9962 };
9963 assert_eq!(modules.len(), 1);
9964 assert_eq!(
9965 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9966 capabilities
9967 );
9968 }
9969
9970 #[test]
9971 fn catalog_list_mirrors_management_operation_description() {
9972 let registry = Arc::new(Registry::default());
9973 let handler = ControlHandler::new(Arc::clone(®istry));
9974 let description = "List managed records and return their identifiers and metadata.";
9975 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9976 manifest.provides = vec![ProviderRole::ManagementSurface {
9977 operations: vec![ManagementOperation {
9978 name: "records.list".to_string(),
9979 kind: ManagementOperationKind::Query,
9980 description: Some(description.to_string()),
9981 }],
9982 config_schema: json!({"type": "object"}),
9983 observability: vec![ObservabilitySurface {
9984 name: "records.stats".to_string(),
9985 kind: ObservabilityKind::Snapshot,
9986 }],
9987 identity_scope: vec![IdentityScope::Project],
9988 concurrency: Concurrency::ModuleManaged,
9989 }];
9990 registry
9991 .register_with_control_ops(
9992 manifest,
9993 PROTOCOL_VERSION,
9994 ConnectionId::new(99),
9995 Vec::new(),
9996 )
9997 .expect("described management manifest registers");
9998
9999 let request = Frame::build(
10000 FrameType::Request,
10001 control_flags(),
10002 0,
10003 0,
10004 100,
10005 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10006 .expect("catalog request serializes"),
10007 )
10008 .expect("catalog request frame builds");
10009 let response = handler
10010 .handle_catalog_list(request, None)
10011 .expect("catalog list succeeds");
10012 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10013 assert_eq!(
10014 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
10015 "catalog.list must preserve the declared operation description verbatim"
10016 );
10017 }
10018
10019 #[test]
10020 fn catalog_list_mirrors_registered_operation_delivery() {
10021 let registry = Arc::new(Registry::default());
10022 let handler = ControlHandler::new(Arc::clone(®istry));
10023 let delivery =
10024 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
10025 let mut body = serde_json::to_value(ModuleHelloBody {
10026 manifest: manifest("watch-management", PROTOCOL_VERSION),
10027 protocol_ver: PROTOCOL_VERSION,
10028 control_ops: None,
10029 launch_nonce: None,
10030 })
10031 .unwrap();
10032 body["manifest"]["provides"] = json!([{
10033 "role": "management_surface",
10034 "operations": [
10035 {"name": "status", "kind": "query"},
10036 {"name": "future", "kind": "query"},
10037 {"name": "reset", "kind": "mutate"}
10038 ],
10039 "config_schema": {}, "observability": [], "identity_scope": []
10040 }]);
10041 body["manifest"]["operation_delivery"] = delivery.clone();
10042 let hello = Frame::build(
10043 FrameType::Hello,
10044 control_flags(),
10045 0,
10046 0,
10047 99,
10048 serde_json::to_vec(&body).unwrap(),
10049 )
10050 .unwrap();
10051 let response = handler
10052 .handle_control(ConnectionId::new(99), hello)
10053 .unwrap();
10054 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10055
10056 let (_, registrations) = registry.list_modules().unwrap();
10057 let registered = ®istrations[0].manifest;
10058 assert!(registered.is_watch_operation("status"));
10059 assert!(!registered.is_watch_operation("future"));
10060 assert!(!registered.is_watch_operation("reset"));
10061 assert!(!registered.is_watch_operation("missing"));
10062
10063 let request = Frame::build(
10064 FrameType::Request,
10065 control_flags(),
10066 0,
10067 0,
10068 100,
10069 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10070 )
10071 .unwrap();
10072 let response = handler.handle_catalog_list(request, None).unwrap();
10073 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
10074 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
10075 let ClientControlResponse::CatalogList { modules, .. } =
10076 serde_json::from_slice(&response[0].body).unwrap()
10077 else {
10078 panic!("expected catalog.list");
10079 };
10080 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
10081 }
10082
10083 #[test]
10084 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
10085 let registry = Arc::new(Registry::default());
10086 let supervisor = SupervisorHandle::new();
10087 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10088 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10089 let descriptor = json!({
10090 "schema": "ckios.richui.manifest/1",
10091 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10092 "bytes": 4096,
10093 "chunks": 4
10094 });
10095 let connection = ConnectionId::new(201);
10096 let hello = handler
10097 .handle_control(
10098 connection,
10099 hello_frame_with_rich_ui(
10100 "vault",
10101 PROTOCOL_VERSION,
10102 201,
10103 Some("the-real-nonce"),
10104 descriptor.clone(),
10105 ),
10106 )
10107 .unwrap();
10108 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10109
10110 let request =
10111 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
10112 let provides = registry
10113 .get_module_by_connection(connection)
10114 .unwrap()
10115 .unwrap()
10116 .manifest
10117 .provides;
10118 handler
10119 .handle_catalog_update(connection, request, provides, None, Some(false))
10120 .unwrap();
10121
10122 let request = Frame::build(
10123 FrameType::Request,
10124 control_flags(),
10125 0,
10126 0,
10127 203,
10128 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10129 )
10130 .unwrap();
10131 let response = handler.handle_catalog_list(request, None).unwrap();
10132 let ClientControlResponse::CatalogList { modules, .. } =
10133 serde_json::from_slice(&response[0].body).unwrap()
10134 else {
10135 panic!("expected catalog.list");
10136 };
10137 assert_eq!(
10138 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
10139 descriptor
10140 );
10141 assert!(modules[0].nonce_verified);
10142 }
10143
10144 fn hello_with_role_declarations(declarations: Value) -> Frame {
10145 let mut body = serde_json::to_value(ModuleHelloBody {
10146 manifest: manifest("role-test", PROTOCOL_VERSION),
10147 protocol_ver: PROTOCOL_VERSION,
10148 control_ops: None,
10149 launch_nonce: None,
10150 })
10151 .unwrap();
10152 body["manifest"]["role_declarations"] = declarations;
10153 Frame::build(
10154 FrameType::Hello,
10155 control_flags(),
10156 0,
10157 0,
10158 301,
10159 serde_json::to_vec(&body).unwrap(),
10160 )
10161 .unwrap()
10162 }
10163
10164 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
10165 let request = Frame::build(
10166 FrameType::Request,
10167 control_flags(),
10168 0,
10169 0,
10170 302,
10171 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10172 )
10173 .unwrap();
10174 let response = handler.handle_catalog_list(request, None).unwrap();
10175 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
10176 let ClientControlResponse::CatalogList { mut modules, .. } =
10177 serde_json::from_slice(&response[0].body).unwrap()
10178 else {
10179 panic!("expected catalog.list");
10180 };
10181 (raw["modules"][0].clone(), modules.remove(0))
10182 }
10183
10184 #[test]
10185 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
10186 let registry = Arc::new(Registry::default());
10187 let handler = ControlHandler::new(Arc::clone(®istry));
10188 let declarations = json!({
10189 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
10190 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
10191 });
10192 let connection = ConnectionId::new(301);
10193 let response = handler
10194 .handle_control(
10195 connection,
10196 hello_with_role_declarations(declarations.clone()),
10197 )
10198 .unwrap();
10199 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10200 let registered = registry
10201 .get_module_by_connection(connection)
10202 .unwrap()
10203 .unwrap()
10204 .manifest;
10205 assert_eq!(
10209 serde_json::to_value(®istered.role_declarations).unwrap(),
10210 declarations
10211 );
10212 let (raw, entry) = role_declarations_catalog(&handler);
10213 assert_eq!(raw["role_declarations"], declarations);
10214 assert_eq!(entry.role_declarations, registered.role_declarations);
10215
10216 let request =
10217 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
10218 handler
10219 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
10220 .unwrap();
10221 let (raw, entry) = role_declarations_catalog(&handler);
10222 assert_eq!(raw["role_declarations"], declarations);
10223 assert_eq!(entry.role_declarations, registered.role_declarations);
10224 }
10225
10226 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
10227 let registry = Arc::new(Registry::default());
10228 let handler = ControlHandler::new(Arc::clone(®istry));
10229 let capture = EventCapture::default();
10230 let _subscriber =
10231 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10232 let response = handler
10233 .handle_control(
10234 ConnectionId::new(301),
10235 hello_with_role_declarations(declarations),
10236 )
10237 .unwrap();
10238 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10239 assert!(registry
10240 .get_module_by_connection(ConnectionId::new(301))
10241 .unwrap()
10242 .unwrap()
10243 .manifest
10244 .role_declarations
10245 .is_empty());
10246 let (raw, entry) = role_declarations_catalog(&handler);
10247 assert!(raw.get("role_declarations").is_none());
10248 assert!(entry.role_declarations.is_empty());
10249 let warnings: Vec<_> = capture
10250 .events()
10251 .into_iter()
10252 .filter(|event| {
10253 event.fields.get("message").map(String::as_str)
10254 == Some("dropping whole role_declarations map from HELLO")
10255 })
10256 .collect();
10257 assert_eq!(warnings.len(), 1);
10258 assert_eq!(warnings[0].level, tracing::Level::WARN);
10259 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
10260 assert_eq!(warnings[0].fields["reason"], reason);
10261 }
10262
10263 #[test]
10264 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
10265 for value in [
10266 json!(null),
10267 json!(false),
10268 json!(0),
10269 json!("opaque"),
10270 json!([]),
10271 ] {
10272 assert_hello_drops_role_declarations_with_warning(
10273 json!({"routing/v1": {}, "broken/v1": value}),
10274 "role_declarations values must be objects",
10275 );
10276 }
10277 assert_hello_drops_role_declarations_with_warning(
10278 json!([]),
10279 "role_declarations must be an object",
10280 );
10281 }
10282
10283 #[test]
10284 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
10285 assert_hello_drops_role_declarations_with_warning(
10286 json!({"routing/v1": {}, "bad key": {}}),
10287 "role_declarations key must use capability-name grammar",
10288 );
10289 }
10290
10291 #[test]
10292 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
10293 assert_hello_drops_role_declarations_with_warning(
10294 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
10295 "role_declarations exceeds 16384 compact JSON bytes",
10296 );
10297 }
10298
10299 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
10300 let mut body = serde_json::to_value(ModuleHelloBody {
10301 manifest: manifest("flow-ops", PROTOCOL_VERSION),
10302 protocol_ver: PROTOCOL_VERSION,
10303 control_ops: None,
10304 launch_nonce: None,
10305 })
10306 .unwrap();
10307 body["manifest"]["provides"] = json!([{
10308 "role": "management_surface",
10309 "operations": [
10310 {"name": "diagnostics.findings", "kind": "query"},
10311 {"name": "reset", "kind": "mutate"}
10312 ],
10313 "config_schema": {}, "observability": [], "identity_scope": []
10314 }]);
10315 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
10316 if let Some(flow_operations) = flow_operations {
10317 body["manifest"]["flow_operations"] = flow_operations;
10318 }
10319 Frame::build(
10320 FrameType::Hello,
10321 control_flags(),
10322 0,
10323 0,
10324 401,
10325 serde_json::to_vec(&body).unwrap(),
10326 )
10327 .unwrap()
10328 }
10329
10330 #[test]
10331 fn catalog_list_mirrors_flow_operations_unchanged() {
10332 for (capability, flow_operations) in [
10335 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
10336 ("call-modes/v1", json!(["diagnostics.findings"])),
10337 ("flow-scopes/v1", json!([])),
10338 ] {
10339 let registry = Arc::new(Registry::default());
10340 let handler = ControlHandler::new(Arc::clone(®istry));
10341 let connection = ConnectionId::new(401);
10342 let response = handler
10343 .handle_control(
10344 connection,
10345 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
10346 )
10347 .unwrap();
10348 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10349 let registered = registry
10350 .get_module_by_connection(connection)
10351 .unwrap()
10352 .unwrap()
10353 .manifest;
10354 assert_eq!(
10355 serde_json::to_value(®istered.flow_operations).unwrap(),
10356 flow_operations
10357 );
10358 let (raw, entry) = role_declarations_catalog(&handler);
10359 assert_eq!(raw["flow_operations"], flow_operations);
10360 assert_eq!(entry.flow_operations, registered.flow_operations);
10361 }
10362
10363 let registry = Arc::new(Registry::default());
10364 let handler = ControlHandler::new(Arc::clone(®istry));
10365 let response = handler
10366 .handle_control(
10367 ConnectionId::new(401),
10368 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
10369 )
10370 .unwrap();
10371 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10372 let (raw, entry) = role_declarations_catalog(&handler);
10373 assert!(raw.get("flow_operations").is_none());
10374 assert_eq!(entry.flow_operations, None);
10375 }
10376
10377 fn assert_hello_drops_flow_operations_with_warning(
10378 capabilities: Value,
10379 flow_operations: Value,
10380 reason: &str,
10381 ) {
10382 let registry = Arc::new(Registry::default());
10383 let handler = ControlHandler::new(Arc::clone(®istry));
10384 let capture = EventCapture::default();
10385 let _subscriber =
10386 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10387 let response = handler
10388 .handle_control(
10389 ConnectionId::new(401),
10390 hello_with_flow_operations(capabilities, Some(flow_operations)),
10391 )
10392 .unwrap();
10393 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10394 let registered = registry
10395 .get_module_by_connection(ConnectionId::new(401))
10396 .unwrap()
10397 .expect("the module still registers")
10398 .manifest;
10399 assert_eq!(registered.module_id, "flow-ops");
10400 assert_eq!(registered.flow_operations, None);
10401 let (raw, entry) = role_declarations_catalog(&handler);
10402 assert_eq!(raw["module_id"], "flow-ops");
10403 assert!(raw.get("flow_operations").is_none());
10404 assert_eq!(entry.flow_operations, None);
10405 let warnings: Vec<_> = capture
10406 .events()
10407 .into_iter()
10408 .filter(|event| {
10409 event.fields.get("message").map(String::as_str)
10410 == Some("dropping flow_operations from HELLO")
10411 })
10412 .collect();
10413 assert_eq!(warnings.len(), 1);
10414 assert_eq!(warnings[0].level, tracing::Level::WARN);
10415 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
10416 assert_eq!(warnings[0].fields["reason"], reason);
10417 }
10418
10419 #[test]
10420 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
10421 assert_hello_drops_flow_operations_with_warning(
10422 json!(["flow-scopes/v1"]),
10423 json!(["diagnostics.findings", "missing"]),
10424 "flow_operations names 'missing', which is not a declared management operation",
10425 );
10426 }
10427
10428 #[test]
10429 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
10430 assert_hello_drops_flow_operations_with_warning(
10431 json!([]),
10432 json!(["diagnostics.findings"]),
10433 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10434 );
10435 assert_hello_drops_flow_operations_with_warning(
10437 json!(["agent-run-scopes/v1"]),
10438 json!([]),
10439 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10440 );
10441 }
10442
10443 #[test]
10444 fn hello_malformed_flow_operations_is_dropped_with_warning() {
10445 assert_hello_drops_flow_operations_with_warning(
10446 json!(["flow-scopes/v1"]),
10447 json!(["reset", "reset"]),
10448 "flow_operations names 'reset' more than once",
10449 );
10450 assert_hello_drops_flow_operations_with_warning(
10451 json!(["flow-scopes/v1"]),
10452 json!(vec!["reset"; 257]),
10453 "flow_operations lists 257 names, more than 256",
10454 );
10455 assert_hello_drops_flow_operations_with_warning(
10456 json!(["flow-scopes/v1"]),
10457 json!("reset"),
10458 "flow_operations must be an array of operation names",
10459 );
10460 }
10461
10462 #[test]
10463 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
10464 let registry = Arc::new(Registry::default());
10465 let handler = ControlHandler::new(registry.clone());
10466 let descriptor = json!({
10467 "schema": "ckios.richui.manifest/1",
10468 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10469 "bytes": 128,
10470 "chunks": 2
10471 });
10472 let malformed = [
10473 ("schema", json!("")),
10474 ("schema", json!("a".repeat(65))),
10475 ("schema", json!("Bad schema")),
10476 (
10477 "manifest_digest",
10478 json!(format!("sha256:{}", "A".repeat(64))),
10479 ),
10480 ("manifest_digest", json!("sha256:abc")),
10481 ("bytes", json!(0)),
10482 ("bytes", json!(16_777_217)),
10483 ("bytes", json!("128")),
10484 ("chunks", json!(0)),
10485 ("chunks", json!(1025)),
10486 ("chunks", json!("2")),
10487 ];
10488 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
10489 let mut invalid = descriptor.clone();
10490 invalid[field] = invalid_value;
10491 let response = handler
10492 .handle_control(
10493 ConnectionId::new(210 + index as u64),
10494 hello_frame_with_rich_ui(
10495 &format!("bad-rich-ui-{index}"),
10496 PROTOCOL_VERSION,
10497 210 + index as u64,
10498 None,
10499 invalid,
10500 ),
10501 )
10502 .unwrap();
10503 assert_eq!(response[0].header.ty, FrameType::Error);
10504 let error = parse_error(&response[0]);
10505 assert_eq!(error["code"], "invalid_manifest");
10506 assert!(error["message"]
10507 .as_str()
10508 .unwrap()
10509 .contains(&format!("rich_ui.{field}")));
10510 assert!(registry
10511 .get_module(&format!("bad-rich-ui-{index}"))
10512 .unwrap()
10513 .is_none());
10514 }
10515 }
10516
10517 #[test]
10518 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
10519 let registry = Arc::new(Registry::default());
10520 let handler = ControlHandler::new(Arc::clone(®istry));
10521 let connection = ConnectionId::new(219);
10522 registry
10523 .register_with_control_ops(
10524 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
10525 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
10526 "bad schema",
10527 format!("sha256:{}", "a".repeat(64)),
10528 128,
10529 2,
10530 )))
10531 .build(),
10532 PROTOCOL_VERSION,
10533 connection,
10534 Vec::new(),
10535 )
10536 .unwrap();
10537 let request =
10538 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
10539 let response = handler
10540 .handle_catalog_update(connection, request, Vec::new(), None, None)
10541 .unwrap();
10542 assert_eq!(response[0].header.ty, FrameType::Error);
10543 let error = parse_error(&response[0]);
10544 assert_eq!(error["code"], "invalid_manifest");
10545 assert!(error["message"]
10546 .as_str()
10547 .unwrap()
10548 .contains("rich_ui.schema"));
10549 }
10550
10551 #[test]
10552 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
10553 let registry = Arc::new(Registry::default());
10554 let supervisor = SupervisorHandle::new();
10555 supervisor.open_swap("provider", "candidate-nonce".to_string());
10556 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10557 let connection = ConnectionId::new(218);
10558 let hello = handler
10559 .handle_control(
10560 connection,
10561 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
10562 )
10563 .unwrap();
10564 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10565 assert!(registry.list_modules().unwrap().1.is_empty());
10566
10567 registry.promote_candidate("provider").unwrap().unwrap();
10568 let request = Frame::build(
10569 FrameType::Request,
10570 control_flags(),
10571 0,
10572 0,
10573 219,
10574 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10575 )
10576 .unwrap();
10577 let response = handler.handle_catalog_list(request, None).unwrap();
10578 let ClientControlResponse::CatalogList { modules, .. } =
10579 serde_json::from_slice(&response[0].body).unwrap()
10580 else {
10581 panic!("expected catalog.list");
10582 };
10583 assert_eq!(modules.len(), 1);
10584 assert!(modules[0].nonce_verified);
10585 }
10586
10587 #[test]
10588 fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
10589 for provenance_json in [
10590 r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
10591 r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
10592 ] {
10593 let registry = Arc::new(Registry::default());
10594 let handler = ControlHandler::new(registry);
10595 let hello = Frame::build(
10596 FrameType::Hello,
10597 control_flags(),
10598 0,
10599 0,
10600 220,
10601 serde_json::to_vec(&ModuleHelloBody {
10602 manifest: ModuleManifest::builder("built", "0.1.0")
10603 .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
10604 .build(),
10605 protocol_ver: PROTOCOL_VERSION,
10606 control_ops: None,
10607 launch_nonce: None,
10608 })
10609 .unwrap(),
10610 )
10611 .unwrap();
10612 assert_eq!(
10613 handler
10614 .handle_control(ConnectionId::new(220), hello)
10615 .unwrap()[0]
10616 .header
10617 .ty,
10618 FrameType::HelloAck
10619 );
10620 let request = Frame::build(
10621 FrameType::Request,
10622 control_flags(),
10623 0,
10624 0,
10625 221,
10626 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10627 )
10628 .unwrap();
10629 let response = handler.handle_catalog_list(request, None).unwrap();
10630 let ClientControlResponse::CatalogList { modules, .. } =
10631 serde_json::from_slice(&response[0].body).unwrap()
10632 else {
10633 panic!("expected catalog.list");
10634 };
10635 assert_eq!(modules.len(), 1);
10636 assert_eq!(
10637 serde_json::to_vec(&modules[0]).unwrap(),
10638 format!(
10639 r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
10640 )
10641 .as_bytes()
10642 );
10643 }
10644 }
10645
10646 #[test]
10647 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
10648 let registry = Arc::new(Registry::default());
10649 let handler = ControlHandler::new(Arc::clone(®istry));
10650 let hello = Frame::build(
10651 FrameType::Hello,
10652 control_flags(),
10653 0,
10654 0,
10655 220,
10656 serde_json::to_vec(&ModuleHelloBody {
10657 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
10658 protocol_ver: PROTOCOL_VERSION,
10659 control_ops: None,
10660 launch_nonce: Some("unverified".to_string()),
10661 })
10662 .unwrap(),
10663 )
10664 .unwrap();
10665 assert_eq!(
10666 handler
10667 .handle_control(ConnectionId::new(220), hello)
10668 .unwrap()[0]
10669 .header
10670 .ty,
10671 FrameType::HelloAck
10672 );
10673 let request = Frame::build(
10674 FrameType::Request,
10675 control_flags(),
10676 0,
10677 0,
10678 221,
10679 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10680 )
10681 .unwrap();
10682 let response = handler.handle_catalog_list(request, None).unwrap();
10683 let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
10684 assert!(wire["modules"][0].get("provenance").is_none());
10685 let ClientControlResponse::CatalogList { modules, .. } =
10686 serde_json::from_slice(&response[0].body).unwrap()
10687 else {
10688 panic!("expected catalog.list");
10689 };
10690 assert_eq!(
10691 serde_json::to_vec(&modules[0]).unwrap(),
10692 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
10693 );
10694 assert!(!modules[0].nonce_verified);
10695 assert!(modules[0].provenance.is_none());
10696 }
10697
10698 #[test]
10699 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
10700 let registry = Arc::new(Registry::default());
10701 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10702 [("vault".to_string(), true), ("squatter".to_string(), true)],
10703 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10704 );
10705 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
10706 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10707 provides: vec!["credentials-provider/v1".to_string()],
10708 requires: Vec::new(),
10709 must_never_reach: Vec::new(),
10710 });
10711 let frame = Frame::build(
10712 FrameType::Hello,
10713 control_flags(),
10714 0,
10715 0,
10716 77,
10717 serde_json::to_vec(&ModuleHelloBody {
10718 manifest: squatter,
10719 protocol_ver: PROTOCOL_VERSION,
10720 control_ops: None,
10721 launch_nonce: None,
10722 })
10723 .expect("HELLO serializes"),
10724 )
10725 .expect("HELLO frame builds");
10726 let response = handler
10727 .handle_control(ConnectionId::new(77), frame)
10728 .expect("reserved claim receives a typed refusal");
10729 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10730 assert_eq!(
10731 registry
10732 .active_registration_count()
10733 .expect("registry reads"),
10734 0,
10735 "a reserved capability refusal must not leave a catalog entry"
10736 );
10737 }
10738
10739 #[test]
10740 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10741 for settlement in ["timeout", "inconclusive", "drop"] {
10742 let breakers = RouteBindBreakers::default();
10743 let RouteBindAdmission::Admitted {
10744 guard: mut old,
10745 probe: false,
10746 } = breakers.admit("prov")
10747 else {
10748 panic!("ordinary relay admitted")
10749 };
10750 let RouteBindAdmission::Admitted {
10751 guard: mut opener, ..
10752 } = breakers.admit("prov")
10753 else {
10754 panic!("second relay admitted")
10755 };
10756 assert!(
10757 !opener
10758 .record_timeout(1, Duration::ZERO)
10759 .unwrap()
10760 .reopened_after_probe
10761 );
10762 let RouteBindAdmission::Admitted {
10763 guard: mut probe,
10764 probe: true,
10765 } = breakers.admit("prov")
10766 else {
10767 panic!("one cooldown probe admitted")
10768 };
10769 match settlement {
10770 "timeout" => assert!(
10771 !old.record_timeout(1, Duration::ZERO)
10772 .unwrap()
10773 .reopened_after_probe
10774 ),
10775 "inconclusive" => old.record_inconclusive(),
10776 "drop" => drop(old),
10777 _ => unreachable!(),
10778 }
10779 assert!(
10780 matches!(
10781 breakers.admit("prov"),
10782 RouteBindAdmission::Refused {
10783 probe_in_flight: true,
10784 ..
10785 }
10786 ),
10787 "{settlement} of a pre-open relay cannot release the real probe"
10788 );
10789 assert!(
10790 probe
10791 .record_timeout(1, Duration::ZERO)
10792 .unwrap()
10793 .reopened_after_probe
10794 );
10795 assert!(matches!(
10796 breakers.admit("prov"),
10797 RouteBindAdmission::Admitted { probe: true, .. }
10798 ));
10799 }
10800 let breakers = RouteBindBreakers::default();
10801 let admit = || match breakers.admit("prov") {
10802 RouteBindAdmission::Admitted { guard, .. } => guard,
10803 _ => panic!("relay admitted"),
10804 };
10805 admit().record_timeout(1, Duration::ZERO);
10806 let mut old_probe = admit();
10807 breakers.reset_for_new_module_connection("prov");
10808 admit().record_timeout(1, Duration::ZERO);
10809 let _new_probe = admit();
10810 old_probe.record_inconclusive();
10811 assert!(matches!(
10812 breakers.admit("prov"),
10813 RouteBindAdmission::Refused {
10814 probe_in_flight: true,
10815 ..
10816 }
10817 ));
10818 }
10819
10820 #[tokio::test]
10821 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
10822 for candidate in [false, true] {
10823 let registry = Arc::new(Registry::default());
10824 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10825 [("vault".to_string(), true), ("squatter".to_string(), true)],
10826 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10827 );
10828 let conn = ConnectionId::new(77);
10829 let (ctx, mut rx) = route_ctx(conn);
10830 let initial = capability_manifest("squatter", &[], &[]);
10831 if candidate {
10832 registry
10833 .register_candidate_with_control_ops(
10834 initial.clone(),
10835 PROTOCOL_VERSION,
10836 conn,
10837 module_baseline_control_ops(),
10838 )
10839 .unwrap();
10840 handler
10841 .forwarding
10842 .register_candidate_module_connection(
10843 conn,
10844 "squatter".to_string(),
10845 PROTOCOL_VERSION,
10846 manifest_concurrency(&initial),
10847 ctx.egress.clone(),
10848 )
10849 .unwrap();
10850 } else {
10851 hello_via_sink(
10852 &handler,
10853 &ctx,
10854 &mut rx,
10855 hello_frame_with_manifest(initial.clone(), 1),
10856 )
10857 .await;
10858 }
10859 let response = handler
10860 .handle_control_frame(
10861 &ctx,
10862 catalog_update_with_capabilities_frame(
10863 2,
10864 capability_manifest("squatter", &["credentials-provider/v1"], &[])
10865 .capabilities
10866 .unwrap(),
10867 ),
10868 )
10869 .await
10870 .unwrap();
10871 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10872 assert_eq!(
10873 registry
10874 .get_module_by_connection(conn)
10875 .unwrap()
10876 .unwrap()
10877 .manifest,
10878 initial
10879 );
10880 }
10881 }
10882
10883 #[test]
10884 fn server_describe_surfaces_required_capability_verdict_fields() {
10885 let registry = Arc::new(Registry::default());
10886 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10887 [
10888 ("consumer".to_string(), true),
10889 ("provider".to_string(), false),
10890 ],
10891 BTreeMap::new(),
10892 );
10893 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10894 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10895 provides: Vec::new(),
10896 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10897 capability: "credentials-provider/v1".to_string(),
10898 need: subc_protocol::manifest::CapabilityNeed::Required,
10899 }],
10900 must_never_reach: Vec::new(),
10901 });
10902 let hello = Frame::build(
10903 FrameType::Hello,
10904 control_flags(),
10905 0,
10906 0,
10907 78,
10908 serde_json::to_vec(&ModuleHelloBody {
10909 manifest: consumer,
10910 protocol_ver: PROTOCOL_VERSION,
10911 control_ops: None,
10912 launch_nonce: None,
10913 })
10914 .expect("HELLO serializes"),
10915 )
10916 .expect("HELLO frame builds");
10917 handler
10918 .handle_control(ConnectionId::new(78), hello)
10919 .expect("consumer registers");
10920 let describe = Frame::build(
10921 FrameType::Request,
10922 control_flags(),
10923 0,
10924 0,
10925 79,
10926 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10927 .expect("request serializes"),
10928 )
10929 .expect("describe frame builds");
10930 let response = handler
10931 .handle_server_describe(describe)
10932 .expect("server.describe succeeds");
10933 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10934 let requirement = &rendered["capability_requirements"][0];
10935 assert_eq!(requirement["consumer"], "consumer");
10936 assert_eq!(requirement["verdict"], "never_provided");
10937 assert_eq!(requirement["episode_seq"], 1);
10938 assert_eq!(requirement["config_satisfiable"], false);
10939 assert_eq!(requirement["runtime_available"], false);
10940 assert!(requirement["detail"]
10941 .as_str()
10942 .expect("detail string")
10943 .contains("credentials-provider/v1"));
10944 }
10945
10946 fn deregister_required_provider(
10952 daemon_shutdown_begun: bool,
10953 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10954 const CAPABILITY: &str = "project-identity/v1";
10955 let registry = Arc::new(Registry::default());
10956 let daemon_shutdown = DaemonShutdownFlag::default();
10957 let handler = ControlHandler::new(Arc::clone(®istry))
10958 .with_capability_config(
10959 [
10960 ("consumer".to_string(), true),
10961 ("provider".to_string(), true),
10962 ],
10963 BTreeMap::new(),
10964 )
10965 .with_daemon_shutdown_flag(daemon_shutdown.clone());
10966 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10967 consumer.capabilities = Some(CapabilityDeclarations {
10968 provides: Vec::new(),
10969 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10970 capability: CAPABILITY.to_string(),
10971 need: subc_protocol::manifest::CapabilityNeed::Required,
10972 }],
10973 must_never_reach: Vec::new(),
10974 });
10975 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10976 let provider_conn = ConnectionId::new(182);
10977 for (conn, manifest, corr) in [
10978 (ConnectionId::new(181), consumer, 181),
10979 (provider_conn, provider, 182),
10980 ] {
10981 let hello = handler
10982 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10983 .expect("module registers");
10984 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10985 }
10986 assert_eq!(
10987 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10988 Some(CapabilityVerdict::Provided),
10989 "the registered provider must satisfy the requirement first"
10990 );
10991 if daemon_shutdown_begun {
10992 daemon_shutdown.set_for_test();
10993 }
10994
10995 let capture = EventCapture::default();
10996 let _subscriber =
10997 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10998 handler
10999 .cleanup_connection(provider_conn)
11000 .expect("provider deregisters");
11001 let events = capture
11002 .events()
11003 .into_iter()
11004 .filter(|event| {
11005 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
11006 })
11007 .collect();
11008 (
11009 events,
11010 handler.capability_evaluator.verdict("consumer", CAPABILITY),
11011 )
11012 }
11013
11014 #[test]
11015 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
11016 let (events, verdict) = deregister_required_provider(true);
11017 assert_eq!(
11018 verdict,
11019 Some(CapabilityVerdict::NeverProvided),
11020 "shutdown must not change the verdict route.open refusals read"
11021 );
11022 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11023 let event = &events[0];
11024 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
11025 assert_eq!(
11026 event.fields.get("cause").map(String::as_str),
11027 Some("\"daemon_shutdown\""),
11028 "{event:?}"
11029 );
11030 assert_eq!(
11031 event.fields.get("verdict").map(String::as_str),
11032 Some("\"never_provided\""),
11033 "{event:?}"
11034 );
11035 }
11036
11037 #[test]
11038 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
11039 let (events, verdict) = deregister_required_provider(false);
11040 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
11041 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11042 let event = &events[0];
11043 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
11044 assert_eq!(event.fields.get("cause"), None, "{event:?}");
11045 assert_eq!(
11046 event.fields.get("verdict").map(String::as_str),
11047 Some("\"never_provided\""),
11048 "{event:?}"
11049 );
11050 }
11051
11052 #[test]
11053 fn catalog_list_omits_capabilities_for_legacy_manifest() {
11054 let registry = Arc::new(Registry::default());
11055 let handler = ControlHandler::new(Arc::clone(®istry));
11056 let hello = handler
11057 .handle_control(
11058 ConnectionId::new(101),
11059 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
11060 )
11061 .expect("legacy HELLO registers");
11062 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11063
11064 let request = Frame::build(
11065 FrameType::Request,
11066 control_flags(),
11067 0,
11068 0,
11069 102,
11070 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
11071 .expect("catalog request serializes"),
11072 )
11073 .expect("catalog request frame builds");
11074 let response = handler
11075 .handle_catalog_list(request, None)
11076 .expect("catalog list succeeds");
11077 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
11078 assert!(
11079 body["modules"][0].get("capabilities").is_none(),
11080 "legacy manifest must retain an absent capabilities field on catalog.list"
11081 );
11082 }
11083
11084 #[test]
11085 fn hello_ack_omits_storage_when_no_storage_config() {
11086 let registry = Arc::new(Registry::default());
11087 let handler = ControlHandler::new(Arc::clone(®istry));
11088 let responses = handler
11089 .handle_control(
11090 ConnectionId::new(1),
11091 hello_frame("aft", PROTOCOL_VERSION, 7),
11092 )
11093 .unwrap();
11094 let ack = parse_ack(&responses[0]);
11095 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
11096 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
11097 }
11098
11099 #[tokio::test]
11100 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
11101 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
11102 let registry = Arc::new(Registry::default());
11103 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
11104 let responses = handler
11105 .handle_control(
11106 ConnectionId::new(1),
11107 hello_frame("aft", PROTOCOL_VERSION, 7),
11108 )
11109 .unwrap();
11110 let ack = parse_ack(&responses[0]);
11111 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
11112
11113 let described = handler
11114 .handle_control_frame(
11115 &route_ctx(ConnectionId::new(2)).0,
11116 Frame::build(
11117 FrameType::Request,
11118 control_flags(),
11119 0,
11120 0,
11121 9,
11122 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11123 )
11124 .unwrap(),
11125 )
11126 .await
11127 .unwrap();
11128 let ClientControlResponse::ServerDescribe { machine_id, .. } =
11129 serde_json::from_slice(&described[0].body).unwrap()
11130 else {
11131 panic!("server.describe answered with another shape");
11132 };
11133 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
11134 }
11135
11136 #[test]
11137 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
11138 let registry = Arc::new(Registry::default());
11141 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
11142 crate::daemon_config::StorageConfig::Sqlite {
11143 data_home: std::path::PathBuf::from("/data"),
11144 },
11145 ));
11146
11147 let responses = handler
11148 .handle_control(
11149 ConnectionId::new(1),
11150 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
11151 )
11152 .unwrap();
11153 let ack = parse_ack(&responses[0]);
11154 assert_eq!(
11155 ack.storage,
11156 Some(serde_json::json!({
11157 "module_id": "alfonso-routing",
11158 "storage_namespace": "default",
11159 "isolation": { "kind": "module" },
11160 "backend": {
11161 "backend": "sqlite",
11162 "path": "/data/cortexkit/alfonso-routing/store.db"
11163 }
11164 })),
11165 "the delivered descriptor is the module's own sqlite store path"
11166 );
11167 }
11168
11169 #[test]
11170 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
11171 let registry = Arc::new(Registry::default());
11172 let handler = ControlHandler::new(Arc::clone(®istry));
11173 let conn = ConnectionId::new(1);
11174 let responses = handler
11175 .handle_control(
11176 conn,
11177 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11178 )
11179 .unwrap();
11180 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11181 let registration = registry.get_module("aft").unwrap().unwrap();
11182 assert_eq!(registration.control_ops, module_baseline_control_ops());
11183
11184 let frame =
11185 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
11186 assert!(handler
11187 .guard_module_control_op(&frame, "aft", "route.bind")
11188 .unwrap()
11189 .is_none());
11190 let error = handler
11191 .guard_module_control_op(&frame, "aft", "test.synthetic")
11192 .unwrap()
11193 .expect("synthetic ungranted op should be rejected");
11194 assert_eq!(error.header.ty, FrameType::Error);
11195 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
11196 }
11197
11198 #[test]
11199 fn hello_control_ops_some_adds_optional_grants() {
11200 let registry = Arc::new(Registry::default());
11201 let handler = ControlHandler::new(Arc::clone(®istry));
11202 handler
11203 .handle_control(
11204 ConnectionId::new(1),
11205 hello_frame_with_control_ops(
11206 "aft",
11207 PROTOCOL_VERSION,
11208 7,
11209 Some(vec![
11210 "future.synthetic".to_string(),
11211 "route.bind".to_string(),
11212 ]),
11213 ),
11214 )
11215 .unwrap();
11216 let registration = registry.get_module("aft").unwrap().unwrap();
11217 assert_eq!(
11218 registration.control_ops,
11219 vec![
11220 "route.bind".to_string(),
11221 "route.status".to_string(),
11222 "future.synthetic".to_string(),
11223 ]
11224 );
11225 let frame =
11226 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
11227 assert!(handler
11228 .guard_module_control_op(&frame, "aft", "future.synthetic")
11229 .unwrap()
11230 .is_none());
11231 }
11232
11233 #[tokio::test]
11234 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
11235 let registry = Arc::new(Registry::default());
11236 let forwarding = Arc::new(ForwardingTable::default());
11237 let handler =
11238 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11239 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
11240 hello_via_sink(
11241 &handler,
11242 &module_ctx,
11243 &mut module_rx,
11244 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11245 )
11246 .await;
11247
11248 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
11249 let responses = handler
11250 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
11251 .await
11252 .unwrap();
11253 assert_eq!(responses.len(), 1);
11254 assert_eq!(responses[0].header.ty, FrameType::Error);
11255 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
11256 assert!(module_rx.try_recv().is_err());
11257 }
11258
11259 #[tokio::test]
11260 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
11261 let registry = Arc::new(Registry::default());
11262 let forwarding = Arc::new(ForwardingTable::default());
11263 let handler =
11264 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11265 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
11266 hello_via_sink(
11267 &handler,
11268 &module_ctx,
11269 &mut module_rx,
11270 hello_frame_with_control_ops(
11271 "aft",
11272 PROTOCOL_VERSION,
11273 7,
11274 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11275 ),
11276 )
11277 .await;
11278
11279 let project_root = unique_project_root("demux");
11280 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
11281 let route_handler = handler.clone();
11282 let route_task = tokio::spawn(async move {
11283 route_handler
11284 .handle_control_frame(
11285 &route_client_ctx,
11286 route_open_frame(100, "aft", project_root),
11287 )
11288 .await
11289 .unwrap()
11290 });
11291 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11292 .await
11293 .unwrap()
11294 .unwrap();
11295 assert!(matches!(
11296 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
11297 ModuleControlRequest::RouteBind { .. }
11298 ));
11299
11300 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
11301 let health_handler = handler.clone();
11302 let health_task = tokio::spawn(async move {
11303 health_handler
11304 .handle_control_frame(
11305 &health_client_ctx,
11306 supervisor_health_probe_frame(101, "aft"),
11307 )
11308 .await
11309 .unwrap()
11310 });
11311 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11312 .await
11313 .unwrap()
11314 .unwrap();
11315 assert_eq!(
11316 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11317 ModuleControlRequest::HealthCheck {}
11318 );
11319
11320 handler
11321 .handle_control_frame(
11322 &module_ctx,
11323 health_response(health_frame.header.corr, HealthStatus::Degraded),
11324 )
11325 .await
11326 .unwrap();
11327 let health_response = health_task.await.unwrap();
11328 assert_eq!(health_response.len(), 1);
11329 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11330 ClientControlResponse::SupervisorHealthProbe {
11331 module_id,
11332 status,
11333 detail,
11334 metrics,
11335 } => {
11336 assert_eq!(module_id, "aft");
11337 assert_eq!(status, HealthStatus::Degraded);
11338 assert_eq!(detail.as_deref(), Some("warming"));
11339 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
11340 }
11341 other => panic!("unexpected health response: {other:?}"),
11342 }
11343
11344 handler
11345 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11346 .await
11347 .unwrap();
11348 let route_response = route_task.await.unwrap();
11349 assert!(route_response.is_empty());
11350 let published = route_client_rx.recv().await.unwrap();
11351 assert!(matches!(
11352 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11353 ClientControlResponse::RouteOpen { .. }
11354 ));
11355 }
11356
11357 async fn relay_route_open(
11362 handler: &ControlHandler,
11363 client_connection: ConnectionId,
11364 client_egress: &FrameSink,
11365 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11366 corr: u64,
11367 module_id: &str,
11368 project_root_label: &str,
11369 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
11370 let ctx = RouteCtx {
11371 connection_id: client_connection,
11372 egress: client_egress.clone(),
11373 };
11374 let handler = handler.clone();
11375 let project_root = unique_project_root(project_root_label);
11376 let module_id = module_id.to_string();
11377 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
11378 let task = tokio::spawn(async move {
11379 let _guard = tracing::dispatcher::set_default(&dispatch);
11380 handler
11381 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
11382 .await
11383 .unwrap()
11384 });
11385 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
11386 .await
11387 .expect("module receives the relayed route.bind")
11388 .expect("module egress is open");
11389 (task, bind.frame)
11390 }
11391
11392 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
11393 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
11394 ModuleControlRequest::RouteBind {
11395 route_channel,
11396 epoch,
11397 ..
11398 } => (route_channel, epoch),
11399 other => panic!("expected a route.bind request, got {other:?}"),
11400 }
11401 }
11402
11403 fn published_route(frame: &Frame) -> (u16, u32) {
11404 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
11405 ClientControlResponse::RouteOpen {
11406 route_channel,
11407 route_epoch,
11408 ..
11409 } => (route_channel, route_epoch),
11410 other => panic!("expected a route.open response, got {other:?}"),
11411 }
11412 }
11413
11414 #[tokio::test]
11434 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
11435 let registry = Arc::new(Registry::default());
11436 let forwarding = Arc::new(ForwardingTable::default());
11437 let handler =
11438 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11439
11440 let module_connection = ConnectionId::new(30);
11441 let (module_ctx, mut module_rx) = route_ctx(module_connection);
11442 hello_via_sink(
11443 &handler,
11444 &module_ctx,
11445 &mut module_rx,
11446 hello_frame("aft", PROTOCOL_VERSION, 7),
11447 )
11448 .await;
11449
11450 let dying_client = ConnectionId::new(31);
11451 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
11452
11453 let (first_task, first_bind) = relay_route_open(
11456 &handler,
11457 dying_client,
11458 &dying_ctx.egress,
11459 &mut module_rx,
11460 100,
11461 "aft",
11462 "closing-first",
11463 )
11464 .await;
11465 handler
11466 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
11467 .await
11468 .unwrap();
11469 assert!(first_task.await.unwrap().is_empty());
11470 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
11471
11472 let (second_task, second_bind) = relay_route_open(
11474 &handler,
11475 dying_client,
11476 &dying_ctx.egress,
11477 &mut module_rx,
11478 101,
11479 "aft",
11480 "closing-second",
11481 )
11482 .await;
11483 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
11484
11485 assert!(forwarding
11488 .escalate_client_delivery_failure(
11489 dying_client,
11490 first_channel,
11491 first_epoch,
11492 CloseReason::new(
11493 "module_to_client_delivery_failed",
11494 "client egress refused a module frame",
11495 ),
11496 crate::forwarding::UndeliveredFrame {
11497 module_id: None,
11498 sink: &dying_ctx.egress,
11499 },
11500 )
11501 .unwrap());
11502 assert!(!dying_ctx.egress.is_closed());
11503
11504 let ack = handler
11506 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
11507 .await;
11508 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11509 if module_loop_error.is_some() {
11510 handler.cleanup_connection(module_connection).unwrap();
11514 }
11515 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11519 .await
11520 .ok()
11521 .flatten();
11522
11523 assert!(
11525 registry
11526 .get_module_by_connection(module_connection)
11527 .unwrap()
11528 .is_some(),
11529 "one client's closing connection ended the shared module connection: \
11530 {module_loop_error:?}"
11531 );
11532 let cotenant = ConnectionId::new(32);
11534 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11535 let (cotenant_task, cotenant_bind) = relay_route_open(
11536 &handler,
11537 cotenant,
11538 &cotenant_ctx.egress,
11539 &mut module_rx,
11540 102,
11541 "aft",
11542 "closing-cotenant",
11543 )
11544 .await;
11545 handler
11546 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
11547 .await
11548 .unwrap();
11549 assert!(cotenant_task.await.unwrap().is_empty());
11550 let (cotenant_channel, cotenant_epoch) =
11551 published_route(&cotenant_rx.recv().await.unwrap());
11552 assert!(matches!(
11553 forwarding
11554 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11555 .unwrap(),
11556 DataRoute::Client(DataRouteState::Bound(_))
11557 ));
11558
11559 let goodbye = post_ack_module_frame
11562 .expect("module receives a GOODBYE for the abandoned route channel");
11563 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11564 assert_eq!(goodbye.header.channel, abandoned_channel);
11565 assert_eq!(goodbye.header.epoch, abandoned_epoch);
11566
11567 assert!(dying_rx.try_recv().is_err());
11571 let second_response = second_task.await.unwrap();
11572 assert_eq!(second_response.len(), 1);
11573 assert_eq!(
11574 parse_error(&second_response[0])["code"],
11575 "target_unavailable"
11576 );
11577 }
11578
11579 async fn catalog_snapshot(
11582 handler: &ControlHandler,
11583 client_ctx: &RouteCtx,
11584 corr: u64,
11585 ) -> (u64, Vec<(String, bool)>) {
11586 let body =
11587 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
11588 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
11589 let reply = handler
11590 .handle_control_frame(client_ctx, request)
11591 .await
11592 .unwrap()
11593 .pop()
11594 .unwrap();
11595 let ClientControlResponse::CatalogList {
11596 generation,
11597 modules,
11598 ..
11599 } = serde_json::from_slice(&reply.body).unwrap()
11600 else {
11601 panic!("not a catalog.list reply");
11602 };
11603 (
11604 generation,
11605 modules
11606 .into_iter()
11607 .map(|entry| (entry.module_id, entry.ready))
11608 .collect(),
11609 )
11610 }
11611
11612 fn hello_held_before_forwarding() -> (
11618 ControlHandler,
11619 Arc<std::sync::Barrier>,
11620 std::thread::JoinHandle<Vec<Frame>>,
11621 RouteCtx,
11622 mpsc::Receiver<crate::router::OutboundFrame>,
11623 ) {
11624 let hold = Arc::new(std::sync::Barrier::new(2));
11625 let handler = ControlHandler::with_forwarding(
11626 Arc::new(Registry::default()),
11627 Arc::new(ForwardingTable::default()),
11628 )
11629 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
11630 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11631 let hello = {
11632 let handler = handler.clone();
11633 let module_ctx = module_ctx.clone();
11634 std::thread::spawn(move || {
11635 handler
11636 .handle_hello(
11637 module_ctx.connection_id,
11638 Some(module_ctx.egress.clone()),
11639 hello_frame("aft", PROTOCOL_VERSION, 7),
11640 )
11641 .unwrap()
11642 })
11643 };
11644 (handler, hold, hello, module_ctx, module_rx)
11645 }
11646
11647 #[tokio::test]
11656 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
11657 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
11658 let client = ConnectionId::new(41);
11659 let (client_ctx, mut client_rx) = route_ctx(client);
11660
11661 hold.wait();
11662 assert!(handler.registry.get_module("aft").unwrap().is_some());
11664 assert!(!handler
11665 .forwarding
11666 .has_live_module_connection("aft")
11667 .unwrap());
11668 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
11669 assert!(
11670 listed.is_empty(),
11671 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
11672 );
11673 let refused = handler
11674 .handle_control_frame(
11675 &client_ctx,
11676 route_open_frame(301, "aft", unique_project_root("listed-window")),
11677 )
11678 .await
11679 .unwrap();
11680 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
11681 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
11682 let duplicate = handler
11683 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
11684 .await
11685 .unwrap();
11686 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11687
11688 hold.wait();
11689 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11690 let ack = module_rx.try_recv().unwrap().frame;
11691 assert_eq!(ack.header.ty, FrameType::HelloAck);
11692 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11693 assert_eq!(listed, vec![("aft".to_string(), true)]);
11694
11695 let (open_task, bind) = relay_route_open(
11696 &handler,
11697 client,
11698 &client_ctx.egress,
11699 &mut module_rx,
11700 303,
11701 "aft",
11702 "listed-after",
11703 )
11704 .await;
11705 assert!(handler
11706 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11707 .await
11708 .unwrap()
11709 .is_empty());
11710 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
11711 published_route(&client_rx.try_recv().unwrap().frame);
11712 }
11713
11714 #[tokio::test]
11721 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11722 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11723 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11724 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11725
11726 hold.wait();
11727 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11728 assert!(listed.is_empty());
11729 assert_eq!(
11730 during, before,
11731 "the generation moved while the module was not listed"
11732 );
11733
11734 hold.wait();
11735 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11736 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11737 assert_eq!(listed, vec![("aft".to_string(), true)]);
11738 assert_eq!(
11739 after,
11740 before + 1,
11741 "the module appeared without exactly one generation bump"
11742 );
11743 }
11744
11745 #[tokio::test]
11750 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11751 let registry = Arc::new(Registry::default());
11752 let handler = ControlHandler::with_forwarding(
11753 Arc::clone(®istry),
11754 Arc::new(ForwardingTable::default()),
11755 );
11756 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11757 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11758 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11759 drop(module_rx);
11760
11761 let replies = handler
11762 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
11763 .await
11764 .unwrap();
11765 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
11766 assert!(registry.get_module("aft").unwrap().is_none());
11767 assert!(registry
11768 .get_module_by_connection(module_ctx.connection_id)
11769 .unwrap()
11770 .is_none());
11771 assert_eq!(
11772 catalog_snapshot(&handler, &client_ctx, 301).await,
11773 (before, Vec::new())
11774 );
11775 }
11776
11777 async fn route_open_held_after_bind_send() -> (
11781 ControlHandler,
11782 Arc<tokio::sync::Barrier>,
11783 RouteCtx,
11784 mpsc::Receiver<crate::router::OutboundFrame>,
11785 ) {
11786 let hold = Arc::new(tokio::sync::Barrier::new(2));
11787 let handler = ControlHandler::with_forwarding(
11788 Arc::new(Registry::default()),
11789 Arc::new(ForwardingTable::default()),
11790 )
11791 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
11792 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11793 hello_via_sink(
11794 &handler,
11795 &module_ctx,
11796 &mut module_rx,
11797 hello_frame("aft", PROTOCOL_VERSION, 7),
11798 )
11799 .await;
11800 (handler, hold, module_ctx, module_rx)
11801 }
11802
11803 #[tokio::test]
11810 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
11811 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11812 let client = ConnectionId::new(41);
11813 let (client_ctx, mut client_rx) = route_ctx(client);
11814
11815 let (open_task, bind) = relay_route_open(
11816 &handler,
11817 client,
11818 &client_ctx.egress,
11819 &mut module_rx,
11820 200,
11821 "aft",
11822 "acked-before-mark",
11823 )
11824 .await;
11825 let (module_channel, module_epoch) = route_bind_channel(&bind);
11826 hold.wait().await;
11828
11829 assert!(handler
11831 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11832 .await
11833 .unwrap()
11834 .is_empty());
11835 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
11836
11837 hold.wait().await;
11838 assert!(
11839 open_task.await.unwrap().is_empty(),
11840 "route.open answers accepted"
11841 );
11842
11843 match module_rx.try_recv() {
11846 Err(mpsc::error::TryRecvError::Empty) => {}
11847 Ok(outbound) => panic!(
11848 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
11849 outbound.frame.header.ty,
11850 outbound.frame.header.channel,
11851 outbound.frame.header.epoch
11852 ),
11853 Err(err) => panic!("module egress closed: {err}"),
11854 }
11855
11856 let router = Router::with_control_handler(Arc::new(handler.clone()));
11859 let request = Frame::build(
11860 FrameType::Request,
11861 Flags::new(true, Priority::Interactive, false),
11862 client_channel,
11863 client_epoch,
11864 201,
11865 b"first request".to_vec(),
11866 )
11867 .unwrap();
11868 router
11869 .route_for_connection(&client_ctx, request)
11870 .await
11871 .unwrap();
11872 let delivered = module_rx.try_recv().unwrap().frame;
11873 assert_eq!(delivered.header.ty, FrameType::Request);
11874 assert_eq!(
11875 (delivered.header.channel, delivered.header.epoch),
11876 (module_channel, module_epoch)
11877 );
11878 assert_eq!(delivered.body, b"first request");
11879 }
11880
11881 #[tokio::test]
11887 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
11888 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11889 let client = ConnectionId::new(42);
11890 let (client_ctx, _client_rx) = route_ctx(client);
11891
11892 let (open_task, bind) = relay_route_open(
11893 &handler,
11894 client,
11895 &client_ctx.egress,
11896 &mut module_rx,
11897 300,
11898 "aft",
11899 "released-before-mark",
11900 )
11901 .await;
11902 let (module_channel, module_epoch) = route_bind_channel(&bind);
11903 hold.wait().await;
11904
11905 handler.cleanup_connection(client).unwrap();
11906 assert!(
11907 module_rx.try_recv().is_err(),
11908 "client teardown must not GOODBYE a bind route.open is still queueing"
11909 );
11910
11911 hold.wait().await;
11912 let response = open_task.await.unwrap();
11913 assert_eq!(response.len(), 1);
11914 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11915
11916 let goodbye = module_rx
11917 .try_recv()
11918 .expect("module is told to drop the binding the daemon abandoned")
11919 .frame;
11920 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11921 assert_eq!(
11922 (goodbye.header.channel, goodbye.header.epoch),
11923 (module_channel, module_epoch)
11924 );
11925 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11926 }
11927
11928 #[tokio::test]
11933 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11934 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11935 let client = ConnectionId::new(43);
11936 let (client_ctx, client_rx) = route_ctx(client);
11937
11938 let (open_task, bind) = relay_route_open(
11939 &handler,
11940 client,
11941 &client_ctx.egress,
11942 &mut module_rx,
11943 400,
11944 "aft",
11945 "gone-client-before-mark",
11946 )
11947 .await;
11948 let (module_channel, module_epoch) = route_bind_channel(&bind);
11949 hold.wait().await;
11950
11951 drop(client_rx);
11952 assert!(client_ctx.egress.is_closed());
11953 handler
11954 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11955 .await
11956 .unwrap();
11957
11958 hold.wait().await;
11959 let response = open_task.await.unwrap();
11960 assert_eq!(response.len(), 1);
11961 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11962
11963 let goodbye = module_rx
11964 .try_recv()
11965 .expect("module is told to drop the binding it acked")
11966 .frame;
11967 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11968 assert_eq!(
11969 (goodbye.header.channel, goodbye.header.epoch),
11970 (module_channel, module_epoch)
11971 );
11972 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11973 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11974 }
11975
11976 #[test]
11983 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11984 let handler = ControlHandler::default();
11985 let module_connection = ConnectionId::new(30);
11986 let client_connection = ConnectionId::new(31);
11987
11988 handler
11989 .refuse_to_end_module_connection_for_a_client(
11990 module_connection,
11991 77,
11992 ForwardingError::ConnectionClosing {
11993 connection_id: client_connection,
11994 },
11995 )
11996 .expect("a closing client must never end the module connection");
11997
11998 assert!(matches!(
11999 handler.refuse_to_end_module_connection_for_a_client(
12000 module_connection,
12001 78,
12002 ForwardingError::ConnectionClosing {
12003 connection_id: module_connection,
12004 },
12005 ),
12006 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
12007 connection_id
12008 })) if connection_id == module_connection
12009 ));
12010 assert!(matches!(
12011 handler.refuse_to_end_module_connection_for_a_client(
12012 module_connection,
12013 79,
12014 ForwardingError::Poisoned,
12015 ),
12016 Err(RouterError::Forwarding(ForwardingError::Poisoned))
12017 ));
12018 assert!(matches!(
12019 handler.refuse_to_end_module_connection_for_a_client(
12020 module_connection,
12021 80,
12022 ForwardingError::StaleModuleEndpoint,
12023 ),
12024 Err(RouterError::Forwarding(
12025 ForwardingError::StaleModuleEndpoint
12026 ))
12027 ));
12028 }
12029
12030 #[tokio::test]
12043 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
12044 let registry = Arc::new(Registry::default());
12045 let forwarding = Arc::new(ForwardingTable::default());
12046 let supervisor = SupervisorHandle::new();
12047 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12048 let handler =
12049 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12050 .with_supervisor(supervisor);
12051
12052 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12053 hello_via_sink(
12054 &handler,
12055 &target_ctx,
12056 &mut target_rx,
12057 hello_frame("target", PROTOCOL_VERSION, 1),
12058 )
12059 .await;
12060
12061 let wrong_nonce = handler
12065 .handle_control_frame(
12066 &route_ctx(ConnectionId::new(91)).0,
12067 route_open_frame_with_admission_facts(
12068 20,
12069 "target",
12070 unique_project_root("admission-facts"),
12071 Some(subc_control::ConsumerIdentity {
12072 module_id: "fed".to_string(),
12073 launch_nonce: "not-the-real-nonce".to_string(),
12074 }),
12075 None,
12076 ),
12077 )
12078 .await
12079 .unwrap();
12080 assert_eq!(
12081 parse_error(&wrong_nonce[0])["code"],
12082 "bad_consumer_identity",
12083 "a mismatched launch nonce must be refused, not stamped Reserved"
12084 );
12085
12086 let never_spawned = handler
12090 .handle_control_frame(
12091 &route_ctx(ConnectionId::new(92)).0,
12092 route_open_frame_with_admission_facts(
12093 21,
12094 "target",
12095 unique_project_root("admission-facts"),
12096 Some(subc_control::ConsumerIdentity {
12097 module_id: "never-spawned".to_string(),
12098 launch_nonce: "any-nonce".to_string(),
12099 }),
12100 None,
12101 ),
12102 )
12103 .await
12104 .unwrap();
12105 assert_eq!(
12106 parse_error(&never_spawned[0])["code"],
12107 "bad_consumer_identity",
12108 "an unspawned module_id must be refused rather than accepted for lack of a record"
12109 );
12110 }
12111
12112 #[tokio::test]
12135 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
12136 let registry = Arc::new(Registry::default());
12137 let forwarding = Arc::new(ForwardingTable::default());
12138 let supervisor = SupervisorHandle::new();
12139 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12140 let handler =
12141 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12142 .with_supervisor(supervisor);
12143
12144 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
12145 hello_via_sink(
12146 &handler,
12147 &target_ctx,
12148 &mut target_rx,
12149 hello_frame("target", PROTOCOL_VERSION, 1),
12150 )
12151 .await;
12152
12153 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
12154 let route_handler = handler.clone();
12155 let route_task = tokio::spawn(async move {
12156 route_handler
12157 .handle_control_frame(
12158 &client_ctx,
12159 route_open_frame_with_admission_facts(
12160 30,
12161 "target",
12162 unique_project_root("admission-facts"),
12163 Some(subc_control::ConsumerIdentity {
12164 module_id: "fed".to_string(),
12165 launch_nonce: "fed-nonce".to_string(),
12166 }),
12167 None,
12168 ),
12169 )
12170 .await
12171 .unwrap()
12172 });
12173
12174 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12181 .await
12182 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
12183 .expect("module control channel closed before route.bind");
12184 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12185 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12186 panic!("expected route.bind")
12187 };
12188 assert_eq!(
12189 principal,
12190 Some(Principal::Reserved {
12191 module_id: "fed".to_string()
12192 }),
12193 "a correctly attested consumer must be stamped Reserved for its own id"
12194 );
12195
12196 handler
12197 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12198 .await
12199 .unwrap();
12200 assert!(route_task.await.unwrap().is_empty());
12201 assert!(
12202 matches!(
12203 serde_json::from_slice::<ClientControlResponse>(
12204 &client_rx.recv().await.unwrap().body
12205 )
12206 .unwrap(),
12207 ClientControlResponse::RouteOpen { .. }
12208 ),
12209 "the route must actually open, not merely avoid an error"
12210 );
12211 }
12212
12213 #[tokio::test(start_paused = true)]
12214 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
12215 let registry = Arc::new(Registry::default());
12216 let forwarding = Arc::new(ForwardingTable::default());
12217 let supervisor = SupervisorHandle::new();
12218 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12219 let handler =
12220 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12221 .with_supervisor(supervisor);
12222
12223 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
12224 hello_via_sink(
12225 &handler,
12226 &target_ctx,
12227 &mut target_rx,
12228 hello_frame("target", PROTOCOL_VERSION, 1),
12229 )
12230 .await;
12231
12232 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
12233 let direct_handler = handler.clone();
12234 let direct_open = tokio::spawn(async move {
12235 direct_handler
12236 .handle_control_frame(
12237 &direct_ctx,
12238 route_open_frame(2, "target", unique_project_root("route-census-direct")),
12239 )
12240 .await
12241 .unwrap()
12242 });
12243 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12244 .await
12245 .expect("no direct route.bind within 5s")
12246 .expect("target control channel closed before direct route.bind");
12247 handler
12248 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
12249 .await
12250 .unwrap();
12251 assert!(direct_open.await.unwrap().is_empty());
12252 let _ = direct_rx.recv().await.unwrap();
12253
12254 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
12255 let reserved_handler = handler.clone();
12256 let reserved_open = tokio::spawn(async move {
12257 reserved_handler
12258 .handle_control_frame(
12259 &reserved_ctx,
12260 route_open_frame_with_admission_facts(
12261 3,
12262 "target",
12263 unique_project_root("admission-facts"),
12264 Some(ConsumerIdentity {
12265 module_id: "fed".to_string(),
12266 launch_nonce: "fed-nonce".to_string(),
12267 }),
12268 None,
12269 ),
12270 )
12271 .await
12272 .unwrap()
12273 });
12274 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12275 .await
12276 .expect("no reserved route.bind within 5s")
12277 .expect("target control channel closed before reserved route.bind");
12278 handler
12279 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
12280 .await
12281 .unwrap();
12282 assert!(reserved_open.await.unwrap().is_empty());
12283 let _ = reserved_rx.recv().await.unwrap();
12284
12285 forwarding
12286 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
12287 .unwrap();
12288 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
12289 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
12290 module_id: Some("target".to_string()),
12291 })
12292 .unwrap();
12293 let census_frame =
12294 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
12295 let response = handler
12296 .handle_control_frame(&census_ctx, census_frame)
12297 .await
12298 .unwrap()
12299 .pop()
12300 .unwrap();
12301 let actual: Value = serde_json::from_slice(&response.body).unwrap();
12302 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
12303 assert!(matches!(
12304 decoded,
12305 ClientControlResponse::SupervisorRoutes { .. }
12306 ));
12307 let routes = actual["modules"][0]["routes"].as_array().unwrap();
12308 assert_eq!(routes.len(), 2);
12309 assert!(routes.iter().all(|route| route["draining"] == true));
12310 assert!(
12313 routes.iter().all(|route| route["drain_reason"] == "reload"),
12314 "draining routes must name the drain's reason: {routes:?}"
12315 );
12316 assert!(routes.iter().any(|route| {
12317 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
12318 }));
12319 assert!(routes.iter().any(|route| {
12320 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
12321 }));
12322
12323 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12324 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
12325 if std::env::var_os("UPDATE_GOLDEN").is_some() {
12326 std::fs::write(
12327 &golden_path,
12328 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
12329 )
12330 .unwrap();
12331 }
12332 let expected: Value =
12333 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
12334 assert_eq!(actual, expected);
12335 }
12336
12337 async fn query_live_roots(
12338 handler: &ControlHandler,
12339 module_ctx: &RouteCtx,
12340 ) -> ModuleControlResponseToModule {
12341 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
12342 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
12343 let response = handler
12344 .handle_control_frame(module_ctx, frame)
12345 .await
12346 .unwrap()
12347 .pop()
12348 .unwrap();
12349 serde_json::from_slice(&response.body).unwrap()
12350 }
12351
12352 #[tokio::test(start_paused = true)]
12353 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
12354 let registry = Arc::new(Registry::default());
12355 let forwarding = Arc::new(ForwardingTable::default());
12356 let handler = ControlHandler::with_forwarding(registry, forwarding);
12357 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
12358 hello_via_sink(
12359 &handler,
12360 &target_ctx,
12361 &mut target_rx,
12362 hello_frame("target", PROTOCOL_VERSION, 1),
12363 )
12364 .await;
12365 let root = unique_project_root("live-roots-known");
12366 let path = ProjectRootId::from_path_allowing_missing(root.path())
12367 .unwrap()
12368 .as_path()
12369 .to_path_buf();
12370 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
12371 let open_handler = handler.clone();
12372 let opened = tokio::spawn(async move {
12373 open_handler
12374 .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
12375 .await
12376 .unwrap()
12377 });
12378 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12379 .await
12380 .unwrap()
12381 .unwrap();
12382 handler
12383 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
12384 .await
12385 .unwrap();
12386 assert!(opened.await.unwrap().is_empty());
12387 let _ = client_rx.recv().await.unwrap();
12388
12389 let root = unique_project_root("live-roots-pending");
12390 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
12391 .unwrap()
12392 .as_path()
12393 .to_path_buf();
12394 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
12395 let open_handler = handler.clone();
12396 let pending = tokio::spawn(async move {
12397 open_handler
12398 .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
12399 .await
12400 .unwrap()
12401 });
12402 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12403 .await
12404 .unwrap()
12405 .unwrap();
12406 let actual = query_live_roots(&handler, &target_ctx).await;
12407 let ModuleControlResponseToModule::LiveRoots {
12408 roots,
12409 unknown_root_bindings,
12410 total_bindings,
12411 } = actual
12412 else {
12413 panic!("expected live roots")
12414 };
12415 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
12416 assert_eq!(unknown_root_bindings, 0);
12417 assert_eq!(
12418 roots.len(),
12419 2,
12420 "root-known arm must retain each canonical root"
12421 );
12422 assert_eq!(
12423 total_bindings,
12424 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12425 );
12426 let counts = roots
12427 .iter()
12428 .map(|root| (root.project_root.clone(), root.bound, root.pending))
12429 .collect::<Vec<_>>();
12430 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
12431 expected.sort_by(|a, b| a.0.cmp(&b.0));
12432 assert_eq!(
12433 counts, expected,
12434 "roots must sort by path and count pending separately"
12435 );
12436 handler
12437 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
12438 .await
12439 .unwrap();
12440 assert!(pending.await.unwrap().is_empty());
12441 }
12442
12443 #[tokio::test(start_paused = true)]
12444 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
12445 let forwarding = Arc::new(ForwardingTable::default());
12446 let handler =
12447 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12448 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
12449 hello_via_sink(
12450 &handler,
12451 &target_ctx,
12452 &mut target_rx,
12453 hello_frame("target", PROTOCOL_VERSION, 1),
12454 )
12455 .await;
12456 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
12457 let pending = forwarding
12458 .begin_route_bind_relay_for_test(
12459 client_ctx.connection_id,
12460 client_ctx.egress.clone(),
12461 2,
12462 "target",
12463 )
12464 .unwrap();
12465 forwarding
12466 .complete_pending_relay(
12467 target_ctx.connection_id,
12468 pending.corr,
12469 RouteBindRelayOutcome::Accepted,
12470 )
12471 .unwrap();
12472 let actual = query_live_roots(&handler, &target_ctx).await;
12473 let ModuleControlResponseToModule::LiveRoots {
12474 roots,
12475 unknown_root_bindings,
12476 total_bindings,
12477 } = actual
12478 else {
12479 panic!("expected live roots")
12480 };
12481 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
12482 assert_eq!(
12483 unknown_root_bindings, 1,
12484 "unknown-root arm must not read as no bindings"
12485 );
12486 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
12487 assert_eq!(
12488 total_bindings,
12489 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12490 );
12491 }
12492
12493 #[tokio::test(start_paused = true)]
12501 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
12502 let forwarding = Arc::new(ForwardingTable::default());
12503 let handler =
12504 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12505 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
12506 let replies = handler
12507 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
12508 .await
12509 .unwrap();
12510 let queued_by_hello = module_rx.len();
12511
12512 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
12513 let open_handler = handler.clone();
12514 let open = tokio::spawn(async move {
12515 open_handler
12516 .handle_control_frame(
12517 &client_ctx,
12518 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
12519 )
12520 .await
12521 .unwrap()
12522 });
12523 let mut spins = 0;
12525 while module_rx.len() == queued_by_hello {
12526 spins += 1;
12527 assert!(spins < 10_000, "route.open never queued a route.bind");
12528 tokio::task::yield_now().await;
12529 }
12530
12531 for reply in replies {
12533 module_ctx.egress.send(reply).await.unwrap();
12534 }
12535
12536 let first = module_rx.recv().await.unwrap().frame;
12537 assert_eq!(
12538 first.header.ty,
12539 FrameType::HelloAck,
12540 "the first frame a registering module reads must be its HELLO_ACK"
12541 );
12542 assert_eq!(first.header.corr, 7);
12543 let second = module_rx.recv().await.unwrap().frame;
12544 assert_eq!(second.header.ty, FrameType::Request);
12545 assert!(
12546 matches!(
12547 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
12548 ModuleControlRequest::RouteBind { .. }
12549 ),
12550 "the route.bind follows the ack"
12551 );
12552 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
12553
12554 handler
12555 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
12556 .await
12557 .unwrap();
12558 assert!(open.await.unwrap().is_empty());
12559 let _ = client_rx.recv().await.unwrap();
12560 }
12561
12562 #[tokio::test(start_paused = true)]
12563 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
12564 let handler = ControlHandler::with_forwarding(
12565 Arc::new(Registry::default()),
12566 Arc::new(ForwardingTable::default()),
12567 );
12568 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
12569 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
12570 hello_via_sink(
12571 &handler,
12572 &first_ctx,
12573 &mut first_rx,
12574 hello_frame("first", PROTOCOL_VERSION, 1),
12575 )
12576 .await;
12577 hello_via_sink(
12578 &handler,
12579 &second_ctx,
12580 &mut second_rx,
12581 hello_frame("second", PROTOCOL_VERSION, 2),
12582 )
12583 .await;
12584 let root = unique_project_root("second-only");
12585 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
12586 let cloned = handler.clone();
12587 let open = tokio::spawn(async move {
12588 cloned
12589 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
12590 .await
12591 .unwrap()
12592 });
12593 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
12594 .await
12595 .unwrap()
12596 .unwrap();
12597 let first = query_live_roots(&handler, &first_ctx).await;
12598 let second = query_live_roots(&handler, &second_ctx).await;
12599 assert!(
12600 matches!(
12601 first,
12602 ModuleControlResponseToModule::LiveRoots {
12603 total_bindings: 0,
12604 ..
12605 }
12606 ),
12607 "cross-module scope must not expose another module's roots"
12608 );
12609 assert!(
12610 matches!(
12611 second,
12612 ModuleControlResponseToModule::LiveRoots {
12613 total_bindings: 1,
12614 ..
12615 }
12616 ),
12617 "second module must see its pending route"
12618 );
12619 handler
12620 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
12621 .await
12622 .unwrap();
12623 assert!(open.await.unwrap().is_empty());
12624 }
12625
12626 #[tokio::test(start_paused = true)]
12627 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
12628 let handler = ControlHandler::with_forwarding(
12629 Arc::new(Registry::default()),
12630 Arc::new(ForwardingTable::default()),
12631 );
12632 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
12633 hello_via_sink(
12634 &handler,
12635 &target_ctx,
12636 &mut target_rx,
12637 hello_frame("target", PROTOCOL_VERSION, 1),
12638 )
12639 .await;
12640 let actual = query_live_roots(&handler, &target_ctx).await;
12641 let ModuleControlResponseToModule::LiveRoots {
12642 roots,
12643 unknown_root_bindings,
12644 total_bindings,
12645 } = actual
12646 else {
12647 panic!("expected live roots")
12648 };
12649 assert!(roots.is_empty());
12650 assert_eq!(unknown_root_bindings, 0);
12651 assert_eq!(total_bindings, 0);
12652 assert_eq!(
12653 total_bindings,
12654 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12655 );
12656 }
12657
12658 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
12665 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12666 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
12667 let text = std::fs::read_to_string(&path)
12668 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
12669 let vectors: Vec<(String, Value)> = text
12670 .lines()
12671 .filter(|line| !line.trim().is_empty())
12672 .map(|line| {
12673 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
12674 let id = entry["corpus_id"]
12675 .as_str()
12676 .expect("every vector carries a corpus_id")
12677 .to_string();
12678 (id, entry["package"].clone())
12679 })
12680 .collect();
12681 assert_eq!(
12685 vectors.len(),
12686 3,
12687 "vendored fed corpus changed size; re-sync from subc-federation"
12688 );
12689
12690 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
12705 let richest = vectors
12706 .iter()
12707 .filter_map(|(_, package)| package.as_object())
12708 .map(|object| {
12709 object
12710 .keys()
12711 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
12712 .count()
12713 })
12714 .max()
12715 .unwrap_or(0);
12716 assert!(
12717 richest >= 2,
12718 "vendored corpus no longer carries a package with unmodelled fields, \
12719 so the relay test can no longer distinguish a verbatim relay from a lossy one"
12720 );
12721
12722 vectors
12723 }
12724
12725 #[tokio::test]
12733 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12734 for (corpus_id, package) in fed_admission_facts_vectors() {
12735 let registry = Arc::new(Registry::default());
12736 let forwarding = Arc::new(ForwardingTable::default());
12737 let supervisor = SupervisorHandle::new();
12738 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12739 let handler =
12740 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12741 .with_supervisor(supervisor)
12742 .with_admission_facts_config(
12743 Some("fed".to_string()),
12744 Some(vec!["target".to_string()]),
12745 );
12746
12747 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12748 hello_via_sink(
12749 &handler,
12750 &target_ctx,
12751 &mut target_rx,
12752 hello_frame("target", PROTOCOL_VERSION, 1),
12753 )
12754 .await;
12755
12756 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12757 let route_handler = handler.clone();
12758 let expected = package.clone();
12759 let route_task = tokio::spawn(async move {
12760 route_handler
12761 .handle_control_frame(
12762 &client_ctx,
12763 route_open_frame_with_admission_facts(
12764 20,
12765 "target",
12766 unique_project_root("admission-facts"),
12767 Some(subc_control::ConsumerIdentity {
12768 module_id: "fed".to_string(),
12769 launch_nonce: "fed-nonce".to_string(),
12770 }),
12771 Some(package),
12772 ),
12773 )
12774 .await
12775 .unwrap()
12776 });
12777
12778 let bind_frame = target_rx.recv().await.unwrap();
12779 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12780 let ModuleControlRequest::RouteBind {
12781 admission_facts, ..
12782 } = bind
12783 else {
12784 panic!("{corpus_id}: expected route.bind")
12785 };
12786 assert_eq!(
12787 admission_facts,
12788 Some(expected),
12789 "{corpus_id}: relay must not add, drop or reshape any field"
12790 );
12791
12792 handler
12793 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12794 .await
12795 .unwrap();
12796 route_task.await.unwrap();
12797 }
12798 }
12799
12800 #[tokio::test]
12801 async fn route_bind_never_relays_the_openers_build_class() {
12802 let registry = Arc::new(Registry::default());
12803 let forwarding = Arc::new(ForwardingTable::default());
12804 let handler =
12805 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12806 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
12807 hello_via_sink(
12808 &handler,
12809 &target_ctx,
12810 &mut target_rx,
12811 hello_frame("target", PROTOCOL_VERSION, 1),
12812 )
12813 .await;
12814
12815 let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
12816 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
12817 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12818 busy_refusals: None,
12819 target: RouteTarget::ToolProvider {
12820 module_id: "target".to_string(),
12821 },
12822 identity,
12823 consumer_identity: None,
12824 consumer_capabilities: None,
12825 role_versions: None,
12826 admission_facts: None,
12827 scope: None,
12828 })
12829 .unwrap();
12830 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
12831
12832 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
12833 let route_handler = handler.clone();
12834 let route_task = tokio::spawn(async move {
12835 route_handler
12836 .handle_control_frame(&client_ctx, open)
12837 .await
12838 .unwrap()
12839 });
12840 let bind_frame = target_rx.recv().await.unwrap();
12841 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12842 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
12843 panic!("expected route.bind")
12844 };
12845 assert_eq!(
12846 identity.build_class, None,
12847 "route.bind carried the build class"
12848 );
12849 assert!(
12850 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
12851 "the route.bind bytes still name build_class"
12852 );
12853 handler
12854 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12855 .await
12856 .unwrap();
12857 route_task.await.unwrap();
12858 }
12859
12860 #[tokio::test]
12861 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
12862 let registry = Arc::new(Registry::default());
12863 let forwarding = Arc::new(ForwardingTable::default());
12864 let supervisor = SupervisorHandle::new();
12865 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12866 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
12867 let handler =
12868 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12869 .with_supervisor(supervisor)
12870 .with_admission_facts_config(
12871 Some("fed".to_string()),
12872 Some(vec!["target".to_string()]),
12873 );
12874
12875 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
12876 hello_via_sink(
12877 &handler,
12878 &target_ctx,
12879 &mut target_rx,
12880 hello_frame("target", PROTOCOL_VERSION, 1),
12881 )
12882 .await;
12883 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
12884 hello_via_sink(
12885 &handler,
12886 &other_ctx,
12887 &mut other_rx,
12888 hello_frame("other", PROTOCOL_VERSION, 2),
12889 )
12890 .await;
12891
12892 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
12893 let expected_facts = facts.clone();
12894 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
12895 let route_handler = handler.clone();
12896 let route_task = tokio::spawn(async move {
12897 route_handler
12898 .handle_control_frame(
12899 &client_ctx,
12900 route_open_frame_with_admission_facts(
12901 10,
12902 "target",
12903 unique_project_root("admission-facts"),
12904 Some(subc_control::ConsumerIdentity {
12905 module_id: "fed".to_string(),
12906 launch_nonce: "fed-nonce".to_string(),
12907 }),
12908 Some(facts.clone()),
12909 ),
12910 )
12911 .await
12912 .unwrap()
12913 });
12914 let bind_frame = target_rx.recv().await.unwrap();
12915 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12916 let ModuleControlRequest::RouteBind {
12917 admission_facts, ..
12918 } = bind
12919 else {
12920 panic!("expected route.bind")
12921 };
12922 assert_eq!(admission_facts, Some(expected_facts));
12923 handler
12924 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12925 .await
12926 .unwrap();
12927 assert!(route_task.await.unwrap().is_empty());
12928 assert!(matches!(
12929 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
12930 .unwrap(),
12931 ClientControlResponse::RouteOpen { .. }
12932 ));
12933
12934 let direct = handler
12935 .handle_control_frame(
12936 &route_ctx(ConnectionId::new(73)).0,
12937 route_open_frame_with_admission_facts(
12938 11,
12939 "target",
12940 unique_project_root("admission-facts"),
12941 None,
12942 Some(json!({"x": 1})),
12943 ),
12944 )
12945 .await
12946 .unwrap();
12947 assert_eq!(
12948 parse_error(&direct[0])["code"],
12949 "admission_facts_not_permitted"
12950 );
12951
12952 let different_reserved = handler
12953 .handle_control_frame(
12954 &route_ctx(ConnectionId::new(77)).0,
12955 route_open_frame_with_admission_facts(
12956 15,
12957 "target",
12958 unique_project_root("admission-facts"),
12959 Some(subc_control::ConsumerIdentity {
12960 module_id: "other".to_string(),
12961 launch_nonce: "other-nonce".to_string(),
12962 }),
12963 Some(json!({"x": 1})),
12964 ),
12965 )
12966 .await
12967 .unwrap();
12968 assert_eq!(
12969 parse_error(&different_reserved[0])["code"],
12970 "admission_facts_not_permitted"
12971 );
12972
12973 let other_target = handler
12974 .handle_control_frame(
12975 &route_ctx(ConnectionId::new(74)).0,
12976 route_open_frame_with_admission_facts(
12977 12,
12978 "other",
12979 unique_project_root("admission-facts"),
12980 Some(subc_control::ConsumerIdentity {
12981 module_id: "fed".to_string(),
12982 launch_nonce: "fed-nonce".to_string(),
12983 }),
12984 Some(json!({"x": 1})),
12985 ),
12986 )
12987 .await
12988 .unwrap();
12989 assert_eq!(
12990 parse_error(&other_target[0])["code"],
12991 "admission_facts_target_not_allowed"
12992 );
12993
12994 let nonexistent = handler
12995 .handle_control_frame(
12996 &route_ctx(ConnectionId::new(75)).0,
12997 route_open_frame_with_admission_facts(
12998 13,
12999 "missing",
13000 unique_project_root("admission-facts"),
13001 None,
13002 Some(json!({"x": 1})),
13003 ),
13004 )
13005 .await
13006 .unwrap();
13007 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
13008
13009 let described = handler
13010 .handle_control_frame(
13011 &route_ctx(ConnectionId::new(76)).0,
13012 Frame::build(
13013 FrameType::Request,
13014 control_flags(),
13015 0,
13016 0,
13017 14,
13018 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13019 )
13020 .unwrap(),
13021 )
13022 .await
13023 .unwrap();
13024 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13025 serde_json::from_slice(&described[0].body).unwrap()
13026 else {
13027 panic!("expected server.describe response")
13028 };
13029 assert!(capabilities
13030 .iter()
13031 .any(|cap| cap == "admission_facts_relay_v1"));
13032 }
13033
13034 #[tokio::test]
13035 async fn admission_facts_without_configured_carrier_are_rejected() {
13036 let registry = Arc::new(Registry::default());
13037 let forwarding = Arc::new(ForwardingTable::default());
13038 let handler = ControlHandler::with_forwarding(registry, forwarding);
13039 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
13040 hello_via_sink(
13041 &handler,
13042 &target_ctx,
13043 &mut target_rx,
13044 hello_frame("target", PROTOCOL_VERSION, 1),
13045 )
13046 .await;
13047
13048 let responses = handler
13049 .handle_control_frame(
13050 &route_ctx(ConnectionId::new(79)).0,
13051 route_open_frame_with_admission_facts(
13052 16,
13053 "target",
13054 unique_project_root("admission-facts"),
13055 None,
13056 Some(json!({"x": 1})),
13057 ),
13058 )
13059 .await
13060 .unwrap();
13061 assert_eq!(
13062 parse_error(&responses[0])["code"],
13063 "admission_facts_not_permitted"
13064 );
13065 }
13066
13067 #[tokio::test]
13068 async fn route_open_relays_consumer_capabilities_verbatim() {
13069 let registry = Arc::new(Registry::default());
13070 let forwarding = Arc::new(ForwardingTable::default());
13071 let handler =
13072 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13073 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
13074 hello_via_sink(
13075 &handler,
13076 &module_ctx,
13077 &mut module_rx,
13078 hello_frame("aft", PROTOCOL_VERSION, 7),
13079 )
13080 .await;
13081
13082 let expected = vec!["elicitation".to_string(), "roots".to_string()];
13083 let expected_for_request = expected.clone();
13084 let project_root = unique_project_root("consumer-capabilities-present");
13085 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
13086 let route_handler = handler.clone();
13087 let route_task = tokio::spawn(async move {
13088 route_handler
13089 .handle_control_frame(
13090 &client_ctx,
13091 route_open_frame_with_consumer_capabilities(
13092 401,
13093 "aft",
13094 project_root,
13095 Some(expected_for_request),
13096 ),
13097 )
13098 .await
13099 .unwrap()
13100 });
13101 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13102 .await
13103 .unwrap()
13104 .unwrap();
13105 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13106 let ModuleControlRequest::RouteBind {
13107 consumer_capabilities,
13108 ..
13109 } = bind
13110 else {
13111 panic!("expected route.bind request, got {bind:?}");
13112 };
13113 assert_eq!(consumer_capabilities, Some(expected.clone()));
13114
13115 handler
13116 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13117 .await
13118 .unwrap();
13119 let route_response = route_task.await.unwrap();
13120 assert!(route_response.is_empty());
13121 let published = client_rx.recv().await.unwrap();
13122 assert!(matches!(
13123 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13124 ClientControlResponse::RouteOpen { .. }
13125 ));
13126 }
13127
13128 #[tokio::test]
13129 async fn route_open_without_consumer_capabilities_relays_none() {
13130 let registry = Arc::new(Registry::default());
13131 let forwarding = Arc::new(ForwardingTable::default());
13132 let handler =
13133 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13134 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
13135 hello_via_sink(
13136 &handler,
13137 &module_ctx,
13138 &mut module_rx,
13139 hello_frame("aft", PROTOCOL_VERSION, 7),
13140 )
13141 .await;
13142
13143 let project_root = unique_project_root("consumer-capabilities-absent");
13144 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
13145 let route_handler = handler.clone();
13146 let route_task = tokio::spawn(async move {
13147 route_handler
13148 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
13149 .await
13150 .unwrap()
13151 });
13152 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13153 .await
13154 .unwrap()
13155 .unwrap();
13156 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13157 let ModuleControlRequest::RouteBind {
13158 consumer_capabilities,
13159 ..
13160 } = bind
13161 else {
13162 panic!("expected route.bind request, got {bind:?}");
13163 };
13164 assert_eq!(consumer_capabilities, None);
13165
13166 handler
13167 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13168 .await
13169 .unwrap();
13170 let route_response = route_task.await.unwrap();
13171 assert!(route_response.is_empty());
13172 let published = client_rx.recv().await.unwrap();
13173 assert!(matches!(
13174 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13175 ClientControlResponse::RouteOpen { .. }
13176 ));
13177 }
13178
13179 async fn bind_role_versions_for(
13183 sent: Option<BTreeMap<String, String>>,
13184 connection: u64,
13185 ) -> Option<BTreeMap<String, String>> {
13186 let registry = Arc::new(Registry::default());
13187 let forwarding = Arc::new(ForwardingTable::default());
13188 let handler =
13189 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13190 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
13191 hello_via_sink(
13192 &handler,
13193 &module_ctx,
13194 &mut module_rx,
13195 hello_frame("aft", PROTOCOL_VERSION, 7),
13196 )
13197 .await;
13198 let project_root = unique_project_root("role-versions");
13199 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
13200 let route_handler = handler.clone();
13201 let route_task = tokio::spawn(async move {
13202 route_handler
13203 .handle_control_frame(
13204 &client_ctx,
13205 route_open_frame_with_role_versions(403, "aft", project_root, sent),
13206 )
13207 .await
13208 .unwrap()
13209 });
13210 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13211 .await
13212 .expect("a well-formed route.open reaches the module as a bind")
13213 .unwrap();
13214 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13215 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
13216 panic!("expected route.bind request, got {bind:?}");
13217 };
13218 handler
13219 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13220 .await
13221 .unwrap();
13222 assert!(route_task.await.unwrap().is_empty());
13223 let published = client_rx.recv().await.unwrap();
13224 assert!(matches!(
13225 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13226 ClientControlResponse::RouteOpen { .. }
13227 ));
13228 role_versions
13229 }
13230
13231 #[tokio::test]
13232 async fn route_open_relays_role_versions_verbatim() {
13233 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
13234 assert_eq!(
13235 bind_role_versions_for(Some(sent.clone()), 141).await,
13236 Some(sent)
13237 );
13238 }
13239
13240 #[tokio::test]
13243 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
13244 assert_eq!(bind_role_versions_for(None, 143).await, None);
13245 assert_eq!(
13246 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
13247 None
13248 );
13249 }
13250
13251 #[tokio::test]
13252 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
13253 let registry = Arc::new(Registry::default());
13254 let forwarding = Arc::new(ForwardingTable::default());
13255 let handler =
13256 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13257 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
13258 hello_via_sink(
13259 &handler,
13260 &module_ctx,
13261 &mut module_rx,
13262 hello_frame("aft", PROTOCOL_VERSION, 7),
13263 )
13264 .await;
13265
13266 let nine: BTreeMap<String, String> = (0..9)
13267 .map(|index| (format!("role-{index}"), "v1".to_string()))
13268 .collect();
13269 for (label, malformed) in [
13270 (
13271 "invalid role name",
13272 role_versions(&[("Tool_Provider", "v1")]),
13273 ),
13274 ("invalid version", role_versions(&[("tool-provider", "v0")])),
13275 ("nine entries", nine),
13276 ] {
13277 let responses = tokio::time::timeout(
13280 Duration::from_secs(1),
13281 handler.handle_control_frame(
13282 &route_ctx(ConnectionId::new(148)).0,
13283 route_open_frame_with_role_versions(
13284 404,
13285 "aft",
13286 unique_project_root("role-versions-malformed"),
13287 Some(malformed),
13288 ),
13289 ),
13290 )
13291 .await
13292 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
13293 .unwrap();
13294 assert_eq!(responses.len(), 1, "{label}");
13295 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
13296 let error = parse_error(&responses[0]);
13297 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
13298 assert_eq!(
13299 error["detail"]["field"], "role_versions",
13300 "{label}: {error}"
13301 );
13302 assert!(
13303 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
13304 "{label}: a malformed declaration is terminal"
13305 );
13306 assert!(
13307 module_rx.try_recv().is_err(),
13308 "{label}: the module must never see a bind"
13309 );
13310 }
13311 }
13312
13313 #[tokio::test]
13317 async fn route_role_versions_capability_is_advertised() {
13318 let handler = ControlHandler::new(Arc::new(Registry::default()));
13319 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
13320 let ack = hello_via_sink(
13321 &handler,
13322 &ctx,
13323 &mut rx,
13324 hello_frame("m", PROTOCOL_VERSION, 1),
13325 )
13326 .await;
13327 let ack = parse_ack(&ack);
13328 assert!(
13329 ack.subc_capabilities
13330 .iter()
13331 .any(|c| c == "route-role-versions/v1"),
13332 "{:?}",
13333 ack.subc_capabilities
13334 );
13335
13336 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13337 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13338 let reply = handler
13339 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
13340 .await
13341 .unwrap()
13342 .pop()
13343 .unwrap();
13344 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13345 serde_json::from_slice(&reply.body).unwrap()
13346 else {
13347 panic!("not a server.describe reply");
13348 };
13349 assert!(
13350 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
13351 "{capabilities:?}"
13352 );
13353 }
13354
13355 #[tokio::test]
13359 async fn catalog_events_capability_is_advertised() {
13360 let handler = ControlHandler::new(Arc::new(Registry::default()));
13361 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13362 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13363 let reply = handler
13364 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
13365 .await
13366 .unwrap()
13367 .pop()
13368 .unwrap();
13369 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13370 serde_json::from_slice(&reply.body).unwrap()
13371 else {
13372 panic!("not a server.describe reply");
13373 };
13374 assert!(
13375 capabilities
13376 .iter()
13377 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
13378 "{capabilities:?}"
13379 );
13380 }
13381
13382 #[tokio::test]
13383 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
13384 let handler = ControlHandler::new(Arc::new(Registry::default()));
13385 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
13386 let ack = hello_via_sink(
13387 &handler,
13388 &module,
13389 &mut module_rx,
13390 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
13391 )
13392 .await;
13393 let ack = parse_ack(&ack);
13394 assert!(
13395 ack.subc_capabilities
13396 .iter()
13397 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13398 "{:?}",
13399 ack.subc_capabilities
13400 );
13401 assert!(ack
13402 .subc_capabilities
13403 .iter()
13404 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13405
13406 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
13407 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13408 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13409 let reply = handler
13410 .handle_control_frame(&client, frame)
13411 .await
13412 .unwrap()
13413 .pop()
13414 .unwrap();
13415 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13416 serde_json::from_slice(&reply.body).unwrap()
13417 else {
13418 panic!("not a server.describe reply");
13419 };
13420 assert!(
13421 capabilities
13422 .iter()
13423 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13424 "{capabilities:?}"
13425 );
13426 assert!(capabilities
13427 .iter()
13428 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13429 }
13430
13431 #[tokio::test]
13432 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
13433 let handler = ControlHandler::new(Arc::new(Registry::default()));
13434 let (module, mut rx) = route_ctx(ConnectionId::new(156));
13435 let ack = hello_via_sink(
13436 &handler,
13437 &module,
13438 &mut rx,
13439 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
13440 )
13441 .await;
13442 assert!(parse_ack(&ack)
13443 .subc_capabilities
13444 .iter()
13445 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13446 assert!(parse_ack(&ack)
13447 .subc_capabilities
13448 .iter()
13449 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13450 let (client, _rx) = route_ctx(ConnectionId::new(157));
13451 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13452 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13453 let reply = handler
13454 .handle_control_frame(&client, frame)
13455 .await
13456 .unwrap()
13457 .pop()
13458 .unwrap();
13459 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13460 serde_json::from_slice(&reply.body).unwrap()
13461 else {
13462 panic!("server.describe reply")
13463 };
13464 assert!(capabilities
13465 .iter()
13466 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13467 assert!(capabilities
13468 .iter()
13469 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13470 }
13471
13472 #[tokio::test]
13473 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
13474 let handler = ControlHandler::new(Arc::new(Registry::default()));
13475 let (module, mut rx) = route_ctx(ConnectionId::new(158));
13476 let ack = hello_via_sink(
13477 &handler,
13478 &module,
13479 &mut rx,
13480 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
13481 )
13482 .await;
13483 assert!(parse_ack(&ack)
13484 .subc_capabilities
13485 .iter()
13486 .any(|cap| cap == "scope-workdir/v1"));
13487 let (client, _rx) = route_ctx(ConnectionId::new(159));
13488 let frame = Frame::build(
13489 FrameType::Request,
13490 control_flags(),
13491 0,
13492 0,
13493 5,
13494 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13495 )
13496 .unwrap();
13497 let reply = handler
13498 .handle_control_frame(&client, frame)
13499 .await
13500 .unwrap()
13501 .remove(0);
13502 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13503 serde_json::from_slice(&reply.body).unwrap()
13504 else {
13505 panic!("server.describe reply")
13506 };
13507 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
13508 }
13509
13510 #[tokio::test]
13511 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
13512 let handler = ControlHandler::new(Arc::new(Registry::default()));
13513 let (module, mut rx) = route_ctx(ConnectionId::new(160));
13514 let ack = hello_via_sink(
13515 &handler,
13516 &module,
13517 &mut rx,
13518 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
13519 )
13520 .await;
13521 assert!(parse_ack(&ack)
13522 .subc_capabilities
13523 .iter()
13524 .any(|cap| cap == "worker-runs/v1"));
13525 let (client, _rx) = route_ctx(ConnectionId::new(161));
13526 let frame = Frame::build(
13527 FrameType::Request,
13528 control_flags(),
13529 0,
13530 0,
13531 5,
13532 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13533 )
13534 .unwrap();
13535 let reply = handler
13536 .handle_control_frame(&client, frame)
13537 .await
13538 .unwrap()
13539 .remove(0);
13540 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13541 serde_json::from_slice(&reply.body).unwrap()
13542 else {
13543 panic!("server.describe reply")
13544 };
13545 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
13546 }
13547
13548 #[tokio::test]
13549 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
13550 let handler = ControlHandler::new(Arc::new(Registry::default()));
13551 let (module, mut rx) = route_ctx(ConnectionId::new(154));
13552 let ack = hello_via_sink(
13553 &handler,
13554 &module,
13555 &mut rx,
13556 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
13557 )
13558 .await;
13559 assert!(parse_ack(&ack)
13560 .subc_capabilities
13561 .iter()
13562 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13563 let (client, _rx) = route_ctx(ConnectionId::new(155));
13564 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13565 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13566 let reply = handler
13567 .handle_control_frame(&client, frame)
13568 .await
13569 .unwrap()
13570 .pop()
13571 .unwrap();
13572 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13573 serde_json::from_slice(&reply.body).unwrap()
13574 else {
13575 panic!("not a server.describe reply")
13576 };
13577 assert!(capabilities
13578 .iter()
13579 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13580 }
13581
13582 #[tokio::test]
13583 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
13584 let registry = Arc::new(Registry::default());
13585 let forwarding = Arc::new(ForwardingTable::default());
13586 let handler =
13587 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13588 .with_health_probe_timeout(Duration::from_secs(5));
13589 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
13590 hello_via_sink(
13591 &handler,
13592 &module_ctx,
13593 &mut module_rx,
13594 non_routable_hello_frame_with_control_ops(
13595 "mcp",
13596 300,
13597 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13598 ),
13599 )
13600 .await;
13601 assert!(registry
13602 .get_module("mcp")
13603 .unwrap()
13604 .unwrap()
13605 .manifest
13606 .provides
13607 .is_empty());
13608
13609 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
13610 let route_response = handler
13611 .handle_control_frame(
13612 &route_client_ctx,
13613 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
13614 )
13615 .await
13616 .unwrap();
13617 assert_eq!(route_response[0].header.ty, FrameType::Error);
13618 assert_eq!(
13619 parse_error(&route_response[0])["code"],
13620 "target_unavailable"
13621 );
13622 assert!(parse_error(&route_response[0])["message"]
13623 .as_str()
13624 .unwrap()
13625 .contains("does not provide the requested target"));
13626 assert!(module_rx.try_recv().is_err());
13627
13628 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
13629 let health_handler = handler.clone();
13630 let health_task = tokio::spawn(async move {
13631 health_handler
13632 .handle_control_frame(
13633 &health_client_ctx,
13634 supervisor_health_probe_frame(302, "mcp"),
13635 )
13636 .await
13637 .unwrap()
13638 });
13639 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13640 .await
13641 .unwrap()
13642 .unwrap();
13643 assert_eq!(
13644 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13645 ModuleControlRequest::HealthCheck {}
13646 );
13647 handler
13648 .handle_control_frame(
13649 &module_ctx,
13650 health_response(health_frame.header.corr, HealthStatus::Ok),
13651 )
13652 .await
13653 .unwrap();
13654 let health_response = health_task.await.unwrap();
13655 assert_eq!(health_response[0].header.ty, FrameType::Response);
13656 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13657 ClientControlResponse::SupervisorHealthProbe {
13658 module_id, status, ..
13659 } => {
13660 assert_eq!(module_id, "mcp");
13661 assert_eq!(status, HealthStatus::Ok);
13662 }
13663 other => panic!("unexpected health response: {other:?}"),
13664 }
13665
13666 forwarding
13671 .cleanup_connection(module_ctx.connection_id)
13672 .unwrap();
13673 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
13674 let cleanup_response = tokio::time::timeout(
13675 Duration::from_millis(200),
13676 handler.handle_control_frame(
13677 &cleanup_probe_ctx,
13678 supervisor_health_probe_frame(303, "mcp"),
13679 ),
13680 )
13681 .await
13682 .expect("probe should fail immediately when the control lane is gone")
13683 .unwrap();
13684 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
13685 assert_eq!(
13686 parse_error(&cleanup_response[0])["code"],
13687 "target_unavailable"
13688 );
13689 assert!(parse_error(&cleanup_response[0])["message"]
13690 .as_str()
13691 .unwrap()
13692 .contains("no module connection"));
13693
13694 handler
13695 .cleanup_connection(module_ctx.connection_id)
13696 .unwrap();
13697 }
13698
13699 #[tokio::test]
13700 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
13701 let registry = Arc::new(Registry::default());
13702 let supervisor_handle = SupervisorHandle::new();
13703 let supervisor =
13704 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13705 .with_handle(supervisor_handle.clone())
13706 .with_connection_file_path(
13707 std::env::temp_dir()
13708 .join(format!("subc-route-open-warming-{}", std::process::id())),
13709 );
13710 let module = supervisor
13711 .supervise_configured(
13712 ModuleSpec {
13713 module_id: "warming".to_string(),
13714 program: fake_aft_stub_path(),
13715 args: Vec::new(),
13716 env: Vec::new(),
13717 reserved: false,
13718 reserved_prefixes: Vec::new(),
13719 protocol: ModuleProtocol::Subc,
13720 overlap: Default::default(),
13721 },
13722 true,
13723 )
13724 .unwrap();
13725 assert_eq!(module.state().unwrap(), ModuleState::Running);
13726
13727 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13728 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13729 let response = handler
13730 .handle_control_frame(
13731 &ctx,
13732 route_open_frame(304, "warming", unique_project_root("warming")),
13733 )
13734 .await
13735 .unwrap();
13736 module.stop().await.unwrap();
13737
13738 assert_eq!(response[0].header.ty, FrameType::Error);
13739 let error = parse_error(&response[0]);
13740 assert_eq!(error["code"], "module_warming");
13741 assert!(error["message"]
13742 .as_str()
13743 .unwrap()
13744 .contains("state=running, enabled=true, live=false"));
13745 }
13746
13747 #[test]
13748 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13749 let handler = ControlHandler::new(Arc::new(Registry::default()));
13750 let capture = EventCapture::default();
13751 let _subscriber =
13752 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13753 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13754 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
13755 let pending = (0..limit).collect::<Vec<_>>();
13756 let response = handler
13757 .route_open_capacity_refusal(
13758 &ctx,
13759 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
13760 "busy",
13761 pending.len(),
13762 limit,
13763 )
13764 .unwrap();
13765 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13766 let event = capture
13767 .events()
13768 .into_iter()
13769 .find(|event| {
13770 event.target == "control"
13771 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
13772 })
13773 .expect("connection admission refusal event");
13774 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13775 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13776 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13779 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13780 }
13781
13782 #[test]
13783 fn route_open_target_cap_logs_admission_reason_and_capacity() {
13784 let handler = ControlHandler::new(Arc::new(Registry::default()));
13785 let capture = EventCapture::default();
13786 let _subscriber =
13787 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13788 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
13789 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
13790 let guards = (0..limit)
13791 .map(|_| {
13792 handler
13793 .route_bind_concurrency
13794 .try_admit("busy", limit)
13795 .unwrap()
13796 })
13797 .collect::<Vec<_>>();
13798 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
13799 Err(in_flight) => in_flight,
13800 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
13801 };
13802 let response = handler
13803 .route_open_target_capacity_refusal(
13804 &ctx,
13805 &RouteOpenCaller::unknown(),
13806 &route_open_frame(397, "busy", unique_project_root("target-cap")),
13807 "busy",
13808 in_flight,
13809 )
13810 .unwrap();
13811 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13812 let event = capture
13813 .events()
13814 .into_iter()
13815 .find(|event| {
13816 event.target == "control"
13817 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
13818 })
13819 .expect("target admission refusal event");
13820 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13821 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13822 drop(guards);
13823 }
13824
13825 #[cfg(unix)]
13826 #[tokio::test(start_paused = true)]
13827 async fn supervisor_list_shows_a_failed_modules_next_retry() {
13828 let registry = Arc::new(Registry::default());
13829 let handle = SupervisorHandle::new();
13830 let supervisor =
13831 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
13832 .with_handle(handle.clone());
13833 let module = supervisor
13834 .spawn(crate::ModuleSpec {
13835 module_id: "retry-list".into(),
13836 program: PathBuf::from("/bin/sh"),
13837 args: vec!["-c".into(), "exit 17".into()],
13838 env: vec![],
13839 reserved: false,
13840 reserved_prefixes: vec![],
13841 protocol: ModuleProtocol::None,
13842 overlap: Default::default(),
13843 })
13844 .unwrap();
13845 let deadline = std::time::Instant::now() + Duration::from_secs(10);
13846 while module.state().unwrap() != crate::ModuleState::Failed {
13847 assert!(
13848 std::time::Instant::now() < deadline,
13849 "waiting for budget failure"
13850 );
13851 tokio::task::yield_now().await;
13852 }
13853 let handler = ControlHandler::new(registry).with_supervisor(handle);
13854 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13855 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13856 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13857 assert_eq!(body["modules"][0]["state"], "failed");
13858 let retry = body["modules"][0]["next_retry_at_ms"]
13859 .as_u64()
13860 .expect("scheduled retry on the wire");
13861 let now = std::time::SystemTime::now()
13862 .duration_since(std::time::UNIX_EPOCH)
13863 .unwrap()
13864 .as_millis() as u64;
13865 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
13866 module.set_enabled(false).await.unwrap();
13867 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13868 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13869 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13870 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
13871 }
13872
13873 #[tokio::test]
13874 async fn slow_step_before_pid_read_does_not_spend_its_budget() {
13875 let _guard = crate::subtree_resources::fault_read_guard().await;
13876 let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
13877 let supervisor =
13878 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
13879 let module = supervisor
13880 .spawn(crate::ModuleSpec {
13881 module_id: "pid-read-budget".into(),
13882 program: std::env::current_exe().unwrap(),
13883 args: vec![
13884 "--exact".into(),
13885 "control::tests::pid_resource_read_fixture".into(),
13886 ],
13887 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
13888 .into_iter()
13889 .map(|key| (key.into(), dir.display().to_string()))
13890 .chain(std::iter::once((
13891 "SUBC_SUBTREE_PID_FIXTURE".into(),
13892 "1".into(),
13893 )))
13894 .collect(),
13895 reserved: false,
13896 reserved_prefixes: Vec::new(),
13897 protocol: ModuleProtocol::None,
13898 overlap: Default::default(),
13899 })
13900 .unwrap();
13901 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
13902 while !matches!(
13903 module.child_resource_usage(),
13904 subc_control::ChildResourceUsage::Measured(_)
13905 ) {
13906 assert!(
13907 tokio::time::Instant::now() < ready_deadline,
13908 "fixture process was not measurable"
13909 );
13910 tokio::time::sleep(Duration::from_millis(10)).await;
13911 }
13912 tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
13913 let reading = crate::subtree_resources::read_process(module.clone()).await;
13914 module.stop().await.unwrap();
13915 assert!(
13916 matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
13917 "{reading:?}"
13918 );
13919 }
13920
13921 #[test]
13922 fn pid_resource_read_fixture() {
13923 if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
13924 loop {
13925 std::thread::park_timeout(Duration::from_secs(600));
13926 }
13927 }
13928 }
13929
13930 #[tokio::test]
13931 async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
13932 let _guard = crate::subtree_resources::fault_read_guard().await;
13933 let registry = Arc::new(Registry::default());
13934 let handle = SupervisorHandle::new();
13935 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
13936 .with_handle(handle.clone());
13937 let module = supervisor
13938 .supervise_configured(
13939 crate::ModuleSpec {
13940 module_id: "memory-read".into(),
13941 program: PathBuf::from("disabled-module"),
13942 args: Vec::new(),
13943 env: Vec::new(),
13944 reserved: false,
13945 reserved_prefixes: Vec::new(),
13946 protocol: ModuleProtocol::Subc,
13947 overlap: Default::default(),
13948 },
13949 false,
13950 )
13951 .unwrap();
13952 let (target, release, began) = crate::subtree_resources::held();
13953 module.set_subtree_target_for_test(target);
13954 let handler = ControlHandler::new(registry).with_supervisor(handle);
13955 let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
13956 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13957 let replies = handler
13958 .handle_client_control_request(
13959 &ctx,
13960 frame,
13961 ClientControlRequest::SupervisorList {},
13962 &mut None,
13963 )
13964 .await
13965 .unwrap();
13966 assert!(
13967 replies.is_empty(),
13968 "list reply must be detached from the connection reader"
13969 );
13970 began.await.unwrap();
13971 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13972 let replies = handler
13973 .handle_client_control_request(
13974 &ctx,
13975 frame,
13976 ClientControlRequest::SupervisorSpawnSnapshot {},
13977 &mut None,
13978 )
13979 .await
13980 .unwrap();
13981 assert_eq!(
13982 replies.len(),
13983 1,
13984 "another request must complete while the subtree read stalls"
13985 );
13986 let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
13987 .await
13988 .unwrap()
13989 .unwrap();
13990 release.send(()).unwrap();
13991 drop(release);
13992 let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
13993 assert_eq!(body["modules"][0]["subtree"]["source"], "unavailable");
13994 assert_eq!(body["modules"][0]["subtree"]["reason"], "timeout");
13995 assert_eq!(
13996 body["modules"][0]["subtree"]["process_count"], 1,
13997 "a count obtained before the stalled read must survive timeout"
13998 );
13999 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14000 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14001 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14002 assert_eq!(
14003 body["modules"][0]["subtree"]["source"], "process_sum",
14004 "the next request must read afresh"
14005 );
14006 }
14007
14008 #[tokio::test]
14009 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
14010 let registry = Arc::new(Registry::default());
14011 let handle = SupervisorHandle::new();
14012 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14013 .with_handle(handle.clone());
14014 supervisor
14015 .supervise_configured(
14016 crate::ModuleSpec {
14017 module_id: "drift".into(),
14018 program: PathBuf::from("disabled-module"),
14019 args: Vec::new(),
14020 env: Vec::new(),
14021 reserved: false,
14022 reserved_prefixes: Vec::new(),
14023 protocol: ModuleProtocol::Subc,
14024 overlap: Default::default(),
14025 },
14026 false,
14027 )
14028 .unwrap();
14029 let handler = ControlHandler::new(registry).with_supervisor(handle);
14030 for configured in [None, Some(false), Some(true)] {
14031 handler.capability_evaluator.configure(
14032 configured.map(|enabled| ("drift".to_string(), enabled)),
14033 BTreeMap::new(),
14034 );
14035 let frame =
14036 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14037 let replies = handler
14038 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
14039 .await
14040 .unwrap();
14041 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14042 assert!(body.get("configured_enabled").is_none());
14043 assert_eq!(body["op"], "supervisor.ack");
14044 let frame =
14045 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14046 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14047 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14048 assert_eq!(
14049 body["modules"][0].get("configured_enabled"),
14050 configured.map(Value::Bool).as_ref()
14051 );
14052 assert_eq!(body["modules"][0]["subtree"]["source"], "unavailable");
14053 assert_eq!(body["modules"][0]["subtree"]["reason"], "not_running");
14054 }
14055 }
14056
14057 #[tokio::test(flavor = "current_thread")]
14058 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
14059 let handler = ControlHandler::new(Arc::new(Registry::default()));
14060 let capture = EventCapture::default();
14061 let _subscriber =
14062 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14063 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
14064 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14065
14066 let _ = handler
14067 .handle_client_control_request(
14068 &ctx,
14069 frame,
14070 ClientControlRequest::SupervisorSetEnabled {
14071 module_id: "broca".to_string(),
14072 enabled: false,
14073 },
14074 &mut None,
14075 )
14076 .await
14077 .unwrap();
14078
14079 let events = capture
14080 .events()
14081 .into_iter()
14082 .filter(|event| {
14083 event.target == "control"
14084 && event.fields.get("message").map(String::as_str)
14085 == Some("supervisor request received")
14086 })
14087 .collect::<Vec<_>>();
14088 assert_eq!(events.len(), 1, "one supervisor request log line");
14089 let event = &events[0];
14090 assert_eq!(event.level, tracing::Level::INFO);
14091 assert_eq!(
14092 event.fields.get("op"),
14093 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
14094 );
14095 assert_eq!(
14096 event.fields.get("module_id"),
14097 Some(&"\"broca\"".to_string())
14098 );
14099 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
14100 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
14101 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
14102 }
14103
14104 #[tokio::test(flavor = "current_thread")]
14105 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
14106 let registry = Arc::new(Registry::default());
14107 let module_connection = ConnectionId::new(303);
14108 registry
14109 .register_with_control_ops(
14110 manifest("aft", PROTOCOL_VERSION),
14111 PROTOCOL_VERSION,
14112 module_connection,
14113 module_baseline_control_ops(),
14114 )
14115 .unwrap();
14116 let handler = ControlHandler::new(registry);
14117 let capture = EventCapture::default();
14118 let _subscriber =
14119 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14120 let (ctx, _rx) = route_ctx(module_connection);
14121 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14122
14123 let _ = handler
14124 .handle_client_control_request(
14125 &ctx,
14126 frame,
14127 ClientControlRequest::SupervisorSetEnabled {
14128 module_id: "broca".to_string(),
14129 enabled: false,
14130 },
14131 &mut None,
14132 )
14133 .await
14134 .unwrap();
14135
14136 let callers = capture
14137 .events()
14138 .into_iter()
14139 .filter(|event| {
14140 event.fields.get("message").map(String::as_str)
14141 == Some("supervisor request received")
14142 })
14143 .map(|event| event.fields.get("caller").cloned())
14144 .collect::<Vec<_>>();
14145 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
14146 }
14147
14148 #[tokio::test(flavor = "current_thread")]
14149 async fn supervisor_list_does_not_log_request_received() {
14150 let handler = ControlHandler::new(Arc::new(Registry::default()));
14151 let capture = EventCapture::default();
14152 let _subscriber =
14153 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14154 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
14155 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14156
14157 handler
14158 .handle_client_control_request(
14159 &ctx,
14160 frame,
14161 ClientControlRequest::SupervisorList {},
14162 &mut None,
14163 )
14164 .await
14165 .unwrap();
14166
14167 assert!(capture.events().into_iter().all(|event| {
14168 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14169 }));
14170 }
14171
14172 #[tokio::test(flavor = "current_thread")]
14173 async fn supervisor_rescan_preview_does_not_log_request_received() {
14174 let handler = ControlHandler::new(Arc::new(Registry::default()));
14175 let capture = EventCapture::default();
14176 let _subscriber =
14177 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14178 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
14179 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14180
14181 handler
14182 .handle_client_control_request(
14183 &ctx,
14184 frame,
14185 ClientControlRequest::SupervisorRescan { preview: true },
14186 &mut None,
14187 )
14188 .await
14189 .unwrap();
14190
14191 assert!(capture.events().into_iter().all(|event| {
14192 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14193 }));
14194 }
14195
14196 struct ApproveOperator;
14197 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
14198 fn prompt(
14199 &self,
14200 _: &str,
14201 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
14202 ) -> crate::operator_confirm::ProviderResult {
14203 crate::operator_confirm::ProviderResult::Approved
14204 }
14205 }
14206
14207 #[tokio::test]
14208 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
14209 assert_eq!(
14210 ControlHandler::default().privileged_operator_prompt,
14211 PrivilegedOperatorPrompt::Log
14212 );
14213 for (initial, changed, running) in [
14214 ("log", "enforce", PrivilegedOperatorPrompt::Log),
14215 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
14216 ] {
14217 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
14218 let path = dir.join("subc.jsonc");
14219 std::fs::write(
14220 &path,
14221 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
14222 )
14223 .unwrap();
14224 let config = crate::daemon_config::load(&path).unwrap().unwrap();
14225 let registry = Arc::new(Registry::default());
14226 let handle = SupervisorHandle::new();
14227 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
14228 .with_handle(handle.clone());
14229 let handler = ControlHandler::new(registry)
14230 .with_supervisor(handle)
14231 .with_privileged_operator_prompt(config.privileged_operator_prompt)
14232 .with_supervisor_rescan(supervisor, &path, config.port);
14233 handler.forwarding.operator_confirms().configure(
14234 Arc::new(ApproveOperator),
14235 crate::operator_confirm::OperatorLimits::default(),
14236 );
14237 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
14238
14239 for (disk_mode, needs_restart) in [
14240 (Some(initial), false),
14241 (Some(changed), true),
14242 (Some(changed), true),
14243 (None, running == PrivilegedOperatorPrompt::Enforce),
14244 (Some(initial), false),
14245 ] {
14246 let mut doc = json!({"version":1});
14247 if let Some(mode) = disk_mode {
14248 doc["privileged_operator_prompt"] = json!(mode);
14249 }
14250 std::fs::write(&path, doc.to_string()).unwrap();
14251 for preview in [true, false] {
14252 let request = ClientControlRequest::SupervisorRescan { preview };
14253 let frame = Frame::build(
14254 FrameType::Request,
14255 control_flags(),
14256 0,
14257 0,
14258 1,
14259 serde_json::to_vec(&request).unwrap(),
14260 )
14261 .unwrap();
14262 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14263 assert_eq!(frames.len(), 1);
14264 let ClientControlResponse::SupervisorRescan { result } =
14265 serde_json::from_slice(&frames[0].body).unwrap()
14266 else {
14267 panic!("expected a rescan response");
14268 };
14269 let expected = if needs_restart {
14270 vec!["privileged_operator_prompt"]
14271 } else {
14272 Vec::new()
14273 };
14274 assert_eq!(result.restart_required, expected);
14275 assert_eq!(handler.privileged_operator_prompt, running);
14276 assert_eq!(
14277 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
14278 running
14279 );
14280 }
14281 }
14282 }
14283 }
14284
14285 #[tokio::test]
14286 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
14287 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
14288 let path = dir.join("subc.jsonc");
14289 let registry = Arc::new(Registry::default());
14290 let handle = SupervisorHandle::new();
14291 let supervisor =
14292 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
14293 let handler = ControlHandler::new(registry)
14294 .with_supervisor(handle)
14295 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
14296 .with_supervisor_rescan(supervisor, &path, None);
14297 handler.forwarding.operator_confirms().configure(
14298 Arc::new(ApproveOperator),
14299 crate::operator_confirm::OperatorLimits::default(),
14300 );
14301 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
14302 for preview in [true, false] {
14303 std::fs::write(
14304 &path,
14305 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
14306 )
14307 .unwrap();
14308 let request = ClientControlRequest::SupervisorRescan { preview };
14309 let frame = Frame::build(
14310 FrameType::Request,
14311 control_flags(),
14312 0,
14313 0,
14314 1,
14315 serde_json::to_vec(&request).unwrap(),
14316 )
14317 .unwrap();
14318 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14319 assert_eq!(frames.len(), 1);
14320 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
14321 assert_eq!(
14322 handler.privileged_operator_prompt,
14323 PrivilegedOperatorPrompt::Enforce
14324 );
14325 }
14326 }
14327
14328 #[tokio::test]
14333 async fn route_open_refusal_names_the_check_that_refused() {
14334 let handler = ControlHandler::new(Arc::new(Registry::default()));
14335 let capture = EventCapture::default();
14336 let _subscriber =
14337 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14338 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14339 let response = handler
14340 .handle_control_frame(
14341 &ctx,
14342 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
14343 )
14344 .await
14345 .unwrap();
14346
14347 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14348 let event = capture
14349 .events()
14350 .into_iter()
14351 .find(|event| {
14352 event.target == "control"
14353 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14354 })
14355 .expect("route.open refusal event");
14356 assert_eq!(
14357 event.fields.get("reason"),
14358 Some(&"\"not_registered\"".to_string())
14359 );
14360 }
14361
14362 #[tokio::test]
14363 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
14364 let registry = Arc::new(Registry::default());
14365 let supervisor_handle = SupervisorHandle::new();
14366 let supervisor =
14367 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
14368 .with_handle(supervisor_handle.clone())
14369 .with_connection_file_path(std::env::temp_dir().join(format!(
14370 "subc-route-open-refusal-info-{}",
14371 std::process::id()
14372 )));
14373 let module = supervisor
14374 .supervise_configured(
14375 ModuleSpec {
14376 module_id: "warming".to_string(),
14377 program: fake_aft_stub_path(),
14378 args: Vec::new(),
14379 env: Vec::new(),
14380 reserved: false,
14381 reserved_prefixes: Vec::new(),
14382 protocol: ModuleProtocol::Subc,
14383 overlap: Default::default(),
14384 },
14385 true,
14386 )
14387 .unwrap();
14388 assert_eq!(module.state().unwrap(), ModuleState::Running);
14389
14390 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14391 assert!(handler
14392 .counters()
14393 .snapshot()
14394 .get("route_open_refused_by_code")
14395 .is_none());
14396 let capture = EventCapture::default();
14397 let _subscriber =
14398 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14399 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
14400 let response = handler
14401 .handle_control_frame(
14402 &ctx,
14403 route_open_frame(394, "warming", unique_project_root("refusal-info")),
14404 )
14405 .await
14406 .unwrap();
14407 module.stop().await.unwrap();
14408
14409 assert_eq!(parse_error(&response[0])["code"], "module_warming");
14410 let event = capture
14411 .events()
14412 .into_iter()
14413 .find(|event| {
14414 event.target == "control"
14415 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
14416 })
14417 .expect("route.open refusal event");
14418 assert_eq!(
14419 event.fields.get("module_id"),
14420 Some(&"\"warming\"".to_string())
14421 );
14422 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
14423 assert_eq!(
14424 event.fields.get("reason"),
14425 Some(&"\"supervised_not_registered\"".to_string())
14426 );
14427 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
14428 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
14429 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
14430 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14431 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14432 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
14433 assert_eq!(
14434 handler.counters().snapshot()["route_open_refused_by_code"],
14435 json!({ "module_warming": 1 })
14436 );
14437 }
14438
14439 const OUTAGE_START: &str = "route.open refusing module: not serving";
14440 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
14441
14442 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
14443 capture
14444 .events()
14445 .into_iter()
14446 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
14447 .collect()
14448 }
14449
14450 fn supervise_stub(
14451 registry: &Arc<Registry>,
14452 module_id: &str,
14453 enabled: bool,
14454 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
14455 let supervisor_handle = SupervisorHandle::new();
14456 let supervisor =
14457 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
14458 .with_handle(supervisor_handle.clone())
14459 .with_connection_file_path(std::env::temp_dir().join(format!(
14460 "subc-route-outage-{module_id}-{}",
14461 std::process::id()
14462 )));
14463 let module = supervisor
14464 .supervise_configured(
14465 ModuleSpec {
14466 module_id: module_id.to_string(),
14467 program: fake_aft_stub_path(),
14468 args: Vec::new(),
14469 env: Vec::new(),
14470 reserved: false,
14471 reserved_prefixes: Vec::new(),
14472 protocol: ModuleProtocol::Subc,
14473 overlap: Default::default(),
14474 },
14475 enabled,
14476 )
14477 .unwrap();
14478 (supervisor_handle, module)
14479 }
14480
14481 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
14482 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
14483 module_id: module_id.to_string(),
14484 drain_timeout_ms: Some(50),
14485 })
14486 .unwrap();
14487 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
14488 }
14489
14490 #[test]
14494 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
14495 let registry = Arc::new(Registry::default());
14496 let forwarding = Arc::new(ForwardingTable::default());
14497 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14498 let second = ControlHandler::with_forwarding(registry, forwarding);
14499 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
14500 }
14501
14502 #[tokio::test(flavor = "current_thread")]
14506 async fn route_open_unknown_module_refusals_add_no_outage_state() {
14507 let handler = ControlHandler::new(Arc::new(Registry::default()));
14508 let capture = EventCapture::default();
14509 let _subscriber =
14510 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14511 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
14512 for corr in 0..8 {
14513 let response = handler
14514 .handle_control_frame(
14515 &ctx,
14516 route_open_frame(
14517 380 + corr,
14518 &format!("nobody-{corr}"),
14519 unique_project_root("outage-unknown"),
14520 ),
14521 )
14522 .await
14523 .unwrap();
14524 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14525 }
14526
14527 assert_eq!(handler.route_outages.tracked_module_count(), 0);
14528 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
14529 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
14530 }
14531
14532 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14537 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
14538 let registry = Arc::new(Registry::default());
14539 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
14540 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14541 let capture = EventCapture::default();
14542 let _subscriber =
14543 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14544 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
14545 handler.route_outages.record_accepted("outage-restart");
14548
14549 let response = handler
14550 .handle_control_frame(
14551 &ctx,
14552 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
14553 )
14554 .await
14555 .unwrap();
14556 assert_eq!(response[0].header.ty, FrameType::Error);
14557 let starts = outage_lines(&capture, OUTAGE_START);
14558 assert_eq!(starts.len(), 1, "{starts:?}");
14559 assert_eq!(starts[0].level, tracing::Level::WARN);
14560 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
14561 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
14562 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
14563 handler.route_outages.record_accepted("outage-restart");
14564 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
14565
14566 let restart = handler
14567 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
14568 .await
14569 .unwrap();
14570 assert_eq!(
14571 restart[0].header.ty,
14572 FrameType::Response,
14573 "{:?}",
14574 parse_error(&restart[0])
14575 );
14576 handler
14577 .handle_control_frame(
14578 &ctx,
14579 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
14580 )
14581 .await
14582 .unwrap();
14583 module.stop().await.unwrap();
14584
14585 let starts = outage_lines(&capture, OUTAGE_START);
14586 assert_eq!(starts.len(), 2, "{starts:?}");
14587 assert_eq!(starts[1].level, tracing::Level::INFO);
14588 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
14589 }
14590
14591 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14595 async fn failed_operator_restart_leaves_no_operator_mark() {
14596 let registry = Arc::new(Registry::default());
14597 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
14598 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14599 let capture = EventCapture::default();
14600 let _subscriber =
14601 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14602 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
14603 handler.route_outages.record_accepted("outage-disabled");
14604
14605 let restart = handler
14606 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
14607 .await
14608 .unwrap();
14609 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
14610 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
14611
14612 handler
14613 .handle_control_frame(
14614 &ctx,
14615 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
14616 )
14617 .await
14618 .unwrap();
14619 let starts = outage_lines(&capture, OUTAGE_START);
14620 assert_eq!(starts.len(), 1, "{starts:?}");
14621 assert_eq!(starts[0].level, tracing::Level::WARN);
14622 }
14623
14624 #[tokio::test(flavor = "current_thread")]
14625 async fn route_open_unknown_module_escapes_target_module_id() {
14626 let handler = ControlHandler::new(Arc::new(Registry::default()));
14627 let capture = EventCapture::default();
14628 let _subscriber =
14629 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14630 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
14631 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14632 let response = handler
14633 .handle_control_frame(
14634 &ctx,
14635 route_open_frame(
14636 395,
14637 hostile_module_id,
14638 unique_project_root("hostile-target-module-id"),
14639 ),
14640 )
14641 .await
14642 .unwrap();
14643
14644 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14645 let event = capture
14646 .events()
14647 .into_iter()
14648 .find(|event| {
14649 event.target == "control"
14650 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14651 })
14652 .expect("route.open unknown-module refusal event");
14653 let logged = event.fields.get("module_id").expect("module_id field");
14654 assert!(!logged.bytes().any(|byte| byte < 0x20));
14655 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14656 }
14657
14658 #[tokio::test(flavor = "current_thread")]
14661 async fn route_open_refusal_carries_daemon_origin() {
14662 let handler = ControlHandler::new(Arc::new(Registry::default()));
14663 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14664 let response = handler
14665 .handle_control_frame(
14666 &ctx,
14667 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
14668 )
14669 .await
14670 .unwrap();
14671
14672 assert_eq!(response.len(), 1);
14673 assert_eq!(response[0].header.ty, FrameType::Error);
14674 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14675 assert!(
14676 response[0].header.flags.is_daemon_origin(),
14677 "route.open refusal must carry daemon origin: {:?}",
14678 response[0].header.flags
14679 );
14680 }
14681
14682 #[tokio::test(flavor = "current_thread")]
14686 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
14687 let registry = Arc::new(Registry::default());
14688 let forwarding = Arc::new(ForwardingTable::default());
14689 let handler =
14690 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14691 let module_connection = ConnectionId::new(95);
14692 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14693 hello_via_sink(
14694 &handler,
14695 &module_ctx,
14696 &mut module_rx,
14697 hello_frame("aft", PROTOCOL_VERSION, 395),
14698 )
14699 .await;
14700 forwarding
14701 .begin_module_drain("aft", RouteCloseReason::Restart)
14702 .unwrap()
14703 .expect("the module endpoint drains");
14704
14705 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
14706 let response = handler
14707 .handle_control_frame(
14708 &client_ctx,
14709 route_open_frame(396, "aft", unique_project_root("drain-origin")),
14710 )
14711 .await
14712 .unwrap();
14713
14714 assert_eq!(response.len(), 1);
14715 assert_eq!(response[0].header.ty, FrameType::Error);
14716 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14717 assert!(
14718 response[0].header.flags.is_daemon_origin(),
14719 "drain module_reloading must carry daemon origin: {:?}",
14720 response[0].header.flags
14721 );
14722 }
14723
14724 #[tokio::test(flavor = "current_thread")]
14728 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
14729 let registry = Arc::new(Registry::default());
14730 let forwarding = Arc::new(ForwardingTable::default());
14731 let handler =
14732 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14733 let module_connection = ConnectionId::new(95);
14734 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14735 hello_via_sink(
14736 &handler,
14737 &module_ctx,
14738 &mut module_rx,
14739 hello_frame("aft", PROTOCOL_VERSION, 395),
14740 )
14741 .await;
14742
14743 let client_connection = ConnectionId::new(96);
14744 let (client_ctx, _client_rx) = route_ctx(client_connection);
14745 let (route_task, bind) = relay_route_open(
14746 &handler,
14747 client_connection,
14748 &client_ctx.egress,
14749 &mut module_rx,
14750 396,
14751 "aft",
14752 "spoofed-reloading",
14753 )
14754 .await;
14755 let rejection = Frame::build(
14758 FrameType::Error,
14759 crate::router::daemon_error_flags(),
14760 0,
14761 0,
14762 bind.header.corr,
14763 serde_json::to_vec(&ErrorBody::new(
14764 "module_reloading",
14765 "module refused route.bind",
14766 ))
14767 .unwrap(),
14768 )
14769 .unwrap();
14770 handler
14771 .handle_control_frame(&module_ctx, rejection)
14772 .await
14773 .unwrap();
14774
14775 let response = route_task.await.unwrap();
14776 assert_eq!(response.len(), 1);
14777 assert_eq!(response[0].header.ty, FrameType::Error);
14778 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14779 assert!(
14780 !response[0].header.flags.is_daemon_origin(),
14781 "a module's own rejection must not carry daemon origin: {:?}",
14782 response[0].header.flags
14783 );
14784 }
14785
14786 #[tokio::test(flavor = "current_thread")]
14787 async fn route_open_module_rejection_uses_daemon_counter_key() {
14788 let registry = Arc::new(Registry::default());
14789 let forwarding = Arc::new(ForwardingTable::default());
14790 let handler =
14791 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14792 let module_connection = ConnectionId::new(95);
14793 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14794 hello_via_sink(
14795 &handler,
14796 &module_ctx,
14797 &mut module_rx,
14798 hello_frame("aft", PROTOCOL_VERSION, 395),
14799 )
14800 .await;
14801
14802 let client_connection = ConnectionId::new(96);
14803 let (client_ctx, _client_rx) = route_ctx(client_connection);
14804 let capture = EventCapture::default();
14805 let _subscriber =
14806 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14807 let (route_task, bind) = relay_route_open(
14808 &handler,
14809 client_connection,
14810 &client_ctx.egress,
14811 &mut module_rx,
14812 396,
14813 "aft",
14814 "hostile-module-code",
14815 )
14816 .await;
14817 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
14818 let rejection = Frame::build(
14819 FrameType::Error,
14820 control_flags(),
14821 0,
14822 0,
14823 bind.header.corr,
14824 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
14825 )
14826 .unwrap();
14827 handler
14828 .handle_control_frame(&module_ctx, rejection)
14829 .await
14830 .unwrap();
14831
14832 let response = route_task.await.unwrap();
14833 assert_eq!(parse_error(&response[0])["code"], hostile_code);
14834 let counters = handler.counters().snapshot();
14835 assert_eq!(
14836 counters["route_open_refused_by_code"],
14837 json!({ "module_rejected": 1 })
14838 );
14839 assert!(counters["route_open_refused_by_code"]
14840 .get(hostile_code)
14841 .is_none());
14842
14843 let event = capture
14844 .events()
14845 .into_iter()
14846 .find(|event| {
14847 event.target == "control"
14848 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
14849 })
14850 .expect("route.open module-rejection refusal event");
14851 let logged = event.fields.get("module_code").expect("module_code field");
14852 assert!(!logged.bytes().any(|byte| byte < 0x20));
14853 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14854 }
14855
14856 #[tokio::test]
14857 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
14858 let registry = Arc::new(Registry::default());
14859 let supervisor_handle = SupervisorHandle::new();
14860 let missing_program = std::env::temp_dir().join(format!(
14861 "subc-route-open-missing-program-{}",
14862 std::process::id()
14863 ));
14864 let supervisor =
14865 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
14866 .with_handle(supervisor_handle.clone());
14867 let module = supervisor
14868 .supervise_configured(
14869 ModuleSpec {
14870 module_id: "failed".to_string(),
14871 program: missing_program,
14872 args: Vec::new(),
14873 env: Vec::new(),
14874 reserved: false,
14875 reserved_prefixes: Vec::new(),
14876 protocol: ModuleProtocol::Subc,
14877 overlap: Default::default(),
14878 },
14879 true,
14880 )
14881 .unwrap();
14882 assert_eq!(module.state().unwrap(), ModuleState::Failed);
14883
14884 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14885 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
14886 let response = handler
14887 .handle_control_frame(
14888 &ctx,
14889 route_open_frame(305, "failed", unique_project_root("failed")),
14890 )
14891 .await
14892 .unwrap();
14893
14894 assert_eq!(response[0].header.ty, FrameType::Error);
14895 let error = parse_error(&response[0]);
14896 assert_eq!(error["code"], "target_unavailable");
14897 assert!(error["message"]
14898 .as_str()
14899 .unwrap()
14900 .contains("state=failed, enabled=true, live=false"));
14901 }
14902
14903 #[tokio::test]
14904 async fn route_open_role_mismatch_remains_target_unavailable() {
14905 let registry = Arc::new(Registry::default());
14906 let handler = ControlHandler::new(Arc::clone(®istry));
14907 handler
14908 .handle_control(
14909 ConnectionId::new(41),
14910 non_routable_hello_frame_with_control_ops("health-only", 306, None),
14911 )
14912 .unwrap();
14913
14914 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
14915 let response = handler
14916 .handle_control_frame(
14917 &ctx,
14918 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
14919 )
14920 .await
14921 .unwrap();
14922
14923 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14924 assert!(parse_error(&response[0])["message"]
14925 .as_str()
14926 .unwrap()
14927 .contains("does not provide the requested target"));
14928 }
14929
14930 #[tokio::test]
14931 async fn route_open_inactive_registration_remains_target_unavailable() {
14932 let registry = Arc::new(Registry::default());
14933 let handler = ControlHandler::new(Arc::clone(®istry));
14934 handler
14935 .handle_control(
14936 ConnectionId::new(43),
14937 hello_frame("inactive", PROTOCOL_VERSION, 308),
14938 )
14939 .unwrap();
14940 assert!(registry
14941 .set_module_state_for_test("inactive", ChannelState::Closed)
14942 .unwrap());
14943
14944 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
14945 let response = handler
14946 .handle_control_frame(
14947 &ctx,
14948 route_open_frame(309, "inactive", unique_project_root("inactive")),
14949 )
14950 .await
14951 .unwrap();
14952
14953 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14954 assert!(parse_error(&response[0])["message"]
14955 .as_str()
14956 .unwrap()
14957 .contains("is not active"));
14958 }
14959
14960 #[tokio::test]
14961 async fn late_health_reply_is_recorded_through_the_module_response_path() {
14962 let registry = Arc::new(Registry::default());
14963 let forwarding = Arc::new(ForwardingTable::default());
14964 let supervisor_handle = SupervisorHandle::new();
14965 let supervisor =
14966 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
14967 .with_forwarding(Arc::clone(&forwarding))
14968 .with_handle(supervisor_handle.clone());
14969 let module = supervisor
14970 .supervise_configured(
14971 crate::ModuleSpec {
14972 module_id: "late-health-response".to_string(),
14973 program: PathBuf::from("disabled-module"),
14974 args: Vec::new(),
14975 env: Vec::new(),
14976 reserved: false,
14977 reserved_prefixes: Vec::new(),
14978 protocol: ModuleProtocol::Subc,
14979 overlap: Default::default(),
14980 },
14981 false,
14982 )
14983 .unwrap();
14984 let handler =
14985 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14986 .with_supervisor(supervisor_handle);
14987 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
14988 handler
14989 .handle_control_frame(
14990 &module_ctx,
14991 hello_frame_with_control_ops(
14992 "late-health-response",
14993 PROTOCOL_VERSION,
14994 7,
14995 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14996 ),
14997 )
14998 .await
14999 .unwrap();
15000 let probe_started_at = Instant::now() - Duration::from_millis(80);
15001 let pending = forwarding
15002 .begin_health_probe_rpc_for(
15003 "late-health-response",
15004 MODULE_CONTROL_OP_HEALTH_CHECK,
15005 probe_started_at,
15006 Instant::now() - Duration::from_millis(1),
15007 )
15008 .unwrap();
15009 assert!(forwarding
15010 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
15011 .unwrap());
15012
15013 let responses = handler
15014 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
15015 .await
15016 .unwrap();
15017
15018 assert!(responses.is_empty());
15019 let health = module.status().unwrap().health;
15020 assert_eq!(health.late_answer_count, 1);
15021 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
15022 }
15023
15024 #[tokio::test]
15025 async fn health_probe_timeout_and_module_death_are_typed() {
15026 let registry = Arc::new(Registry::default());
15027 let forwarding = Arc::new(ForwardingTable::default());
15028 let handler =
15029 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15030 .with_health_probe_timeout(Duration::from_millis(50));
15031 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
15032 hello_via_sink(
15033 &handler,
15034 &module_ctx,
15035 &mut module_rx,
15036 hello_frame_with_control_ops(
15037 "aft",
15038 PROTOCOL_VERSION,
15039 7,
15040 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15041 ),
15042 )
15043 .await;
15044
15045 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
15046 let responses = handler
15047 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
15048 .await
15049 .unwrap();
15050 assert_eq!(responses[0].header.ty, FrameType::Error);
15051 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
15052 let _ = module_rx.try_recv();
15053
15054 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
15055 let health_handler = handler.clone();
15056 let death_task = tokio::spawn(async move {
15057 health_handler
15058 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
15059 .await
15060 .unwrap()
15061 });
15062 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15063 .await
15064 .unwrap()
15065 .unwrap();
15066 handler
15067 .cleanup_connection(module_ctx.connection_id)
15068 .unwrap();
15069 let responses = death_task.await.unwrap();
15070 assert_eq!(responses[0].header.ty, FrameType::Error);
15071 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
15072 }
15073
15074 #[test]
15075 fn hello_requires_exact_protocol_version() {
15076 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
15077 let registry = Arc::new(Registry::default());
15078 let handler = ControlHandler::new(Arc::clone(®istry));
15079 let responses = handler
15080 .handle_control(
15081 ConnectionId::new(connection),
15082 hello_frame("aft", offered, 9),
15083 )
15084 .unwrap();
15085
15086 assert_eq!(responses.len(), 1);
15087 assert_eq!(responses[0].header.ty, FrameType::Error);
15088 let error = parse_error(&responses[0]);
15089 assert_eq!(error["code"], "version_unsupported");
15090 assert!(registry.get_module("aft").unwrap().is_none());
15091 assert_eq!(registry.active_registration_count().unwrap(), 0);
15092 }
15093 }
15094
15095 #[test]
15096 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
15097 let registry = Arc::new(Registry::default());
15098 let forwarding = Arc::new(ForwardingTable::default());
15099 let handler =
15100 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15101 let module_connection = ConnectionId::new(301);
15102 let registration = registry
15103 .register_with_control_ops(
15104 manifest("aft-push", PROTOCOL_VERSION),
15105 PROTOCOL_VERSION,
15106 module_connection,
15107 module_baseline_control_ops(),
15108 )
15109 .unwrap();
15110 let (module_tx, _module_rx) = mpsc::channel(8);
15111 let endpoint = forwarding
15112 .register_module_connection(
15113 module_connection,
15114 "aft-push".to_string(),
15115 PROTOCOL_VERSION,
15116 manifest_concurrency(®istration.manifest),
15117 FrameSink::new(module_tx),
15118 )
15119 .unwrap();
15120
15121 let unknown = Frame::build(
15123 FrameType::Push,
15124 control_flags(),
15125 0,
15126 0,
15127 5,
15128 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
15129 )
15130 .unwrap();
15131 let out = handler.handle_status_update(endpoint, unknown).unwrap();
15132 assert!(
15133 out.is_empty(),
15134 "unknown push op must be ignored, got {out:?}"
15135 );
15136
15137 let malformed = Frame::build(
15139 FrameType::Push,
15140 control_flags(),
15141 0,
15142 0,
15143 6,
15144 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
15145 )
15146 .unwrap();
15147 let out = handler.handle_status_update(endpoint, malformed).unwrap();
15148 assert_eq!(out.len(), 1);
15149 assert_eq!(out[0].header.ty, FrameType::Error);
15150 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
15151 }
15152
15153 #[test]
15154 fn hello_rejected_when_connection_already_owns_client_routes() {
15155 let registry = Arc::new(Registry::default());
15156 let forwarding = Arc::new(ForwardingTable::default());
15157 let handler =
15158 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15159 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
15161 let client_connection = ConnectionId::new(202);
15162
15163 let responses = handler
15166 .handle_control(
15167 client_connection,
15168 hello_frame("aft-second", PROTOCOL_VERSION, 9),
15169 )
15170 .unwrap();
15171 assert_eq!(responses[0].header.ty, FrameType::Error);
15172 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
15173 assert!(registry.get_module("aft-second").unwrap().is_none());
15174 }
15175
15176 #[tokio::test]
15177 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
15178 let registry = Arc::new(Registry::default());
15179 let forwarding = Arc::new(ForwardingTable::default());
15180 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
15181 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
15182 hello_via_sink(
15183 &handler,
15184 &module_ctx,
15185 &mut module_rx,
15186 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
15187 )
15188 .await;
15189 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
15190 let pending = forwarding
15191 .begin_route_bind_relay_for_test(
15192 client_ctx.connection_id,
15193 client_ctx.egress.clone(),
15194 2,
15195 "alpha",
15196 )
15197 .unwrap();
15198 forwarding
15199 .complete_pending_relay(
15200 module_ctx.connection_id,
15201 pending.corr,
15202 RouteBindRelayOutcome::Accepted,
15203 )
15204 .unwrap();
15205 client_rx.try_recv().unwrap();
15206 for module_id in ["beta", "alpha"] {
15207 let replies = handler
15208 .handle_control_frame(
15209 &module_ctx,
15210 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
15211 )
15212 .await
15213 .unwrap();
15214 assert_eq!(replies.len(), 1, "second HELLO must be refused");
15215 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
15216 }
15217 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
15218 assert!(registry.get_module("beta").unwrap().is_none());
15219 assert!(matches!(
15220 forwarding
15221 .lookup_data_route(
15222 client_ctx.connection_id,
15223 pending.client_channel,
15224 pending.client_epoch,
15225 )
15226 .unwrap(),
15227 DataRoute::Client(DataRouteState::Bound(_))
15228 ));
15229 assert!(handler
15230 .hello_launch_nonces
15231 .lock()
15232 .unwrap()
15233 .presented(module_ctx.connection_id, Some("alpha-nonce")));
15234 assert!(module_rx.try_recv().is_err());
15235 }
15236
15237 #[test]
15238 fn reserved_module_hello_requires_matching_launch_nonce() {
15239 let registry = Arc::new(Registry::default());
15240 let supervisor = SupervisorHandle::new();
15241 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
15244 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15245
15246 let no_nonce = handler
15248 .handle_control(
15249 ConnectionId::new(1),
15250 hello_frame("vault", PROTOCOL_VERSION, 1),
15251 )
15252 .unwrap();
15253 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
15254 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
15255 assert!(registry.get_module("vault").unwrap().is_none());
15256
15257 let wrong = handler
15259 .handle_control(
15260 ConnectionId::new(2),
15261 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
15262 )
15263 .unwrap();
15264 assert_eq!(wrong[0].header.ty, FrameType::Error);
15265 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
15266 assert!(registry.get_module("vault").unwrap().is_none());
15267
15268 let ok = handler
15270 .handle_control(
15271 ConnectionId::new(3),
15272 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
15273 )
15274 .unwrap();
15275 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
15276 assert!(registry.get_module("vault").unwrap().is_some());
15277 }
15278
15279 #[test]
15280 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
15281 let registry = Arc::new(Registry::default());
15282 let supervisor = SupervisorHandle::new();
15283 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15284 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15285 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15286
15287 let squat = handler
15288 .handle_control(
15289 ConnectionId::new(1),
15290 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
15291 )
15292 .unwrap();
15293 assert_eq!(squat[0].header.ty, FrameType::Error);
15294 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
15295 assert!(parse_error(&squat[0])["message"]
15296 .as_str()
15297 .unwrap()
15298 .contains("fed:"));
15299
15300 let accepted_peer = handler
15301 .handle_control(
15302 ConnectionId::new(2),
15303 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
15304 )
15305 .unwrap();
15306 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
15307
15308 let accepted_short = handler
15309 .handle_control(
15310 ConnectionId::new(3),
15311 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
15312 )
15313 .unwrap();
15314 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
15315
15316 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
15317 let response = handler
15318 .handle_control(
15319 ConnectionId::new(conn),
15320 hello_frame(module_id, PROTOCOL_VERSION, conn),
15321 )
15322 .unwrap();
15323 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
15324 }
15325 }
15326
15327 #[test]
15328 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
15329 let registry = Arc::new(Registry::default());
15330 let supervisor = SupervisorHandle::new();
15331 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15332 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15333 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
15334 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15335
15336 let owner_nonce = handler
15337 .handle_control(
15338 ConnectionId::new(1),
15339 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
15340 )
15341 .unwrap();
15342 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
15343 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
15344 assert!(registry.get_module("fed:special").unwrap().is_none());
15345
15346 let exact_nonce = handler
15347 .handle_control(
15348 ConnectionId::new(2),
15349 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
15350 )
15351 .unwrap();
15352 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
15353 assert!(registry.get_module("fed:special").unwrap().is_some());
15354 }
15355
15356 #[test]
15357 fn non_reserved_module_ignores_launch_nonce() {
15358 let registry = Arc::new(Registry::default());
15359 let handler = ControlHandler::new(Arc::clone(®istry));
15362 let no_nonce = handler
15363 .handle_control(
15364 ConnectionId::new(1),
15365 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
15366 )
15367 .unwrap();
15368 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
15369 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
15370
15371 let echoed_nonce = handler
15372 .handle_control(
15373 ConnectionId::new(2),
15374 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
15375 )
15376 .unwrap();
15377 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
15378 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
15379 }
15380
15381 #[test]
15382 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
15383 let handler = ControlHandler::default();
15384 let conn = ConnectionId::new(1);
15385 let malformed = Frame::build(
15386 FrameType::Hello,
15387 control_flags(),
15388 0,
15389 0,
15390 3,
15391 b"{not json".to_vec(),
15392 )
15393 .unwrap();
15394
15395 let error = handler.handle_control(conn, malformed).unwrap();
15396 assert_eq!(error[0].header.ty, FrameType::Error);
15397 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
15398
15399 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15400 let pong = handler.handle_control(conn, ping).unwrap();
15401 assert_eq!(pong[0].header.ty, FrameType::Pong);
15402 assert_eq!(pong[0].header.corr, 4);
15403 }
15404
15405 #[test]
15406 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
15407 let registry = Arc::new(Registry::default());
15408 let handler = ControlHandler::new(Arc::clone(®istry));
15409
15410 handler
15411 .handle_control(
15412 ConnectionId::new(1),
15413 hello_frame("aft", PROTOCOL_VERSION, 1),
15414 )
15415 .unwrap();
15416 let duplicate = handler
15417 .handle_control(
15418 ConnectionId::new(2),
15419 hello_frame("aft", PROTOCOL_VERSION, 2),
15420 )
15421 .unwrap();
15422
15423 assert_eq!(duplicate[0].header.ty, FrameType::Error);
15424 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15425 let registration = registry.get_module("aft").unwrap().unwrap();
15426 assert_eq!(registration.connection_id, ConnectionId::new(1));
15427 }
15428
15429 #[test]
15430 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
15431 let registry = Arc::new(Registry::default());
15432 let forwarding = Arc::new(ForwardingTable::default());
15433 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
15434 let handler =
15435 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15436 .with_process_liveness(process_liveness);
15437 let (ctx, route_channel, route_epoch) =
15438 bind_liveness_route(®istry, &forwarding, "aft-dead");
15439 let responses = handler
15440 .handle_route_poll(
15441 &ctx,
15442 route_poll_frame(41, PollKind::Liveness, route_channel),
15443 route_channel,
15444 route_epoch,
15445 PollKind::Liveness,
15446 )
15447 .unwrap();
15448
15449 assert_eq!(responses.len(), 1);
15450 assert_eq!(responses[0].header.ty, FrameType::Response);
15451 assert_route_poll_liveness(&responses[0], false);
15452 }
15453
15454 #[test]
15455 fn liveness_poll_without_process_source_uses_bound_route() {
15456 let registry = Arc::new(Registry::default());
15457 let forwarding = Arc::new(ForwardingTable::default());
15458 let handler =
15459 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15460 let (ctx, route_channel, route_epoch) =
15461 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
15462 let responses = handler
15463 .handle_route_poll(
15464 &ctx,
15465 route_poll_frame(42, PollKind::Liveness, route_channel),
15466 route_channel,
15467 route_epoch,
15468 PollKind::Liveness,
15469 )
15470 .unwrap();
15471
15472 assert_route_poll_liveness(&responses[0], true);
15473 }
15474
15475 #[test]
15476 fn liveness_poll_untracked_process_source_uses_bound_route() {
15477 let registry = Arc::new(Registry::default());
15478 let forwarding = Arc::new(ForwardingTable::default());
15479 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
15480 let handler =
15481 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15482 .with_process_liveness(process_liveness);
15483 let (ctx, route_channel, route_epoch) =
15484 bind_liveness_route(®istry, &forwarding, "aft-untracked");
15485 let responses = handler
15486 .handle_route_poll(
15487 &ctx,
15488 route_poll_frame(43, PollKind::Liveness, route_channel),
15489 route_channel,
15490 route_epoch,
15491 PollKind::Liveness,
15492 )
15493 .unwrap();
15494
15495 assert_route_poll_liveness(&responses[0], true);
15496 }
15497
15498 #[tokio::test]
15499 async fn unknown_op_returns_unknown_control_op() {
15500 let handler = ControlHandler::default();
15501 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
15502 let request = Frame::build(
15503 FrameType::Request,
15504 control_flags(),
15505 0,
15506 0,
15507 55,
15508 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
15509 )
15510 .unwrap();
15511
15512 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15513
15514 assert_eq!(response.len(), 1);
15515 assert_eq!(response[0].header.ty, FrameType::Error);
15516 assert_eq!(response[0].header.corr, 55);
15517 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
15518 }
15519
15520 #[tokio::test]
15521 async fn supervisor_provenance_rejects_unknown_exact_module() {
15522 let handler = ControlHandler::default();
15523 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
15524 let request = Frame::build(
15525 FrameType::Request,
15526 control_flags(),
15527 0,
15528 0,
15529 57,
15530 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
15531 )
15532 .unwrap();
15533
15534 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15535
15536 assert_eq!(response.len(), 1);
15537 assert_eq!(response[0].header.ty, FrameType::Error);
15538 assert_eq!(response[0].header.corr, 57);
15539 let error = parse_error(&response[0]);
15540 assert_eq!(error["code"], "unknown_module");
15541 assert_eq!(error["message"], "module_id 'missing' is not supervised");
15542 }
15543
15544 #[test]
15545 fn provenance_probe_override_keeps_handler_tests_deterministic() {
15546 let expected = subc_control::RunningImageAgreement::Unavailable {
15547 reason: subc_control::RunningImageUnavailableReason::HashFailed,
15548 };
15549 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
15550 assert_eq!(handler.provenance_probe_override, Some(expected));
15551 }
15552
15553 #[test]
15554 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
15555 let verdict = reload_verdict(
15556 std::path::Path::new("/bin/new"),
15557 Some(std::path::Path::new("/bin/old")),
15558 subc_control::RunningImageAgreement::Unavailable {
15559 reason: subc_control::RunningImageUnavailableReason::HashFailed,
15560 },
15561 );
15562 assert!(matches!(
15563 verdict.path,
15564 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
15565 if configured == std::path::Path::new("/bin/new")
15566 && spawned_from == std::path::Path::new("/bin/old")
15567 ));
15568 }
15569
15570 #[test]
15571 fn launch_spec_without_running_digest_is_unavailable_not_match() {
15572 use subc_control::{
15573 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
15574 };
15575 let configured = crate::supervise::ModuleSpec {
15576 module_id: "legacy".into(),
15577 program: "/module".into(),
15578 args: Vec::new(),
15579 env: Vec::new(),
15580 reserved: false,
15581 reserved_prefixes: Vec::new(),
15582 protocol: subc_control::ModuleProtocol::Subc,
15583 overlap: Default::default(),
15584 };
15585 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
15586 assert_eq!(
15587 launch_spec_agreement(&configured, false, Some(&digest)),
15588 Agreement::Unavailable {
15589 reason: Reason::NotRunning
15590 }
15591 );
15592 assert_eq!(
15593 launch_spec_agreement(&configured, true, None),
15594 Agreement::Unavailable {
15595 reason: Reason::SpawnedSpecUnavailable
15596 }
15597 );
15598 assert_eq!(
15599 launch_spec_agreement(&configured, true, Some(&digest)),
15600 Agreement::Match
15601 );
15602 }
15603
15604 #[cfg(windows)]
15605 #[test]
15606 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
15607 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
15608 let configured = dir.join("module.exe");
15609 std::fs::write(&configured, b"executable path fixture").unwrap();
15610 let spawned = std::fs::canonicalize(&configured).unwrap();
15611 assert_ne!(
15612 configured, spawned,
15613 "fixture must exercise different path spellings"
15614 );
15615 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
15616 let lower_case = configured.to_string_lossy().to_lowercase();
15617 for spelling in [
15618 &configured,
15619 Path::new(&forward_slashes),
15620 Path::new(&lower_case),
15621 ] {
15622 let verdict = reload_verdict(
15623 spelling,
15624 Some(&spawned),
15625 subc_control::RunningImageAgreement::Unavailable {
15626 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15627 },
15628 );
15629 assert_eq!(
15630 verdict.path,
15631 ReloadPathAgreement::Match,
15632 "configured {spelling:?}, spawned {spawned:?}"
15633 );
15634 }
15635 }
15636
15637 #[cfg(windows)]
15638 #[test]
15639 fn windows_reload_path_keeps_different_existing_program_pending() {
15640 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
15641 let old = dir.join("old.exe");
15642 let new = dir.join("new.exe");
15643 std::fs::write(&old, b"same executable content").unwrap();
15644 std::fs::copy(&old, &new).unwrap();
15645 let spawned = std::fs::canonicalize(&old).unwrap();
15646 let image = subc_control::RunningImageAgreement::Unavailable {
15647 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15648 };
15649 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
15650 assert!(
15651 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
15652 if configured == new && spawned_from == spawned)
15653 );
15654 let missing = dir.join("missing.exe");
15655 assert!(matches!(
15656 reload_verdict(&missing, Some(&spawned), image).path,
15657 ReloadPathAgreement::Mismatch { .. }
15658 ));
15659 }
15660
15661 #[test]
15662 fn reload_verdict_detects_replaced_image_at_same_path() {
15663 let image = subc_control::RunningImageAgreement::Mismatch {
15664 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
15665 digest: "old".into(),
15666 },
15667 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
15668 digest: "new".into(),
15669 },
15670 };
15671 let verdict = reload_verdict(
15672 std::path::Path::new("/bin/same"),
15673 Some(std::path::Path::new("/bin/same")),
15674 image.clone(),
15675 );
15676 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15677 assert_eq!(verdict.image, image);
15678 }
15679
15680 #[test]
15681 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
15682 let image = subc_control::RunningImageAgreement::Unavailable {
15683 reason: subc_control::RunningImageUnavailableReason::NotRunning,
15684 };
15685 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
15686 assert_eq!(
15687 verdict.path,
15688 subc_control::ReloadPathAgreement::Unavailable {
15689 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
15690 }
15691 );
15692 assert_eq!(verdict.image, image);
15693
15694 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
15695 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15696 };
15697 let verdict = reload_verdict(
15698 std::path::Path::new("/bin/same"),
15699 Some(std::path::Path::new("/bin/same")),
15700 unconfirmed.clone(),
15701 );
15702 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15703 assert_eq!(verdict.image, unconfirmed);
15704 }
15705
15706 #[test]
15707 fn reload_verdict_preserves_each_image_unavailability_reason() {
15708 use subc_control::RunningImageUnavailableReason as Reason;
15709
15710 for reason in [
15711 Reason::NotRunning,
15712 Reason::UnsupportedPlatform,
15713 Reason::RunningExecutableUnreadable,
15714 Reason::SpawnedPathUnreadable,
15715 Reason::HashFailed,
15716 Reason::ProcessIdentityUnconfirmed,
15717 Reason::Unknown("future_probe_reason".to_string()),
15718 ] {
15719 let image = subc_control::RunningImageAgreement::Unavailable {
15720 reason: reason.clone(),
15721 };
15722 let verdict = reload_verdict(
15723 std::path::Path::new("/bin/same"),
15724 Some(std::path::Path::new("/bin/same")),
15725 image.clone(),
15726 );
15727 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15728 assert_eq!(verdict.image, image, "{reason:?}");
15729 }
15730 }
15731
15732 #[tokio::test]
15733 async fn malformed_control_bodies_return_invalid_control_body() {
15734 let handler = ControlHandler::default();
15735 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
15736
15737 for (corr, body) in [
15738 (56, br#"{"route_channel":1}"#.as_slice()),
15739 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
15740 (
15741 58,
15742 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
15743 ),
15744 ] {
15745 let request = Frame::build(
15746 FrameType::Request,
15747 control_flags(),
15748 0,
15749 0,
15750 corr,
15751 body.to_vec(),
15752 )
15753 .unwrap();
15754 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15755
15756 assert_eq!(response.len(), 1);
15757 assert_eq!(response[0].header.ty, FrameType::Error);
15758 assert_eq!(response[0].header.corr, corr);
15759 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
15760 }
15761 }
15762
15763 #[tokio::test]
15764 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
15765 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15766 let registry = Arc::new(Registry::default());
15767 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
15768 let router = Router::with_control_handler(Arc::clone(&control));
15769 let connection = router.begin_connection();
15770 let (ctx, mut rx) = route_ctx(connection.id());
15771
15772 router
15773 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
15774 .await
15775 .unwrap();
15776 let response = rx.recv().await.unwrap();
15777 let ack = parse_ack(&response);
15778 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15779 let channel = 1;
15780
15781 let goodbye =
15782 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
15783 router.route_for_connection(&ctx, goodbye).await.unwrap();
15784 assert!(rx.try_recv().is_err());
15785 assert!(registry.get_module("aft").unwrap().is_none());
15786
15787 router
15788 .route_for_connection(&ctx, channel_request(channel, 13))
15789 .await
15790 .unwrap();
15791 let error_frame = rx.recv().await.unwrap();
15792 assert_eq!(error_frame.header.ty, FrameType::Error);
15793 assert_eq!(error_frame.header.channel, channel);
15794 let captured = crate::router::test_log::captured_logs(&logs);
15795 assert_eq!(
15796 captured
15797 .lines()
15798 .filter(|line| {
15799 line.contains("module_id=aft")
15800 && line.contains("reason=explicit_goodbye")
15801 && line.contains("module registration ended")
15802 })
15803 .count(),
15804 1,
15805 "unexpected GOODBYE registry log: {captured}"
15806 );
15807 }
15808
15809 #[tokio::test]
15810 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
15811 let registry = Arc::new(Registry::default());
15812 let handler = ControlHandler::new(registry).with_capability_config(
15813 [("prov".to_string(), true), ("cons".to_string(), true)],
15814 BTreeMap::new(),
15815 );
15816 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
15817 register_capability_manifest(
15818 &handler,
15819 &provider_ctx,
15820 &mut provider_rx,
15821 capability_manifest("prov", &["thing/v1"], &[]),
15822 1,
15823 )
15824 .await;
15825 let mut consumer = capability_manifest("cons", &[], &[]);
15826 consumer.capabilities.as_mut().unwrap().requires.push(
15827 subc_protocol::manifest::CapabilityRequirement {
15828 capability: "thing/v1".to_string(),
15829 need: subc_protocol::manifest::CapabilityNeed::Required,
15830 },
15831 );
15832 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
15833 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
15834 assert_eq!(
15835 handler.capability_evaluator.verdict("cons", "thing/v1"),
15836 Some(CapabilityVerdict::Provided)
15837 );
15838 let (mut client_rx, _) = open_route_for_capability_test(
15839 &handler,
15840 &provider_ctx,
15841 &mut provider_rx,
15842 703,
15843 3,
15844 "prov",
15845 None,
15846 )
15847 .await;
15848 let goodbye =
15849 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15850 handler
15851 .handle_control_frame(&provider_ctx, goodbye)
15852 .await
15853 .unwrap();
15854 assert_eq!(
15855 handler.capability_evaluator.verdict("cons", "thing/v1"),
15856 Some(CapabilityVerdict::NeverProvided)
15857 );
15858 let closed = client_rx
15859 .try_recv()
15860 .expect("GOODBYE pushes route.closed before route GOODBYE");
15861 assert!(
15862 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
15863 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
15864 );
15865 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
15866 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
15867 }
15868
15869 struct RouteClosedFixture {
15870 handler: ControlHandler,
15871 #[cfg(unix)]
15872 supervisor: Supervisor,
15873 module_ctx: RouteCtx,
15874 #[cfg(unix)]
15875 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15876 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15877 route: Arc<crate::forwarding::RouteBinding>,
15878 }
15879
15880 async fn route_closed_fixture() -> RouteClosedFixture {
15881 let registry = Arc::new(Registry::default());
15882 let handler = ControlHandler::new(Arc::clone(®istry));
15883 #[cfg(unix)]
15884 let supervisor = Supervisor::new(registry, RestartPolicy::default())
15885 .with_forwarding(Arc::clone(&handler.forwarding));
15886 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
15887 register_capability_manifest(
15888 &handler,
15889 &module_ctx,
15890 &mut module_rx,
15891 capability_manifest("prov", &[], &[]),
15892 1,
15893 )
15894 .await;
15895 let (client_rx, bind) = open_route_for_capability_test(
15896 &handler,
15897 &module_ctx,
15898 &mut module_rx,
15899 702,
15900 2,
15901 "prov",
15902 None,
15903 )
15904 .await;
15905 let ModuleControlRequest::RouteBind {
15906 route_channel: channel,
15907 epoch: route_epoch,
15908 ..
15909 } = bind
15910 else {
15911 panic!("expected route.bind");
15912 };
15913 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
15914 handler
15915 .forwarding
15916 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
15917 .unwrap()
15918 else {
15919 panic!("the acknowledged route must be bound");
15920 };
15921 RouteClosedFixture {
15922 handler,
15923 #[cfg(unix)]
15924 supervisor,
15925 module_ctx,
15926 #[cfg(unix)]
15927 module_rx,
15928 client_rx,
15929 route,
15930 }
15931 }
15932
15933 impl RouteClosedFixture {
15934 #[cfg(unix)]
15935 async fn cut(&mut self) {
15936 let notices = async {
15937 let mut draining = self.module_rx.recv().await.unwrap();
15938 assert!(matches!(
15939 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
15940 &draining.body
15941 )
15942 .unwrap(),
15943 subc_protocol::session::ModuleControlCommand::Draining {
15944 reason: RouteCloseReason::Disable,
15945 ..
15946 }
15947 ));
15948 draining.flushed.take().unwrap().send(()).unwrap();
15949 let mut closing = self.client_rx.recv().await.unwrap();
15950 assert!(matches!(
15951 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
15952 ClientControlPush::RouteClosing {
15953 reason: RouteCloseReason::Restart,
15954 ..
15955 }
15956 ));
15957 closing.flushed.take().unwrap().send(()).unwrap();
15958 };
15959 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
15960 result.unwrap();
15961 }
15962
15963 fn disconnect(&mut self) -> ClientControlPush {
15964 self.handler
15965 .cleanup_connection(self.module_ctx.connection_id)
15966 .unwrap();
15967 let closed = self.client_rx.try_recv().unwrap();
15968 assert_eq!(closed.header.ty, FrameType::Push);
15969 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
15970 assert!(matches!(
15971 &push,
15972 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
15973 if channels == &[self.route.client_channel]
15974 ));
15975 assert_eq!(
15976 self.client_rx.try_recv().unwrap().header.ty,
15977 FrameType::Goodbye
15978 );
15979 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
15980 push
15981 }
15982 }
15983
15984 #[cfg(unix)]
15985 #[tokio::test(start_paused = true)]
15986 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
15987 let mut fixture = route_closed_fixture().await;
15988 fixture.cut().await;
15989 assert!(matches!(
15990 fixture.disconnect(),
15991 ClientControlPush::RouteClosed {
15992 reason: RouteCloseReason::Restart,
15993 drained: true,
15994 ..
15995 }
15996 ));
15997 }
15998
15999 #[cfg(unix)]
16000 #[tokio::test(start_paused = true)]
16001 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
16002 let mut fixture = route_closed_fixture().await;
16003 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
16004 let started = tokio::time::Instant::now();
16005 fixture.cut().await;
16006 assert!(started.elapsed() >= Duration::from_secs(2));
16007 assert_eq!(fixture.route.flow.in_flight(), 1);
16008 assert!(matches!(
16009 fixture.disconnect(),
16010 ClientControlPush::RouteClosed {
16011 reason: RouteCloseReason::Restart,
16012 drained: false,
16013 ..
16014 }
16015 ));
16016 }
16017
16018 #[tokio::test]
16019 async fn module_connection_crash_reports_crash_and_not_drained() {
16020 let mut fixture = route_closed_fixture().await;
16021 assert!(matches!(
16025 fixture.disconnect(),
16026 ClientControlPush::RouteClosed {
16027 reason: RouteCloseReason::Crash,
16028 drained: false,
16029 ..
16030 }
16031 ));
16032 }
16033
16034 #[tokio::test]
16035 async fn dropping_router_connection_releases_registration() {
16036 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16037 let registry = Arc::new(Registry::default());
16038 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
16039 let router = Router::with_control_handler(Arc::clone(&control));
16040 let connection = router.begin_connection();
16041 let connection_id = connection.id();
16042 let (ctx, mut rx) = route_ctx(connection_id);
16043
16044 router
16045 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
16046 .await
16047 .unwrap();
16048 let response = rx.recv().await.unwrap();
16049 let ack = parse_ack(&response);
16050 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16051 assert!(registry.get_module("aft").unwrap().is_some());
16052
16053 drop(connection);
16054
16055 assert!(registry.get_module("aft").unwrap().is_none());
16056 assert_eq!(registry.active_registration_count().unwrap(), 0);
16057
16058 control
16059 .cleanup_connection(ConnectionId::new(u64::MAX))
16060 .unwrap();
16061 let captured = crate::router::test_log::captured_logs(&logs);
16062 println!("captured registry lifecycle logs:\n{captured}");
16063 let events: Vec<_> = captured
16064 .lines()
16065 .filter(|line| {
16066 line.contains("module registered module_id=aft ")
16067 || line.contains("module registration ended")
16068 })
16069 .collect();
16070 assert_eq!(
16071 events.len(),
16072 2,
16073 "unexpected registry lifecycle logs: {captured}"
16074 );
16075 assert!(events[0].contains("module registered module_id=aft "));
16076 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
16077 assert!(events[1].contains(&format!(
16078 "module_id=aft connection_id={}",
16079 connection_id.get()
16080 )));
16081 assert!(events[1].contains("reason=connection_closed"));
16082 assert!(events[1].contains("module registration ended"));
16083 assert_eq!(
16084 captured
16085 .lines()
16086 .filter(|line| line.contains("module registered module_id=aft "))
16087 .count(),
16088 1,
16089 "legacy registration admission line must appear once: {captured}"
16090 );
16091 }
16092
16093 #[tokio::test]
16094 async fn hello_registration_keeps_legacy_module_registered_line_once() {
16095 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16096 let registry = Arc::new(Registry::default());
16097 let control = ControlHandler::new(Arc::clone(®istry));
16098 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
16099 hello_via_sink(
16100 &control,
16101 &ctx,
16102 &mut rx,
16103 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
16104 )
16105 .await;
16106
16107 let captured = crate::router::test_log::captured_logs(&logs);
16108 let admissions: Vec<_> = captured
16109 .lines()
16110 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
16111 .collect();
16112 assert_eq!(
16113 admissions.len(),
16114 1,
16115 "legacy admission line must remain exactly once: {captured}"
16116 );
16117 assert!(admissions[0].contains("routable_provider=true"));
16118 assert!(admissions[0].contains("connection_id=777"));
16119 }
16120
16121 fn capability_manifest(
16122 module_id: &str,
16123 provides: &[&str],
16124 must_never_reach: &[&str],
16125 ) -> ModuleManifest {
16126 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
16127 manifest.capabilities = Some(CapabilityDeclarations {
16128 provides: provides
16129 .iter()
16130 .map(|capability| (*capability).to_string())
16131 .collect(),
16132 requires: Vec::new(),
16133 must_never_reach: must_never_reach
16134 .iter()
16135 .map(|capability| (*capability).to_string())
16136 .collect(),
16137 });
16138 manifest
16139 }
16140
16141 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
16142 Frame::build(
16143 FrameType::Hello,
16144 control_flags(),
16145 0,
16146 0,
16147 corr,
16148 serde_json::to_vec(&ModuleHelloBody {
16149 protocol_ver: manifest.protocol_ver,
16150 manifest,
16151 control_ops: None,
16152 launch_nonce: None,
16153 })
16154 .expect("capability test HELLO serializes"),
16155 )
16156 .expect("capability test HELLO frame builds")
16157 }
16158
16159 fn catalog_update_with_capabilities_frame(
16160 corr: u64,
16161 capabilities: CapabilityDeclarations,
16162 ) -> Frame {
16163 Frame::build(
16164 FrameType::Request,
16165 control_flags(),
16166 0,
16167 0,
16168 corr,
16169 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16170 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
16171 capabilities: Some(capabilities),
16172 ready: None,
16173 })
16174 .expect("capability catalog.update serializes"),
16175 )
16176 .expect("capability catalog.update frame builds")
16177 }
16178
16179 async fn register_capability_manifest(
16180 handler: &ControlHandler,
16181 ctx: &RouteCtx,
16182 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16183 manifest: ModuleManifest,
16184 corr: u64,
16185 ) {
16186 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
16187 }
16188
16189 async fn open_route_for_capability_test(
16190 handler: &ControlHandler,
16191 target_ctx: &RouteCtx,
16192 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16193 client_connection_id: u64,
16194 corr: u64,
16195 target_module_id: &str,
16196 consumer_identity: Option<ConsumerIdentity>,
16197 ) -> (
16198 mpsc::Receiver<crate::router::OutboundFrame>,
16199 ModuleControlRequest,
16200 ) {
16201 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
16202 let route_handler = handler.clone();
16203 let target_module_id = target_module_id.to_string();
16204 let route_task = tokio::spawn(async move {
16205 route_handler
16206 .handle_control_frame(
16207 &client_ctx,
16208 route_open_frame_with_admission_facts(
16209 corr,
16210 &target_module_id,
16211 unique_project_root("admission-facts"),
16212 consumer_identity,
16213 None,
16214 ),
16215 )
16216 .await
16217 .expect("capability test route.open succeeds")
16218 });
16219 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
16220 .await
16221 .expect("capability test route.open must reach route.bind")
16222 .expect("target control receiver stays open");
16223 let bind_request: ModuleControlRequest =
16224 serde_json::from_slice(&bind.body).expect("route.bind decodes");
16225 handler
16226 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
16227 .await
16228 .expect("capability test route.bind ACK succeeds");
16229 assert!(route_task.await.expect("route.open task joins").is_empty());
16230 let opened = client_rx
16231 .recv()
16232 .await
16233 .expect("successful route.open publishes a response");
16234 assert!(matches!(
16235 serde_json::from_slice::<ClientControlResponse>(&opened.body),
16236 Ok(ClientControlResponse::RouteOpen { .. })
16237 ));
16238 (client_rx, bind_request)
16239 }
16240
16241 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
16242 assert_eq!(frame.header.ty, FrameType::Push);
16243 assert_eq!(frame.header.channel, 0);
16244 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
16245 .expect("route.closed control push decodes");
16246 let ClientControlPush::RouteClosed { channels, .. } = &push else {
16247 panic!("expected route.closed");
16248 };
16249 assert_eq!(channels.len(), 1, "exactly one violating route closed");
16250 let closed_channel = channels[0];
16251 let channels = channels.clone();
16252 assert_eq!(
16253 push,
16254 ClientControlPush::RouteClosed {
16255 module_id: target_module_id.to_string(),
16256 channels,
16257 reason: RouteCloseReason::CapabilityDenied,
16258 drained: false,
16259 abandoned: 0,
16260 excluded_subscriptions: 0,
16261 terminal: Some(false),
16262 }
16263 );
16264 closed_channel
16265 }
16266
16267 #[tokio::test]
16268 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
16269 let registry = Arc::new(Registry::default());
16270 let forwarding = Arc::new(ForwardingTable::default());
16271 let supervisor = SupervisorHandle::new();
16272 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16273 let handler =
16274 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16275 .with_supervisor(supervisor);
16276 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
16277 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
16278 register_capability_manifest(
16279 &handler,
16280 &target_ctx,
16281 &mut target_rx,
16282 capability_manifest("target", &["credentials-provider/v1"], &[]),
16283 1,
16284 )
16285 .await;
16286 register_capability_manifest(
16287 &handler,
16288 &opener_ctx,
16289 &mut opener_rx,
16290 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16291 2,
16292 )
16293 .await;
16294
16295 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
16296 let replies = handler
16297 .handle_control_frame(
16298 &client_ctx,
16299 route_open_frame_with_admission_facts(
16300 3,
16301 "target",
16302 unique_project_root("admission-facts"),
16303 Some(ConsumerIdentity {
16304 module_id: "opener".to_string(),
16305 launch_nonce: "opener-nonce".to_string(),
16306 }),
16307 None,
16308 ),
16309 )
16310 .await
16311 .expect("denied route.open returns a typed frame");
16312 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16313 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16314 assert!(
16315 target_rx.try_recv().is_err(),
16316 "forbidden route.open must not relay route.bind"
16317 );
16318 }
16319
16320 #[tokio::test]
16321 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
16322 let registry = Arc::new(Registry::default());
16323 let forwarding = Arc::new(ForwardingTable::default());
16324 let supervisor = SupervisorHandle::new();
16325 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16326 let handler =
16327 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16328 .with_supervisor(supervisor);
16329 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
16330 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
16331 register_capability_manifest(
16332 &handler,
16333 &target_ctx,
16334 &mut target_rx,
16335 capability_manifest("target", &["credentials-provider/v1"], &[]),
16336 1,
16337 )
16338 .await;
16339 register_capability_manifest(
16340 &handler,
16341 &old_opener_ctx,
16342 &mut old_opener_rx,
16343 capability_manifest("opener", &[], &[]),
16344 2,
16345 )
16346 .await;
16347 let (mut client_rx, _) = open_route_for_capability_test(
16348 &handler,
16349 &target_ctx,
16350 &mut target_rx,
16351 712,
16352 3,
16353 "target",
16354 Some(ConsumerIdentity {
16355 module_id: "opener".to_string(),
16356 launch_nonce: "opener-nonce".to_string(),
16357 }),
16358 )
16359 .await;
16360 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16361
16362 handler
16363 .cleanup_connection(old_opener_ctx.connection_id)
16364 .expect("old opener registration cleans up");
16365 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
16366 register_capability_manifest(
16367 &handler,
16368 &new_opener_ctx,
16369 &mut new_opener_rx,
16370 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16371 4,
16372 )
16373 .await;
16374
16375 let closed_channel = assert_capability_denied_push(
16376 client_rx
16377 .try_recv()
16378 .expect("HELLO deny addition must emit route.closed")
16379 .frame,
16380 "target",
16381 );
16382 let goodbye = client_rx
16383 .try_recv()
16384 .expect("HELLO deny addition must send the client a route GOODBYE")
16385 .frame;
16386 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16387 assert_eq!(goodbye.header.channel, closed_channel);
16388 assert_ne!(goodbye.header.channel, 0);
16389 assert_ne!(goodbye.header.epoch, 0);
16390 assert!(client_rx.try_recv().is_err());
16391 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16392 assert!(matches!(
16393 target_rx.try_recv(),
16394 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16395 ));
16396 }
16397
16398 #[tokio::test]
16399 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
16400 let registry = Arc::new(Registry::default());
16401 let forwarding = Arc::new(ForwardingTable::default());
16402 let supervisor = SupervisorHandle::new();
16403 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16404 let handler =
16405 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16406 .with_supervisor(supervisor);
16407 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
16408 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
16409 register_capability_manifest(
16410 &handler,
16411 &target_ctx,
16412 &mut target_rx,
16413 capability_manifest("target", &[], &[]),
16414 1,
16415 )
16416 .await;
16417 register_capability_manifest(
16418 &handler,
16419 &opener_ctx,
16420 &mut opener_rx,
16421 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16422 2,
16423 )
16424 .await;
16425 let (mut client_rx, _) = open_route_for_capability_test(
16426 &handler,
16427 &target_ctx,
16428 &mut target_rx,
16429 722,
16430 3,
16431 "target",
16432 Some(ConsumerIdentity {
16433 module_id: "opener".to_string(),
16434 launch_nonce: "opener-nonce".to_string(),
16435 }),
16436 )
16437 .await;
16438 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16439
16440 let replies = handler
16441 .handle_control_frame(
16442 &target_ctx,
16443 catalog_update_with_capabilities_frame(
16444 4,
16445 CapabilityDeclarations {
16446 provides: vec!["credentials-provider/v1".to_string()],
16447 requires: Vec::new(),
16448 must_never_reach: Vec::new(),
16449 },
16450 ),
16451 )
16452 .await
16453 .expect("claim catalog.update succeeds");
16454 assert!(matches!(
16455 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
16456 Ok(ModuleControlResponseToModule::CatalogUpdate {})
16457 ));
16458 let closed_channel = assert_capability_denied_push(
16459 client_rx
16460 .try_recv()
16461 .expect("claim addition must emit route.closed")
16462 .frame,
16463 "target",
16464 );
16465 let goodbye = client_rx
16466 .try_recv()
16467 .expect("claim addition must send the client a route GOODBYE")
16468 .frame;
16469 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16470 assert_eq!(goodbye.header.channel, closed_channel);
16471 assert_ne!(goodbye.header.channel, 0);
16472 assert_ne!(goodbye.header.epoch, 0);
16473 assert!(client_rx.try_recv().is_err());
16474 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16475 assert!(matches!(
16476 target_rx.try_recv(),
16477 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16478 ));
16479 }
16480
16481 #[tokio::test]
16482 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
16483 let registry = Arc::new(Registry::default());
16484 let forwarding = Arc::new(ForwardingTable::default());
16485 let supervisor = SupervisorHandle::new();
16486 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16487 let handler =
16488 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16489 .with_supervisor(supervisor);
16490 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
16491 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
16492 register_capability_manifest(
16493 &handler,
16494 &target_ctx,
16495 &mut target_rx,
16496 capability_manifest("target", &["credentials-provider/v1"], &[]),
16497 1,
16498 )
16499 .await;
16500 register_capability_manifest(
16501 &handler,
16502 &opener_ctx,
16503 &mut opener_rx,
16504 capability_manifest("opener", &[], &[]),
16505 2,
16506 )
16507 .await;
16508 let (mut client_rx, _) = open_route_for_capability_test(
16509 &handler,
16510 &target_ctx,
16511 &mut target_rx,
16512 732,
16513 3,
16514 "target",
16515 Some(ConsumerIdentity {
16516 module_id: "opener".to_string(),
16517 launch_nonce: "opener-nonce".to_string(),
16518 }),
16519 )
16520 .await;
16521 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16522
16523 handler
16524 .handle_control_frame(
16525 &target_ctx,
16526 catalog_update_with_capabilities_frame(
16527 4,
16528 CapabilityDeclarations {
16529 provides: Vec::new(),
16530 requires: Vec::new(),
16531 must_never_reach: Vec::new(),
16532 },
16533 ),
16534 )
16535 .await
16536 .expect("claim removal catalog.update succeeds");
16537 assert_eq!(
16538 forwarding.active_binding_count().unwrap(),
16539 1,
16540 "removing an attested target claim must leave the route census unchanged"
16541 );
16542 assert!(
16543 client_rx.try_recv().is_err(),
16544 "claim removal must not emit route.closed or a client route GOODBYE"
16545 );
16546 assert!(
16547 target_rx.try_recv().is_err(),
16548 "claim removal must not send the target a route GOODBYE"
16549 );
16550 }
16551
16552 #[tokio::test]
16555 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
16556 let registry = Arc::new(Registry::default());
16557 let forwarding = Arc::new(ForwardingTable::default());
16558 let supervisor = SupervisorHandle::new();
16559 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16560 let handler =
16561 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16562 .with_supervisor(supervisor);
16563 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
16564 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
16565 register_capability_manifest(
16566 &handler,
16567 &target_ctx,
16568 &mut target_rx,
16569 capability_manifest("target", &["credentials-provider/v1"], &[]),
16570 1,
16571 )
16572 .await;
16573 register_capability_manifest(
16574 &handler,
16575 &opener_ctx,
16576 &mut opener_rx,
16577 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16578 2,
16579 )
16580 .await;
16581
16582 let (_client_rx, bind) = open_route_for_capability_test(
16583 &handler,
16584 &target_ctx,
16585 &mut target_rx,
16586 742,
16587 3,
16588 "target",
16589 None,
16590 )
16591 .await;
16592 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
16593 panic!("direct scope-honesty route must bind");
16594 };
16595 assert_eq!(principal, Some(Principal::Direct));
16596 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16597 }
16598
16599 #[tokio::test]
16602 async fn must_never_reach_self_route_is_capability_forbidden() {
16603 let registry = Arc::new(Registry::default());
16604 let forwarding = Arc::new(ForwardingTable::default());
16605 let supervisor = SupervisorHandle::new();
16606 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
16607 let handler =
16608 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16609 .with_supervisor(supervisor);
16610 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
16611 register_capability_manifest(
16612 &handler,
16613 &self_ctx,
16614 &mut self_rx,
16615 capability_manifest(
16616 "self-provider",
16617 &["credentials-provider/v1"],
16618 &["credentials-provider/v1"],
16619 ),
16620 1,
16621 )
16622 .await;
16623
16624 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
16625 let replies = handler
16626 .handle_control_frame(
16627 &client_ctx,
16628 route_open_frame_with_admission_facts(
16629 2,
16630 "self-provider",
16631 unique_project_root("admission-facts"),
16632 Some(ConsumerIdentity {
16633 module_id: "self-provider".to_string(),
16634 launch_nonce: "self-nonce".to_string(),
16635 }),
16636 None,
16637 ),
16638 )
16639 .await
16640 .expect("self-route refusal returns a typed frame");
16641 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16642 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16643 assert!(
16644 self_rx.try_recv().is_err(),
16645 "self denial must not relay route.bind"
16646 );
16647 }
16648
16649 #[test]
16650 fn unsupported_channel_zero_frame_returns_error() {
16651 let handler = ControlHandler::default();
16652 let request = Frame::build(
16653 FrameType::Request,
16654 control_flags(),
16655 0,
16656 0,
16657 21,
16658 b"opaque".to_vec(),
16659 )
16660 .unwrap();
16661
16662 let response = handler
16663 .handle_control(ConnectionId::new(1), request)
16664 .unwrap();
16665
16666 assert_eq!(response[0].header.ty, FrameType::Error);
16667 assert_eq!(
16668 parse_error(&response[0])["code"],
16669 "unsupported_control_frame"
16670 );
16671 }
16672
16673 mod swap {
16678 use super::*;
16679
16680 const INCUMBENT: ConnectionId = ConnectionId::new(30);
16681 const CANDIDATE: ConnectionId = ConnectionId::new(40);
16682
16683 struct Swap {
16684 registry: Arc<Registry>,
16685 forwarding: Arc<ForwardingTable>,
16686 handler: ControlHandler,
16687 incumbent_ctx: RouteCtx,
16688 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16689 candidate_ctx: RouteCtx,
16690 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16691 }
16692
16693 async fn swap_with_incumbent() -> Swap {
16694 let registry = Arc::new(Registry::default());
16695 let forwarding = Arc::new(ForwardingTable::default());
16696 let handler =
16697 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
16698 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
16699 hello_via_sink(
16700 &handler,
16701 &incumbent_ctx,
16702 &mut incumbent_rx,
16703 hello_frame("aft", PROTOCOL_VERSION, 7),
16704 )
16705 .await;
16706 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
16707 Swap {
16708 registry,
16709 forwarding,
16710 handler,
16711 incumbent_ctx,
16712 incumbent_rx,
16713 candidate_ctx,
16714 candidate_rx,
16715 }
16716 }
16717
16718 fn register_candidate(swap: &Swap, ready: Option<bool>) {
16719 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
16720 candidate_manifest.ready = ready;
16721 let registration = swap
16722 .registry
16723 .register_candidate_with_control_ops(
16724 candidate_manifest,
16725 PROTOCOL_VERSION,
16726 CANDIDATE,
16727 module_baseline_control_ops(),
16728 )
16729 .unwrap();
16730 swap.forwarding
16731 .register_candidate_module_connection(
16732 CANDIDATE,
16733 "aft".to_string(),
16734 PROTOCOL_VERSION,
16735 manifest_concurrency(®istration.manifest),
16736 swap.candidate_ctx.egress.clone(),
16737 )
16738 .unwrap();
16739 }
16740
16741 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
16742 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
16743 swap.registry.promote_candidate("aft").unwrap().unwrap();
16744 cutover.incumbent.unwrap()
16745 }
16746
16747 #[tokio::test]
16756 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
16757 let mut swap = swap_with_incumbent().await;
16758 let hold = Arc::new(std::sync::Barrier::new(2));
16759 let handler = swap
16760 .handler
16761 .clone()
16762 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
16763 handler
16764 .supervisor
16765 .open_swap("aft", "candidate-nonce".to_string());
16766 let hello = {
16767 let handler = handler.clone();
16768 let candidate_ctx = swap.candidate_ctx.clone();
16769 std::thread::spawn(move || {
16770 handler
16771 .handle_hello(
16772 candidate_ctx.connection_id,
16773 Some(candidate_ctx.egress.clone()),
16774 hello_frame_with_nonce(
16775 "aft",
16776 PROTOCOL_VERSION,
16777 8,
16778 Some("candidate-nonce"),
16779 ),
16780 )
16781 .unwrap()
16782 })
16783 };
16784
16785 hold.wait();
16786 assert!(swap
16789 .registry
16790 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
16791 .unwrap()
16792 .is_some());
16793 assert!(swap
16794 .registry
16795 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16796 .unwrap()
16797 .is_none());
16798 assert!(swap
16799 .forwarding
16800 .module_endpoint_for_connection(CANDIDATE)
16801 .unwrap()
16802 .is_none());
16803
16804 hold.wait();
16805 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
16806 assert_eq!(
16807 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
16808 FrameType::HelloAck
16809 );
16810 let published = swap
16811 .registry
16812 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16813 .unwrap()
16814 .expect("candidate is published once its HELLO completes");
16815 assert_eq!(published.connection_id, CANDIDATE);
16816 assert!(swap
16817 .forwarding
16818 .module_endpoint_for_connection(CANDIDATE)
16819 .unwrap()
16820 .is_some());
16821 }
16822
16823 fn keyed_total(counters: &Value, key: &str) -> u64 {
16824 counters[key]
16825 .as_object()
16826 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
16827 .unwrap_or(0)
16828 }
16829
16830 #[tokio::test]
16831 async fn replacement_logs_old_end_and_new_admission_once() {
16832 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16833 let mut swap = swap_with_incumbent().await;
16834 swap.handler
16835 .supervisor
16836 .open_swap("aft", "candidate-nonce".to_string());
16837 hello_via_sink(
16838 &swap.handler,
16839 &swap.candidate_ctx,
16840 &mut swap.candidate_rx,
16841 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
16842 )
16843 .await;
16844 cutover(&swap);
16845 swap.handler.cleanup_connection(INCUMBENT).unwrap();
16846
16847 let captured = crate::router::test_log::captured_logs(&logs);
16848 println!("captured replacement registry logs:\n{captured}");
16849 let new_admissions: Vec<_> = captured
16850 .lines()
16851 .filter(|line| {
16852 line.contains("connection_id=40")
16853 && line.contains("swap candidate registered; not routable until cutover")
16854 })
16855 .collect();
16856 assert_eq!(
16857 new_admissions.len(),
16858 1,
16859 "unexpected admission logs: {captured}"
16860 );
16861 assert!(new_admissions[0].contains("module_id=aft"));
16862 assert!(new_admissions[0].contains("connection_id=40"));
16863 assert!(
16864 new_admissions[0].contains("swap candidate registered; not routable until cutover")
16865 );
16866 assert!(new_admissions[0].contains("ready=true"));
16867
16868 let old_admissions: Vec<_> = captured
16869 .lines()
16870 .filter(|line| {
16871 line.contains("module registered module_id=aft ")
16872 && line.contains("connection_id=30")
16873 })
16874 .collect();
16875 assert_eq!(
16876 old_admissions.len(),
16877 1,
16878 "unexpected incumbent admission: {captured}"
16879 );
16880
16881 let old_ends: Vec<_> = captured
16882 .lines()
16883 .filter(|line| {
16884 line.contains("connection_id=30") && line.contains("module registration ended")
16885 })
16886 .collect();
16887 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
16888 assert!(old_ends[0].contains("module_id=aft"));
16889 assert!(old_ends[0].contains("connection_id=30"));
16890 assert!(old_ends[0].contains("reason=replaced"));
16891 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
16892 assert_eq!(
16893 captured
16894 .lines()
16895 .filter(|line| line.contains("module registration promoted"))
16896 .count(),
16897 1,
16898 "unexpected promotion logs: {captured}"
16899 );
16900 }
16901
16902 #[tokio::test]
16908 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
16909 let mut swap = swap_with_incumbent().await;
16910 let handler = swap.handler.clone();
16911
16912 let cotenant = ConnectionId::new(31);
16914 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
16915 let (cotenant_task, cotenant_bind) = relay_route_open(
16916 &handler,
16917 cotenant,
16918 &cotenant_ctx.egress,
16919 &mut swap.incumbent_rx,
16920 100,
16921 "aft",
16922 "swap-cotenant",
16923 )
16924 .await;
16925 handler
16926 .handle_control_frame(
16927 &swap.incumbent_ctx,
16928 route_bind_ack(cotenant_bind.header.corr),
16929 )
16930 .await
16931 .unwrap();
16932 assert!(cotenant_task.await.unwrap().is_empty());
16933 let (cotenant_channel, cotenant_epoch) =
16934 published_route(&cotenant_rx.recv().await.unwrap());
16935
16936 let caller = ConnectionId::new(32);
16938 let (caller_ctx, mut caller_rx) = route_ctx(caller);
16939 let (caller_task, caller_bind) = relay_route_open(
16940 &handler,
16941 caller,
16942 &caller_ctx.egress,
16943 &mut swap.incumbent_rx,
16944 101,
16945 "aft",
16946 "swap-caller",
16947 )
16948 .await;
16949 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
16950
16951 register_candidate(&swap, None);
16952 cutover(&swap);
16953
16954 let ack = handler
16956 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
16957 .await;
16958 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
16959 if module_loop_error.is_some() {
16960 handler.cleanup_connection(INCUMBENT).unwrap();
16963 }
16964
16965 assert!(
16967 cotenant_rx.try_recv().is_err(),
16968 "the co-tenant route on the incumbent was torn down by one late ack: \
16969 {module_loop_error:?}"
16970 );
16971 assert!(matches!(
16972 swap.forwarding
16973 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
16974 .unwrap(),
16975 DataRoute::Client(DataRouteState::Bound(_))
16976 ));
16977 assert_eq!(module_loop_error, None);
16978 assert!(swap
16979 .registry
16980 .get_module_by_connection(INCUMBENT)
16981 .unwrap()
16982 .is_some());
16983
16984 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
16986 .await
16987 .expect("the incumbent is told to drop the abandoned binding")
16988 .unwrap()
16989 .frame;
16990 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16991 assert_eq!(goodbye.header.channel, abandoned_channel);
16992 assert_eq!(goodbye.header.epoch, abandoned_epoch);
16993 assert!(swap.incumbent_rx.try_recv().is_err());
16994
16995 let response = caller_task.await.unwrap();
16997 assert_eq!(response.len(), 1);
16998 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
16999 assert!(caller_rx.try_recv().is_err());
17000
17001 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
17005 let counters = handler.counters().snapshot();
17006 assert_eq!(
17007 keyed_total(&counters, "route_open_accepted_by_principal"),
17008 1
17009 );
17010 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
17011 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
17012 }
17013
17014 #[tokio::test]
17018 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
17019 let mut swap = swap_with_incumbent().await;
17020 register_candidate(&swap, None);
17021 let incumbent = cutover(&swap);
17022 swap.forwarding
17023 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
17024 .unwrap()
17025 .expect("the incumbent is still registered");
17026
17027 let client = ConnectionId::new(33);
17028 let (client_ctx, mut client_rx) = route_ctx(client);
17029 let route_handler = swap.handler.clone();
17030 let open_ctx = RouteCtx {
17031 connection_id: client,
17032 egress: client_ctx.egress.clone(),
17033 };
17034 let mut route_task = tokio::spawn(async move {
17035 route_handler
17036 .handle_control_frame(
17037 &open_ctx,
17038 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
17039 )
17040 .await
17041 .unwrap()
17042 });
17043 let bind = tokio::select! {
17044 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
17045 response = &mut route_task => {
17046 let response = response.unwrap();
17047 panic!(
17048 "post-cutover route.open was refused instead of relayed to the candidate: {}",
17049 parse_error(&response[0])["code"]
17050 );
17051 }
17052 };
17053 swap.handler
17054 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
17055 .await
17056 .unwrap();
17057 assert!(route_task.await.unwrap().is_empty());
17058 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
17059 match swap
17060 .forwarding
17061 .lookup_data_route(client, channel, epoch)
17062 .unwrap()
17063 {
17064 DataRoute::Client(DataRouteState::Bound(route)) => {
17065 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
17066 }
17067 other => panic!("expected a bound route on the candidate, got {other:?}"),
17068 }
17069 assert!(swap.incumbent_rx.try_recv().is_err());
17070 }
17071
17072 #[tokio::test]
17077 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
17078 let swap = swap_with_incumbent().await;
17079 register_candidate(&swap, Some(false));
17080 let update = Frame::build(
17081 FrameType::Request,
17082 control_flags(),
17083 0,
17084 0,
17085 55,
17086 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
17087 provides: manifest("aft", PROTOCOL_VERSION).provides,
17088 capabilities: None,
17089 ready: Some(true),
17090 })
17091 .unwrap(),
17092 )
17093 .unwrap();
17094
17095 let replies = swap
17096 .handler
17097 .handle_control_frame(&swap.candidate_ctx, update)
17098 .await
17099 .unwrap();
17100
17101 assert_eq!(replies.len(), 1);
17102 assert_eq!(
17103 replies[0].header.ty,
17104 FrameType::Response,
17105 "candidate catalog.update was refused: {:?}",
17106 serde_json::from_slice::<Value>(&replies[0].body).ok()
17107 );
17108 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
17109 assert_eq!(
17110 swap.registry
17111 .get_module("aft")
17112 .unwrap()
17113 .unwrap()
17114 .connection_id,
17115 INCUMBENT
17116 );
17117 }
17118 }
17119
17120 mod swap_admission {
17124 use super::*;
17125
17126 const INCUMBENT_NONCE: &str = "incumbent-nonce";
17127 const CANDIDATE_NONCE: &str = "candidate-nonce";
17128
17129 fn handler_with_incumbent(
17130 module_id: &str,
17131 reserved: bool,
17132 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
17133 let registry = Arc::new(Registry::default());
17134 let supervisor = SupervisorHandle::new();
17135 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
17136 if reserved {
17137 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
17138 }
17139 let handler =
17140 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
17141 let incumbent = handler
17142 .handle_control(
17143 ConnectionId::new(1),
17144 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
17145 )
17146 .unwrap();
17147 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
17148 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
17149 (registry, supervisor, handler)
17150 }
17151
17152 #[test]
17158 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
17159 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
17160
17161 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
17162 let replies = handler
17163 .handle_control(
17164 ConnectionId::new(connection),
17165 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
17166 )
17167 .unwrap();
17168 assert_eq!(replies[0].header.ty, FrameType::Error);
17169 assert_eq!(
17170 parse_error(&replies[0])["code"],
17171 "swap_token_invalid",
17172 "nonce {nonce:?}"
17173 );
17174 }
17175 assert!(registry.get_candidate("aft").unwrap().is_none());
17176 assert_eq!(
17177 registry.get_module("aft").unwrap().unwrap().connection_id,
17178 ConnectionId::new(1)
17179 );
17180
17181 let admitted = handler
17184 .handle_control(
17185 ConnectionId::new(4),
17186 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
17187 )
17188 .unwrap();
17189 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
17190 assert_eq!(
17191 registry
17192 .get_candidate("aft")
17193 .unwrap()
17194 .unwrap()
17195 .connection_id,
17196 ConnectionId::new(4)
17197 );
17198 assert_eq!(
17199 registry.get_module("aft").unwrap().unwrap().connection_id,
17200 ConnectionId::new(1),
17201 "the candidate must not take the active slot"
17202 );
17203 let replayed = handler
17204 .handle_control(
17205 ConnectionId::new(5),
17206 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
17207 )
17208 .unwrap();
17209 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
17210
17211 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
17215 let squatter = handler
17216 .handle_control(
17217 ConnectionId::new(6),
17218 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
17219 )
17220 .unwrap();
17221 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
17222 assert!(
17223 registry.get_module("aft").unwrap().is_none(),
17224 "a squatter took the active slot of an id being swapped"
17225 );
17226 }
17227
17228 #[test]
17233 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
17234 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
17235
17236 let replies = handler
17237 .handle_control(
17238 ConnectionId::new(2),
17239 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17240 )
17241 .unwrap();
17242
17243 assert_eq!(
17244 replies[0].header.ty,
17245 FrameType::HelloAck,
17246 "reserved candidate refused: {:?}",
17247 serde_json::from_slice::<Value>(&replies[0].body).ok()
17248 );
17249 assert_eq!(
17250 registry
17251 .get_candidate("vault")
17252 .unwrap()
17253 .unwrap()
17254 .connection_id,
17255 ConnectionId::new(2)
17256 );
17257 }
17258
17259 #[test]
17262 fn without_an_open_swap_the_ordinary_gates_decide() {
17263 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
17264 supervisor.close_swap("vault");
17265
17266 let candidate = handler
17267 .handle_control(
17268 ConnectionId::new(2),
17269 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17270 )
17271 .unwrap();
17272 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
17273 let duplicate = handler
17274 .handle_control(
17275 ConnectionId::new(3),
17276 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
17277 )
17278 .unwrap();
17279 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
17280 assert!(registry.get_candidate("vault").unwrap().is_none());
17281 }
17282 }
17283
17284 mod scopes {
17288 mod idle_tests {
17289 use super::*;
17290 include!("scope_idle_tests.rs");
17291 }
17292 mod route_open_timings {
17293 use super::*;
17294 include!("route_open_timing_tests.rs");
17295 }
17296 mod route_open_refusal_log {
17297 use super::*;
17298 include!("route_open_refusal_log_tests.rs");
17299 }
17300 mod activity {
17301 include!("activity_scope_tests.rs");
17302 }
17303 mod delta {
17304 include!("scope_handler_delta_tests.rs");
17305 }
17306 mod call_modes {
17307 include!("scope_call_mode_tests.rs");
17308 }
17309 #[tokio::test]
17310 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
17311 let rig = rig().await;
17312 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
17313 "s".into(),
17314 1,
17315 "tool".into(),
17316 ))
17317 .unwrap();
17318 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
17319 let (direct, _rx) = wide_ctx(999);
17320 for (ctx, expected) in [
17321 (&rig.owner, FrameType::Response),
17322 (&direct, FrameType::Error),
17323 ] {
17324 let frame =
17325 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
17326 .unwrap();
17327 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
17328 assert_eq!(replies[0].header.ty, expected);
17329 if expected == FrameType::Response {
17330 let reply: subc_control::ScopeServedByReply =
17331 serde_json::from_slice(&replies[0].body).unwrap();
17332 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
17333 } else {
17334 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
17335 }
17336 }
17337 let reply = call(
17338 &rig.handler,
17339 &rig.owner,
17340 &ModuleControlRequestFromModule::ScopeDescribe {
17341 owner: Principal::Reserved {
17342 module_id: OWNER.into(),
17343 },
17344 scope_ref: "s".into(),
17345 },
17346 )
17347 .await;
17348 assert!(matches!(
17349 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
17350 ModuleControlResponseToModule::ScopeDescribe { .. }
17351 ));
17352 }
17353
17354 use subc_protocol::scope::{
17355 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
17356 ScopeStatus,
17357 };
17358
17359 use super::*;
17360
17361 const OWNER: &str = "prefrontal-core";
17362
17363 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
17364 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
17365 }
17366
17367 async fn call(
17368 handler: &ControlHandler,
17369 ctx: &RouteCtx,
17370 request: &ModuleControlRequestFromModule,
17371 ) -> Frame {
17372 let body = serde_json::to_vec(request).unwrap();
17373 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
17374 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
17375 assert_eq!(replies.len(), 1, "{replies:?}");
17376 replies.pop().unwrap()
17377 }
17378
17379 async fn sync(
17380 handler: &ControlHandler,
17381 ctx: &RouteCtx,
17382 generation: u64,
17383 scopes: Vec<ScopeRecord>,
17384 ) -> Result<ModuleControlResponseToModule, String> {
17385 let reply = call(
17386 handler,
17387 ctx,
17388 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
17389 )
17390 .await;
17391 match reply.header.ty {
17392 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
17393 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
17394 }
17395 }
17396
17397 async fn describe(
17398 handler: &ControlHandler,
17399 ctx: &RouteCtx,
17400 owner: &str,
17401 scope_ref: &str,
17402 ) -> ModuleControlResponseToModule {
17403 let reply = call(
17404 handler,
17405 ctx,
17406 &ModuleControlRequestFromModule::ScopeDescribe {
17407 owner: Principal::Reserved {
17408 module_id: owner.to_string(),
17409 },
17410 scope_ref: scope_ref.to_string(),
17411 },
17412 )
17413 .await;
17414 assert_eq!(
17415 reply.header.ty,
17416 FrameType::Response,
17417 "{:?}",
17418 parse_error(&reply)
17419 );
17420 serde_json::from_slice(&reply.body).unwrap()
17421 }
17422
17423 #[tokio::test]
17424 async fn scope_describe_never_echoes_bind_proof_or_hash() {
17425 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17426 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17427 let mut rig = rig().await;
17428 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17429 .await;
17430 let _bound = rig
17431 .bound(
17432 Some(AFT),
17433 PLEXUS,
17434 Some(
17435 rig.selector("s", Some(1))
17436 .with_bind_proof(Some(PROOF.into())),
17437 ),
17438 )
17439 .await;
17440 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
17441 let output = serde_json::to_string(&reply).unwrap();
17442 assert!(output.contains("\"status\":\"live\""));
17443 assert!(!output.contains(PROOF));
17444 assert!(!output.contains(HASH));
17445 }
17446
17447 async fn module(
17449 handler: &ControlHandler,
17450 connection: u64,
17451 module_id: &str,
17452 nonce: Option<&str>,
17453 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17454 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
17455 hello_via_sink(
17456 handler,
17457 &ctx,
17458 &mut rx,
17459 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
17460 )
17461 .await;
17462 (ctx, rx)
17463 }
17464
17465 #[tokio::test]
17468 async fn a_client_connection_cannot_sync_or_describe() {
17469 let handler = ControlHandler::new(Arc::new(Registry::default()));
17470 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
17471 for request in [
17472 ModuleControlRequestFromModule::ScopeSync {
17473 generation: 1,
17474 scopes: vec![head("s", 1)],
17475 },
17476 ModuleControlRequestFromModule::ScopeDescribe {
17477 owner: Principal::Direct,
17478 scope_ref: "s".to_string(),
17479 },
17480 ] {
17481 let reply = call(&handler, &ctx, &request).await;
17482 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
17483 }
17484 assert!(
17485 !handler
17486 .scopes
17487 .read()
17488 .unwrap()
17489 .describe(
17490 &Principal::Reserved {
17491 module_id: OWNER.to_string()
17492 },
17493 "s"
17494 )
17495 .owner_synced
17496 );
17497 }
17498
17499 #[tokio::test]
17502 async fn a_module_without_a_supervised_launch_cannot_sync() {
17503 let handler = ControlHandler::new(Arc::new(Registry::default()));
17504 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
17505 assert_eq!(
17506 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
17507 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17508 );
17509 }
17510
17511 #[tokio::test]
17512 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
17513 let supervisor = SupervisorHandle::new();
17514 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17515 let handler = ControlHandler::new(Arc::new(Registry::default()))
17516 .with_supervisor(supervisor.clone());
17517 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17518 sync(&handler, &incumbent, 1, vec![head("s", 1)])
17519 .await
17520 .expect("the current launch syncs");
17521
17522 supervisor.open_swap(OWNER, "n2".to_string());
17525 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
17526 assert_eq!(
17527 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17528 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17529 );
17530 sync(&handler, &incumbent, 2, vec![head("s", 1)])
17531 .await
17532 .expect("the serving owner syncs during the swap");
17533
17534 supervisor.close_swap(OWNER);
17537 assert_eq!(
17538 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17539 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17540 );
17541 sync(&handler, &incumbent, 3, vec![head("s", 1)])
17542 .await
17543 .expect("the serving owner syncs after the rollback");
17544 handler.cleanup_connection(candidate.connection_id).unwrap();
17545
17546 supervisor.open_swap(OWNER, "n3".to_string());
17551 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
17552 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
17553 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
17554 .await
17555 .expect("the promoted launch takes authority");
17556 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17557 panic!("unexpected reply {reply:?}");
17558 };
17559 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
17560 assert_eq!(
17561 sync(&handler, &incumbent, 4, Vec::new()).await,
17562 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17563 );
17564 }
17565
17566 #[tokio::test]
17569 async fn closing_the_authority_connection_frees_sync_authority() {
17570 let supervisor = SupervisorHandle::new();
17571 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17572 let handler = ControlHandler::new(Arc::new(Registry::default()))
17573 .with_supervisor(supervisor.clone());
17574 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17575 sync(&handler, &first, 10, vec![head("s", 1)])
17576 .await
17577 .unwrap();
17578 handler.cleanup_connection(first.connection_id).unwrap();
17579
17580 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
17581 sync(&handler, &second, 1, vec![head("s", 1)])
17582 .await
17583 .expect("the next connection takes the released authority");
17584 }
17585
17586 #[tokio::test]
17587 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
17588 let supervisor = SupervisorHandle::new();
17589 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17590 let handler =
17591 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
17592 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
17593 sync(&handler, &first, 10, vec![head("s", 1)])
17594 .await
17595 .unwrap();
17596 handler
17597 .handle_control_frame(
17598 &first,
17599 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
17600 )
17601 .await
17602 .unwrap();
17603 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
17604 sync(&handler, &second, 1, vec![head("s", 1)])
17605 .await
17606 .expect("GOODBYE releases authority even if the old socket remains open");
17607 }
17608
17609 #[tokio::test]
17610 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
17611 let registry = Arc::new(Registry::default());
17612 let supervisor_handle = SupervisorHandle::new();
17613 let supervisor =
17614 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
17615 .with_handle(supervisor_handle.clone())
17616 .with_daemon_incarnation("incarnation-7".to_string());
17617 supervisor
17620 .supervise_configured(
17621 ModuleSpec {
17622 module_id: OWNER.to_string(),
17623 program: PathBuf::from("/nonexistent/prefrontal-core"),
17624 args: Vec::new(),
17625 env: Vec::new(),
17626 reserved: false,
17627 reserved_prefixes: Vec::new(),
17628 protocol: ModuleProtocol::Subc,
17629 overlap: Default::default(),
17630 },
17631 false,
17632 )
17633 .unwrap();
17634 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
17635 let handler =
17636 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17637 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
17638
17639 let ModuleControlResponseToModule::ScopeDescribe {
17641 status,
17642 daemon_incarnation,
17643 owner_synced,
17644 owner_configured,
17645 scope,
17646 ..
17647 } = describe(&handler, &reader, OWNER, "s").await
17648 else {
17649 panic!("not a describe reply");
17650 };
17651 assert_eq!(status, ScopeStatus::NotLive);
17652 assert_eq!(daemon_incarnation, "incarnation-7");
17653 assert!(!owner_synced);
17654 assert!(owner_configured);
17655 assert!(scope.is_none());
17656
17657 let ModuleControlResponseToModule::ScopeDescribe {
17660 status,
17661 owner_configured,
17662 ..
17663 } = describe(&handler, &reader, "ghost", "s").await
17664 else {
17665 panic!("not a describe reply");
17666 };
17667 assert_eq!(status, ScopeStatus::NotLive);
17668 assert!(!owner_configured);
17669
17670 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
17672 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
17673 let ModuleControlResponseToModule::ScopeDescribe {
17674 status,
17675 scope_epoch,
17676 owner_synced,
17677 scope,
17678 ..
17679 } = describe(&handler, &reader, OWNER, "s").await
17680 else {
17681 panic!("not a describe reply");
17682 };
17683 assert_eq!(status, ScopeStatus::Live);
17684 assert_eq!(scope_epoch, Some(4));
17685 assert!(owner_synced);
17686 let stamp = scope.expect("a live scope carries its stamp");
17687 assert!(
17688 stamp.owner_authorized,
17689 "prefrontal-core is the default authority"
17690 );
17691 assert_eq!(stamp.kind, ScopeKind::Head);
17692 }
17693
17694 #[tokio::test]
17695 async fn scope_authority_owners_decides_owner_authorized() {
17696 let supervisor = SupervisorHandle::new();
17697 supervisor.set_spawn_nonce("broca", "b1".to_string());
17698 let handler = ControlHandler::new(Arc::new(Registry::default()))
17699 .with_supervisor(supervisor)
17700 .with_scope_authority_owners(vec!["broca".to_string()]);
17701 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
17702 let mut gated = head("s", 1);
17703 gated.attributes.agent_id = Some("agent".to_string());
17704 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
17705 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
17706 describe(&handler, &broca, "broca", "s").await
17707 else {
17708 panic!("not a describe reply");
17709 };
17710 assert!(scope.unwrap().owner_authorized);
17711 }
17712
17713 #[tokio::test]
17717 async fn scope_ops_and_the_scopes_capability_are_advertised() {
17718 let handler = ControlHandler::new(Arc::new(Registry::default()));
17719 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
17720 let ack = hello_via_sink(
17721 &handler,
17722 &ctx,
17723 &mut rx,
17724 hello_frame("m", PROTOCOL_VERSION, 1),
17725 )
17726 .await;
17727 let ack = parse_ack(&ack);
17728 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
17729 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
17730 }
17731 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
17732
17733 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
17734 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
17735 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
17736 let reply = handler
17737 .handle_control_frame(&client, frame)
17738 .await
17739 .unwrap()
17740 .pop()
17741 .unwrap();
17742 let ClientControlResponse::ServerDescribe { capabilities, .. } =
17743 serde_json::from_slice(&reply.body).unwrap()
17744 else {
17745 panic!("not a server.describe reply");
17746 };
17747 assert!(
17748 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
17749 "{capabilities:?}"
17750 );
17751 }
17752
17753 const PLEXUS: &str = "plexus";
17756 const OTHER: &str = "other";
17757 const AFT: &str = "aft";
17758 const BROCA: &str = "broca";
17759 const MAGIC: &str = "magic-context";
17760
17761 fn nonce(module_id: &str) -> String {
17762 format!("nonce-{module_id}")
17763 }
17764
17765 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17766 let (tx, rx) = mpsc::channel(64);
17767 (
17768 RouteCtx {
17769 connection_id: ConnectionId::new(connection),
17770 egress: FrameSink::new(tx),
17771 },
17772 rx,
17773 )
17774 }
17775
17776 struct Rig {
17780 handler: ControlHandler,
17781 forwarding: Arc<ForwardingTable>,
17782 owner: RouteCtx,
17783 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17784 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
17785 generation: u64,
17786 next_connection: u64,
17787 _supervisor: Supervisor,
17788 }
17789
17790 async fn rig() -> Rig {
17791 rig_with_flow_support(true).await
17792 }
17793
17794 async fn rig_with_flow_support(flow_support: bool) -> Rig {
17795 rig_with_scope_capabilities(flow_support, false).await
17796 }
17797
17798 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
17799 let mut provides = Vec::new();
17800 if flow_support {
17801 provides.push("flow-scopes/v1");
17802 }
17803 if run_support {
17804 provides.push("agent-run-scopes/v1");
17805 }
17806 rig_with_target_capabilities(&provides).await
17807 }
17808
17809 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
17810 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
17811 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17812 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17815 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
17816 Frame::build(
17817 FrameType::Hello,
17818 control_flags(),
17819 0,
17820 0,
17821 connection,
17822 serde_json::to_vec(&body).unwrap(),
17823 )
17824 .unwrap()
17825 }
17826
17827 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
17828 let registry = Arc::new(Registry::default());
17829 let forwarding = Arc::new(ForwardingTable::default());
17830 let supervisor_handle = SupervisorHandle::new();
17831 let supervisor =
17832 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
17833 .with_handle(supervisor_handle.clone());
17834 supervisor
17835 .supervise_configured(
17836 ModuleSpec {
17837 module_id: OWNER.to_string(),
17838 program: PathBuf::from("/nonexistent/prefrontal-core"),
17839 args: Vec::new(),
17840 env: Vec::new(),
17841 reserved: false,
17842 reserved_prefixes: Vec::new(),
17843 protocol: ModuleProtocol::Subc,
17844 overlap: Default::default(),
17845 },
17846 false,
17847 )
17848 .unwrap();
17849 for module_id in [OWNER, AFT, BROCA, MAGIC] {
17850 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
17851 }
17852 let handler =
17853 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17854 .with_supervisor(supervisor_handle);
17855 let (owner, mut owner_rx) = wide_ctx(1);
17856 hello_via_sink(
17857 &handler,
17858 &owner,
17859 &mut owner_rx,
17860 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
17861 )
17862 .await;
17863 let mut modules = BTreeMap::new();
17864 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
17865 let (ctx, mut rx) = wide_ctx(connection);
17866 let hello = hello_with_capabilities(module_id, connection, provides);
17867 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
17868 modules.insert(module_id.to_string(), (ctx, rx));
17869 }
17870 Rig {
17871 handler,
17872 forwarding,
17873 owner,
17874 _owner_rx: owner_rx,
17875 modules,
17876 generation: 0,
17877 next_connection: 100,
17878 _supervisor: supervisor,
17879 }
17880 }
17881
17882 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
17883 ScopeCarrier::new(Principal::Reserved {
17884 module_id: module_id.to_string(),
17885 })
17886 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
17887 }
17888
17889 fn session(scope_epoch: u64) -> ScopeRecord {
17892 let mut record = head("s", scope_epoch);
17893 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
17894 record.attributes.agent_id = Some("agent-1".to_string());
17895 record.attributes.delegates = true;
17896 record
17897 }
17898
17899 impl Rig {
17900 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
17901 self.generation += 1;
17902 sync(&self.handler, &self.owner, self.generation, scopes)
17903 .await
17904 .expect("the owner's sync is accepted");
17905 }
17906
17907 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17908 ScopeSelector::new(
17909 Principal::Reserved {
17910 module_id: OWNER.to_string(),
17911 },
17912 scope_ref.to_string(),
17913 scope_epoch,
17914 )
17915 }
17916
17917 fn open_frame(
17918 &mut self,
17919 opener: Option<&str>,
17920 target: &str,
17921 scope: Option<ScopeSelector>,
17922 ) -> (
17923 RouteCtx,
17924 mpsc::Receiver<crate::router::OutboundFrame>,
17925 Frame,
17926 ) {
17927 self.next_connection += 1;
17928 let (ctx, rx) = wide_ctx(self.next_connection);
17929 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
17930 busy_refusals: None,
17931 target: RouteTarget::ToolProvider {
17932 module_id: target.to_string(),
17933 },
17934 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
17935 consumer_identity: opener.map(|module_id| ConsumerIdentity {
17936 module_id: module_id.to_string(),
17937 launch_nonce: nonce(module_id),
17938 }),
17939 consumer_capabilities: None,
17940 role_versions: None,
17941 admission_facts: None,
17942 scope,
17943 })
17944 .unwrap();
17945 let frame = Frame::build(
17946 FrameType::Request,
17947 control_flags(),
17948 0,
17949 0,
17950 self.next_connection,
17951 body,
17952 )
17953 .unwrap();
17954 (ctx, rx, frame)
17955 }
17956
17957 async fn refused(
17959 &mut self,
17960 opener: Option<&str>,
17961 target: &str,
17962 scope: Option<ScopeSelector>,
17963 ) -> String {
17964 self.refusal_body(opener, target, scope).await["code"]
17965 .as_str()
17966 .unwrap()
17967 .to_string()
17968 }
17969
17970 async fn refusal_body(
17971 &mut self,
17972 opener: Option<&str>,
17973 target: &str,
17974 scope: Option<ScopeSelector>,
17975 ) -> Value {
17976 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
17977 let replies = tokio::time::timeout(
17978 Duration::from_secs(2),
17979 self.handler.handle_control_frame(&ctx, frame),
17980 )
17981 .await
17982 .expect("the open must be refused before waiting for a bind ack")
17983 .unwrap();
17984 assert_eq!(replies.len(), 1, "{replies:?}");
17985 assert_eq!(replies[0].header.ty, FrameType::Error);
17986 let (_, module_rx) = self.modules.get_mut(target).unwrap();
17987 assert!(
17988 module_rx.try_recv().is_err(),
17989 "a refused open relays nothing"
17990 );
17991 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
17992 parse_error(&replies[0])
17993 }
17994
17995 async fn relayed(
17997 &mut self,
17998 opener: Option<&str>,
17999 target: &str,
18000 scope: Option<ScopeSelector>,
18001 ) -> Relayed {
18002 self.relayed_with_legacy_root(opener, target, scope, None)
18003 .await
18004 }
18005
18006 async fn relayed_with_legacy_root(
18007 &mut self,
18008 opener: Option<&str>,
18009 target: &str,
18010 scope: Option<ScopeSelector>,
18011 legacy_root: Option<&std::path::Path>,
18012 ) -> Relayed {
18013 let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
18014 if let Some(root) = legacy_root {
18015 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
18016 body["identity"]["project_root"] = serde_json::json!(root);
18017 frame.body = serde_json::to_vec(&body).unwrap();
18018 }
18019 let handler = self.handler.clone();
18020 let task_ctx = ctx.clone();
18021 let task = tokio::spawn(async move {
18022 handler
18023 .handle_control_frame(&task_ctx, frame)
18024 .await
18025 .unwrap()
18026 });
18027 let (_, module_rx) = self.modules.get_mut(target).unwrap();
18028 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
18029 .await
18030 .expect("the target receives the relayed route.bind")
18031 .unwrap()
18032 .frame;
18033 Relayed {
18034 target: target.to_string(),
18035 client: ctx,
18036 client_rx: rx,
18037 task,
18038 bind,
18039 }
18040 }
18041
18042 async fn ack(&self, relayed: &Relayed) {
18043 let (module, _) = &self.modules[&relayed.target];
18044 self.handler
18045 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
18046 .await
18047 .unwrap();
18048 }
18049
18050 async fn bound(
18052 &mut self,
18053 opener: Option<&str>,
18054 target: &str,
18055 scope: Option<ScopeSelector>,
18056 ) -> Bound {
18057 let relayed = self.relayed(opener, target, scope).await;
18058 self.commit_relayed(relayed).await
18059 }
18060
18061 async fn bound_with_legacy_root(
18062 &mut self,
18063 opener: Option<&str>,
18064 target: &str,
18065 scope: Option<ScopeSelector>,
18066 root: &std::path::Path,
18067 ) -> Bound {
18068 let relayed = self
18069 .relayed_with_legacy_root(opener, target, scope, Some(root))
18070 .await;
18071 self.commit_relayed(relayed).await
18072 }
18073
18074 async fn commit_relayed(&self, relayed: Relayed) -> Bound {
18075 self.ack(&relayed).await;
18076 let Relayed {
18077 target,
18078 client,
18079 mut client_rx,
18080 task,
18081 bind,
18082 } = relayed;
18083 assert!(
18084 task.await.unwrap().is_empty(),
18085 "the open is answered by commit"
18086 );
18087 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
18088 Bound {
18089 target,
18090 client,
18091 client_rx,
18092 channel,
18093 epoch,
18094 bind,
18095 }
18096 }
18097
18098 fn live(&self, route: &Bound) -> bool {
18099 matches!(
18100 self.forwarding
18101 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18102 .unwrap(),
18103 DataRoute::Client(DataRouteState::Bound(_))
18104 )
18105 }
18106 }
18107
18108 struct Relayed {
18109 target: String,
18110 client: RouteCtx,
18111 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18112 task: tokio::task::JoinHandle<Vec<Frame>>,
18113 bind: Frame,
18114 }
18115
18116 struct Bound {
18117 target: String,
18118 client: RouteCtx,
18119 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18120 channel: u16,
18121 epoch: u32,
18122 bind: Frame,
18123 }
18124
18125 impl Bound {
18126 fn closed_reason(&mut self) -> RouteCloseReason {
18129 let mut reason = None;
18130 let mut goodbye = false;
18131 while let Ok(outbound) = self.client_rx.try_recv() {
18132 let frame = outbound.frame;
18133 match frame.header.ty {
18134 FrameType::Goodbye => {
18135 assert_eq!(
18136 (frame.header.channel, frame.header.epoch),
18137 (self.channel, self.epoch)
18138 );
18139 goodbye = true;
18140 }
18141 FrameType::Push => {
18142 let ClientControlPush::RouteClosed {
18143 reason: r,
18144 module_id,
18145 ..
18146 } = serde_json::from_slice(&frame.body).unwrap()
18147 else {
18148 panic!("unexpected push");
18149 };
18150 assert_eq!(module_id, self.target);
18151 reason = Some(r);
18152 }
18153 other => panic!("unexpected frame {other:?}"),
18154 }
18155 }
18156 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
18157 reason.expect("the client is told why the route closed")
18158 }
18159
18160 fn untouched(&mut self) -> bool {
18161 self.client_rx.try_recv().is_err()
18162 }
18163
18164 fn stamp(&self) -> Option<ScopeStamp> {
18165 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
18166 ModuleControlRequest::RouteBind { scope, .. } => scope,
18167 other => panic!("expected a route.bind, got {other:?}"),
18168 }
18169 }
18170 }
18171
18172 #[tokio::test]
18173 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
18174 ) {
18175 let mut rig = rig().await;
18176 let mut record = session(1);
18177 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
18178 record.child_owners = vec![Principal::Reserved {
18179 module_id: MAGIC.to_string(),
18180 }];
18181 rig.sync(vec![record]).await;
18182 let scope = || Some(rig_selector("s", Some(1)));
18183
18184 rig.bound(Some(OWNER), PLEXUS, scope()).await;
18187 rig.bound(Some(AFT), OTHER, scope()).await;
18188 rig.bound(Some(BROCA), PLEXUS, scope()).await;
18189
18190 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
18194 assert_eq!(
18195 rig.refused(opener, target, scope()).await,
18196 error_codes::SCOPE_NOT_CARRIER,
18197 "{opener:?} -> {target}"
18198 );
18199 }
18200 }
18201
18202 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18203 ScopeSelector::new(
18204 Principal::Reserved {
18205 module_id: OWNER.to_string(),
18206 },
18207 scope_ref.to_string(),
18208 scope_epoch,
18209 )
18210 }
18211
18212 #[tokio::test]
18213 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
18214 let mut rig = rig().await;
18215 rig.sync(vec![session(1)]).await;
18216 for opener in [OWNER, AFT] {
18217 assert_eq!(
18218 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
18219 .await,
18220 error_codes::SCOPE_EPOCH_REQUIRED,
18221 "{opener}"
18222 );
18223 }
18224 }
18225
18226 #[tokio::test]
18227 async fn admission_separates_not_synced_not_live_and_ended() {
18228 let mut rig = rig().await;
18229 let code = rig
18231 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18232 .await;
18233 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
18234 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
18235
18236 let ghost = ScopeSelector::new(
18238 Principal::Reserved {
18239 module_id: "ghost".to_string(),
18240 },
18241 "s".to_string(),
18242 Some(1),
18243 );
18244 assert_eq!(
18245 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
18246 error_codes::SCOPE_NOT_LIVE
18247 );
18248
18249 rig.sync(vec![session(2)]).await;
18250 assert_eq!(
18251 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
18252 .await,
18253 error_codes::SCOPE_NOT_LIVE
18254 );
18255 for epoch in [1, 3] {
18256 assert_eq!(
18257 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
18258 .await,
18259 error_codes::SCOPE_ENDED,
18260 "epoch {epoch}"
18261 );
18262 }
18263 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
18265 .await;
18266 }
18267
18268 #[tokio::test]
18269 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
18270 let mut rig = rig().await;
18271 rig.sync(vec![session(1)]).await;
18272 let route = rig
18273 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18274 .await;
18275 let stamp = route.stamp().expect("a scoped bind carries the stamp");
18276 assert_eq!(stamp.scope_ref, "s");
18277 assert_eq!(stamp.scope_epoch, 1);
18278 assert_eq!(stamp.kind, ScopeKind::Head);
18279 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
18280 assert!(stamp.attributes.delegates);
18281 assert!(stamp.owner_authorized);
18282 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18283 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
18284
18285 let (broca, mut broca_rx) = wide_ctx(50);
18288 hello_via_sink(
18289 &rig.handler,
18290 &broca,
18291 &mut broca_rx,
18292 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
18293 )
18294 .await;
18295 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
18296 .await
18297 .unwrap();
18298 let own = ScopeSelector::new(
18299 Principal::Reserved {
18300 module_id: BROCA.to_string(),
18301 },
18302 "b".to_string(),
18303 Some(1),
18304 );
18305 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
18306 assert!(!route.stamp().unwrap().owner_authorized);
18307 }
18308
18309 #[tokio::test]
18310 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
18311 let mut rig = rig().await;
18312 let mut record = head("s", 1);
18313 record.carriers = vec![carrier(AFT, None)];
18314 let flow_id = "Flow:run-7/step_2!~";
18315 record.attributes.flow_id = Some(flow_id.to_string());
18316 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
18317 .await
18318 .unwrap();
18319 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
18320 panic!("not a sync reply");
18321 };
18322 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
18323 let route = rig
18324 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18325 .await;
18326 assert!(rig.live(&route), "the stamped bind committed");
18327 let stamp = route.stamp().expect("a flow scope carries a stamp");
18328 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
18329 assert_eq!(stamp.attributes.agent_id, None);
18330 assert!(!stamp.attributes.delegates);
18331 assert!(stamp.owner_authorized);
18332 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18333 assert_eq!(unscoped.stamp(), None);
18334 }
18335
18336 #[tokio::test]
18337 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
18338 let mut rig = rig_with_flow_support(false).await;
18339 let mut record = session(1);
18340 record.attributes.flow_id = Some("flow:7".to_string());
18341 rig.sync(vec![record]).await;
18342 for opener in [OWNER, AFT] {
18343 let body = rig
18344 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18345 .await;
18346 assert_eq!(body["code"], "target_flow_unsupported");
18347 let message = body["message"].as_str().unwrap();
18348 for required in [PLEXUS, "flow-scopes/v1"] {
18349 assert!(message.contains(required), "{message}");
18350 }
18351 }
18352 }
18353
18354 #[tokio::test]
18355 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
18356 let mut rig = rig_with_flow_support(true).await;
18357 let mut record = session(1);
18358 record.attributes.flow_id = Some("flow:7".to_string());
18359 rig.sync(vec![record]).await;
18360 for opener in [OWNER, AFT] {
18361 let route = rig
18362 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18363 .await;
18364 assert!(rig.live(&route));
18365 assert_eq!(
18366 route.stamp().unwrap().attributes.flow_id.as_deref(),
18367 Some("flow:7")
18368 );
18369 }
18370 }
18371
18372 #[tokio::test]
18373 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
18374 use std::future::Future;
18375
18376 let mut rig = rig_with_flow_support(true).await;
18377 let mut record = session(1);
18378 record.attributes.flow_id = Some("flow:7".to_string());
18379 rig.sync(vec![record]).await;
18380 let (client, mut client_rx, frame) =
18381 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
18382 for _ in 0..64 {
18385 client.egress.try_send(route_bind_ack(1)).unwrap();
18386 }
18387 let handler = rig.handler.clone();
18388 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18389 std::future::poll_fn(|cx| {
18390 assert!(open.as_mut().poll(cx).is_pending());
18391 std::task::Poll::Ready(())
18392 })
18393 .await;
18394 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18395
18396 let old_connection = rig.modules[PLEXUS].0.connection_id;
18397 rig.handler.cleanup_connection(old_connection).unwrap();
18398 let (replacement, mut replacement_rx) = wide_ctx(200);
18399 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
18400 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18401 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18402 let hello = Frame::build(
18403 FrameType::Hello,
18404 control_flags(),
18405 0,
18406 0,
18407 200,
18408 serde_json::to_vec(&body).unwrap(),
18409 )
18410 .unwrap();
18411 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
18412
18413 client_rx.try_recv().unwrap();
18414 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18415 .await
18416 .expect("the replacement is refused without waiting for a bind ack")
18417 .unwrap();
18418 assert_eq!(replies.len(), 1);
18419 let body = parse_error(&replies[0]);
18420 assert_eq!(body["code"], "target_flow_unsupported");
18421 for required in [PLEXUS, "flow-scopes/v1"] {
18422 assert!(body["message"].as_str().unwrap().contains(required));
18423 }
18424 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
18425 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18426 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18427 assert!(rig
18428 .handler
18429 .registry
18430 .get_module_by_connection(replacement.connection_id)
18431 .unwrap()
18432 .is_some());
18433 }
18434
18435 #[tokio::test]
18436 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
18437 use std::future::Future;
18438 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
18439 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18440 const NEXT_HASH: &str =
18441 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18442 let mut rig = rig().await;
18443 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
18444 record.carriers.push(ScopeCarrier::new(Principal::Direct));
18445 rig.sync(vec![record.clone()]).await;
18446 let (client, mut rx, frame) = rig.open_frame(
18447 None,
18448 PLEXUS,
18449 Some(
18450 rig.selector("s", Some(1))
18451 .with_bind_proof(Some(PROOF.into())),
18452 ),
18453 );
18454 for _ in 0..64 {
18457 client.egress.try_send(route_bind_ack(1)).unwrap();
18458 }
18459 let handler = rig.handler.clone();
18460 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18461 std::future::poll_fn(|cx| {
18462 assert!(open.as_mut().poll(cx).is_pending());
18463 std::task::Poll::Ready(())
18464 })
18465 .await;
18466 record.bind_proof_sha256 = Some(NEXT_HASH.into());
18467 rig.sync(vec![record]).await;
18468 rx.try_recv().unwrap();
18469 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18470 .await
18471 .unwrap()
18472 .unwrap();
18473 assert_eq!(
18474 parse_error(&replies[0])["code"],
18475 error_codes::SCOPE_BIND_PROOF_MISMATCH
18476 );
18477 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
18478 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18479 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18480 }
18481
18482 #[tokio::test]
18483 async fn scope_bind_proof_reserved_openers_ignore_proof() {
18484 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18485 let mut rig = rig().await;
18486 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18487 .await;
18488 for opener in [OWNER, AFT] {
18489 for proof in [None, Some("malformed+proof=")] {
18490 let route = rig
18491 .bound(
18492 Some(opener),
18493 PLEXUS,
18494 Some(
18495 rig.selector("s", Some(1))
18496 .with_bind_proof(proof.map(str::to_string)),
18497 ),
18498 )
18499 .await;
18500 assert!(rig.live(&route));
18501 }
18502 }
18503 }
18504
18505 #[tokio::test]
18506 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
18507 use std::future::Future;
18508 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18509 const NEXT_HASH: &str =
18510 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18511 let mut rig = rig().await;
18512 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18513 .await;
18514 let (client, mut rx, frame) =
18515 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
18516 for _ in 0..64 {
18517 client.egress.try_send(route_bind_ack(1)).unwrap();
18518 }
18519 let handler = rig.handler.clone();
18520 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18521 std::future::poll_fn(|cx| {
18522 assert!(open.as_mut().poll(cx).is_pending());
18523 std::task::Poll::Ready(())
18524 })
18525 .await;
18526 rig.sync(vec![
18527 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
18528 ])
18529 .await;
18530 rx.try_recv().unwrap();
18531 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18532 let bind = std::future::poll_fn(|cx| {
18533 assert!(open.as_mut().poll(cx).is_pending());
18534 module_rx.poll_recv(cx)
18535 })
18536 .await
18537 .unwrap()
18538 .frame;
18539 handler
18540 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
18541 .await
18542 .unwrap();
18543 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18544 .await
18545 .unwrap()
18546 .unwrap();
18547 assert_eq!(replies.len(), 1);
18548 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
18551 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18552 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
18553 let route = rig
18554 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
18555 .await;
18556 assert!(rig.live(&route));
18557 }
18558
18559 #[tokio::test]
18560 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
18561 {
18562 let mut rig = rig_with_flow_support(false).await;
18563 rig.sync(vec![session(1)]).await;
18564 let route = rig
18565 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18566 .await;
18567 assert!(rig.live(&route));
18568 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
18569 let mut record = session(1);
18570 record.attributes.flow_id = Some("flow:7".to_string());
18571 rig.sync(vec![record]).await;
18572 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
18573 assert!(rig.live(&unscoped));
18574 assert_eq!(unscoped.stamp(), None);
18575 }
18576
18577 #[tokio::test]
18578 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
18579 let mut rig = rig().await;
18580 let mut record = session(1);
18581 record.attributes.flow_id = Some("flow:7".to_string());
18582 rig.sync(vec![record.clone()]).await;
18583 let before = rig.forwarding.published_scope_tag(OWNER, "s");
18584 let mut owner_route = rig
18585 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18586 .await;
18587 let mut carrier_route = rig
18588 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
18589 .await;
18590 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18591 rig.sync(vec![record.clone()]).await;
18592 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18593 assert!(rig.live(&owner_route) && owner_route.untouched());
18594 assert!(rig.live(&carrier_route) && carrier_route.untouched());
18595
18596 record.attributes.flow_id = Some("flow:8".to_string());
18597 rig.sync(vec![record]).await;
18598 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
18599 let before = before.unwrap();
18600 assert_eq!(after.scope_epoch, before.scope_epoch);
18601 assert!(after.version > before.version);
18602 for route in [&mut owner_route, &mut carrier_route] {
18603 assert!(!rig.live(route));
18604 assert_eq!(
18605 route.closed_reason(),
18606 RouteCloseReason::ScopeDelegationChanged
18607 );
18608 }
18609 assert!(rig.live(&unscoped) && unscoped.untouched());
18610 for target in [PLEXUS, OTHER] {
18613 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
18614 let goodbye = module_rx
18615 .try_recv()
18616 .expect("the provider sees the drain")
18617 .frame;
18618 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18619 assert!(module_rx.try_recv().is_err());
18620 }
18621 let rebound = rig
18622 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18623 .await;
18624 assert_eq!(
18625 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
18626 Some("flow:8")
18627 );
18628 }
18629
18630 #[tokio::test]
18635 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
18636 let mut rig = rig().await;
18637 rig.sync(vec![session(1)]).await;
18638 let mut cotenant = rig
18639 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
18640 .await;
18641
18642 let mut changed = session(1);
18643 changed.child_owners.push(Principal::Reserved {
18644 module_id: MAGIC.to_string(),
18645 });
18646 let mut ended = None;
18647 for (code, next) in [
18648 (error_codes::SCOPE_CHANGED, vec![changed]),
18649 (error_codes::SCOPE_ENDED, Vec::new()),
18650 ] {
18651 let relayed = rig
18652 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18653 .await;
18654 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
18655 ended = Some(next.is_empty());
18656 rig.sync(next).await;
18657 rig.ack(&relayed).await;
18658 let replies = relayed.task.await.unwrap();
18659 assert_eq!(replies.len(), 1, "{replies:?}");
18660 assert_eq!(parse_error(&replies[0])["code"], code);
18661 assert_eq!(
18662 subc_protocol::error_codes::is_retryable_route_open(code),
18663 code == error_codes::SCOPE_CHANGED
18664 );
18665 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18666 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18668 let mut goodbyes = Vec::new();
18671 while let Ok(outbound) = plexus_rx.try_recv() {
18672 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
18673 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
18674 }
18675 assert!(
18676 goodbyes.contains(&(bind_channel, bind_epoch)),
18677 "{goodbyes:?}"
18678 );
18679 assert!(rig
18680 .handler
18681 .registry
18682 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
18683 .unwrap()
18684 .is_some());
18685 }
18686 assert_eq!(ended, Some(true));
18687 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
18690 }
18691
18692 #[tokio::test]
18695 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
18696 struct Case {
18697 name: &'static str,
18698 change: fn(&mut ScopeRecord),
18699 closed: [Option<RouteCloseReason>; 4],
18702 }
18703 use RouteCloseReason::*;
18704 let cases = [
18705 Case {
18706 name: "a carrier entry removed",
18707 change: |r| {
18708 r.carriers.retain(|c| {
18709 c.principal
18710 != Principal::Reserved {
18711 module_id: AFT.to_string(),
18712 }
18713 })
18714 },
18715 closed: [None, Some(ScopeCarrierRemoved), None, None],
18716 },
18717 Case {
18718 name: "a target removed from a carrier",
18719 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
18720 closed: [None, None, None, Some(ScopeCarrierRemoved)],
18721 },
18722 Case {
18723 name: "a bare carrier narrowed to targets",
18724 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
18725 closed: [None, Some(ScopeCarrierRemoved), None, None],
18726 },
18727 Case {
18728 name: "delegates turned off",
18729 change: |r| r.attributes.delegates = false,
18730 closed: [Some(ScopeDelegationChanged); 4],
18731 },
18732 Case {
18733 name: "agent_id changed",
18734 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
18735 closed: [Some(ScopeDelegationChanged); 4],
18736 },
18737 Case {
18738 name: "a carrier added, child owners changed, the record re-sent",
18739 change: |r| {
18740 r.carriers.push(carrier(MAGIC, None));
18741 r.child_owners.push(Principal::Reserved {
18742 module_id: MAGIC.to_string(),
18743 });
18744 },
18745 closed: [None; 4],
18746 },
18747 Case {
18748 name: "a target added",
18749 change: |r| {
18750 r.carriers[1]
18751 .targets
18752 .as_mut()
18753 .unwrap()
18754 .push("third".to_string())
18755 },
18756 closed: [None; 4],
18757 },
18758 Case {
18759 name: "delegates turned on",
18760 change: |r| r.attributes.delegates = true,
18761 closed: [None; 4],
18762 },
18763 ];
18764 for case in cases {
18765 let mut rig = rig().await;
18766 rig.sync(vec![session(1)]).await;
18767 let scope = || Some(rig_selector("s", Some(1)));
18768 let mut routes = [
18769 rig.bound(Some(OWNER), PLEXUS, scope()).await,
18770 rig.bound(Some(AFT), PLEXUS, scope()).await,
18771 rig.bound(Some(BROCA), PLEXUS, scope()).await,
18772 rig.bound(Some(BROCA), OTHER, scope()).await,
18773 ];
18774 let mut record = session(1);
18775 (case.change)(&mut record);
18776 rig.sync(vec![record]).await;
18777 for (index, expected) in case.closed.iter().enumerate() {
18778 let route = &mut routes[index];
18779 match expected {
18780 Some(reason) => {
18781 assert!(!rig.live(route), "{}: route {index} still live", case.name);
18782 assert_eq!(
18783 route.closed_reason(),
18784 *reason,
18785 "{}: route {index}",
18786 case.name
18787 );
18788 }
18789 None => {
18790 assert!(rig.live(route), "{}: route {index} closed", case.name);
18791 assert!(
18792 route.untouched(),
18793 "{}: route {index} was told something",
18794 case.name
18795 );
18796 }
18797 }
18798 }
18799 }
18800 }
18801
18802 #[tokio::test]
18803 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
18804 for next in [Vec::new(), vec![session(2)]] {
18806 let mut rig = rig().await;
18807 rig.sync(vec![session(1)]).await;
18808 let mut route = rig
18809 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18810 .await;
18811 rig.sync(next).await;
18812 assert!(!rig.live(&route));
18813 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
18814 }
18815
18816 let mut rig = rig().await;
18819 let mut child = session(1);
18820 child.scope_ref = "child".to_string();
18821 child.kind = ScopeKind::Worker;
18822 child.parent = Some(ScopeParent::new(
18823 Principal::Reserved {
18824 module_id: OWNER.to_string(),
18825 },
18826 "s".to_string(),
18827 1,
18828 ));
18829 rig.sync(vec![session(1), child.clone()]).await;
18830 let mut child_route = rig
18831 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
18832 .await;
18833 assert_eq!(
18834 child_route.stamp().unwrap().parent_state,
18835 Some(ParentState::Linked)
18836 );
18837 rig.sync(vec![child]).await;
18838 assert!(!rig.live(&child_route));
18839 assert_eq!(
18840 child_route.closed_reason(),
18841 RouteCloseReason::ScopeParentEnded
18842 );
18843 }
18844
18845 #[tokio::test]
18846 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
18847 ) {
18848 let mut rig = rig().await;
18849 rig.sync(vec![session(1)]).await;
18850 let mut route = rig
18851 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18852 .await;
18853 let before = rig.forwarding.published_scope_tag(OWNER, "s");
18854 rig.sync(vec![session(1)]).await;
18855 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18856 assert!(rig.live(&route) && route.untouched());
18857
18858 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
18862 .forwarding
18863 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18864 .unwrap()
18865 else {
18866 panic!("the route is bound");
18867 };
18868 binding.flow.acquire_tagged(9, false).await.unwrap();
18869 let mut widened = session(1);
18870 widened.carriers.push(carrier(MAGIC, None));
18871 rig.sync(vec![widened]).await;
18872 assert!(rig.live(&route) && route.untouched());
18873 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18874 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
18875 assert_eq!(binding.flow.in_flight(), 1);
18878 binding
18879 .flow
18880 .acquire_tagged(10, false)
18881 .await
18882 .expect("the flow is still open");
18883 }
18884
18885 #[tokio::test]
18888 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
18889 let mut rig = rig().await;
18890 rig.sync(vec![session(1)]).await;
18891 let mut on_incumbent = rig
18892 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18893 .await;
18894
18895 let (candidate, _candidate_rx) = wide_ctx(9);
18898 let registration = rig
18899 .handler
18900 .registry
18901 .register_candidate_with_control_ops(
18902 manifest(PLEXUS, PROTOCOL_VERSION),
18903 PROTOCOL_VERSION,
18904 candidate.connection_id,
18905 module_baseline_control_ops(),
18906 )
18907 .unwrap();
18908 rig.forwarding
18909 .register_candidate_module_connection(
18910 candidate.connection_id,
18911 PLEXUS.to_string(),
18912 PROTOCOL_VERSION,
18913 manifest_concurrency(®istration.manifest),
18914 candidate.egress.clone(),
18915 )
18916 .unwrap();
18917 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
18918 rig.handler
18919 .registry
18920 .promote_candidate(PLEXUS)
18921 .unwrap()
18922 .unwrap();
18923 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
18924
18925 rig.sync(Vec::new()).await;
18926 assert!(!rig.live(&on_incumbent));
18927 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
18928 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18929 let goodbye = incumbent_rx
18930 .try_recv()
18931 .expect("the superseded endpoint is told")
18932 .frame;
18933 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18934 }
18935 }
18936}
18937
18938#[cfg(test)]
18939mod concurrency_default_exposure_tests {
18940 use super::*;
18941
18942 fn hello_body(role_json: &str) -> Vec<u8> {
18943 format!(
18944 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
18945 )
18946 .into_bytes()
18947 }
18948
18949 fn manifest_from(body: &[u8]) -> ModuleManifest {
18950 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
18951 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
18952 .expect("manifest parses")
18953 }
18954
18955 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
18956
18957 #[test]
18958 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
18959 let body = hello_body(&format!(
18960 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
18961 ));
18962 let manifest = manifest_from(&body);
18963 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18966 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
18967 }
18968
18969 #[test]
18970 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
18971 let body = hello_body(&format!(
18972 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
18973 ));
18974 let manifest = manifest_from(&body);
18975 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18976 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
18977 }
18978
18979 #[test]
18980 fn non_management_roles_are_never_reported() {
18981 let body = hello_body(
18982 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
18983 );
18984 let manifest = manifest_from(&body);
18985 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
18986 }
18987}